一、问题
我们在使用 HttpClient 访问一个网站时，通常的做法是建立连接，访问，断开连接，若我们的访问量非常的大，那么就会无数次重复上述的步骤。但是对于同一个网站的访问，我们其实可以做到不断开连接，续用上一次的连接，毕竟断开连接，再重新连接，是相当耗时耗费资源的。
二、解决思路
一开始我的想法是通过一个阻塞队列，将创建的 HttpClient 实例放入队列中，从而形成一个池，每次访问结束时，不再关闭 HttpClient 实例，而是将其返回到池中。
但是这样做，只是做到了 HttpClient 实例的复用，依然没有解决一开始的问题，再次访问依然需要重新建立连接。
后来，通过查阅资料，发现如下解决办法，具体代码见下：
package com.yuedu.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.*;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.*;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import javax.net.ssl.*;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.UnsupportedEncodingException;
import java.net.UnknownHostException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* HttpClient工具类
* @author 咸鱼
* @date 2019-03-18 17:12
*/
public class HttpClientUtil {
/**
* 超时时间
*/
private static final int TIMEOUT = 30 * 1000;
/**
* 最大连接数
*/
private static final int MAX_TOTAL = 200;
/**
* 每个路由的默认最大连接数
*/
private static final int MAX_PER_ROUTE = 40;
/**
* 目标主机的最大连接数
*/
private static final int MAX_ROUTE = 100;
/**
* 访问失败时最大重试次数
*/
private static final int MAX_RETRY_TIME = 5;
private static CloseableHttpClient httpClient = null;
private static final Object SYNC_LOCK = new Object();
private static final String DEFAULT_CHARSET = "UTF-8";
private static void config(HttpRequestBase httpRequestBase) {
//配置请求的超时时间
RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(TIMEOUT)
.setConnectTimeout(TIMEOUT)
.setSocketTimeout(TIMEOUT)
.build();
httpRequestBase.setConfig(requestConfig);
}
/**
* 获取HttpClient对象
*/
private static CloseableHttpClient getHttpClient(String url) throws NoSuchAlgorithmException, KeyManagementException {
String hostName = url.split("/")[2];
int port = 80;
if (hostName.contains(":")) {
String[] attr = hostName.split(":");
hostName = attr[0];
port = Integer.parseInt(attr[1]);
}
if (httpClient == null) {
synchronized (SYNC_LOCK) {
if (httpClient == null) {
httpClient = createHttpClient(MAX_TOTAL, MAX_PER_ROUTE, MAX_ROUTE, hostName, port);
}
}
}
return httpClient;
}
/**
* 创建HttpClient对象
*/
private static CloseableHttpClient createHttpClient(int maxTotal, int maxPerRoute, int maxRoute,
String hostName, int port) throws KeyManagementException, NoSuchAlgorithmException {
PlainConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(createIgnoreVerifySSL());
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", plainsf)
.register("https", sslsf)
.build();
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
//增加最大连接数
cm.setMaxTotal(maxTotal);
//增加每个路由的默认最大连接
cm.setDefaultMaxPerRoute(maxPerRoute);
//增加目标主机的最大连接数
cm.setMaxPerRoute(new HttpRoute(new HttpHost(hostName, port)), maxRoute);
//请求重试
HttpRequestRetryHandler httpRequestRetryHandler = (exception, executionCount, context) -> {
//若重试5次，放弃
if (executionCount >= MAX_RETRY_TIME) {
return false;
}
//若服务器丢掉了连接，那就重试
if (exception instanceof NoHttpResponseException) {
return true;
}
//不重试SSL握手异常
if (exception instanceof SSLHandshakeException) {
return false;
}
//超时
if (exception instanceof InterruptedIOException) {
return false;
}
//目标服务器不可达
if (exception instanceof UnknownHostException) {
return false;
}
//SSL握手异常
if (exception instanceof SSLException) {
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
//若请求时幂等的，就再次尝试
return !(request instanceof HttpEntityEnclosingRequest);
};
return HttpClients.custom().setConnectionManager(cm)
.setRetryHandler(httpRequestRetryHandler)
.build();
}
/**
* HttpClient配置SSL绕过https证书（因为我的网站是有https证书的，所以在访问https网站时，会自动读取我的证书，
* 和目标网站不符，会报错），所以这里需要绕过https证书
*/
private static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
SSLContext sslContext = SSLContext.getInstance("SSLv3");
// 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
X509TrustManager trustManager = new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
}
public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
}
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
};
sslContext.init(null, new TrustManager[] {trustManager}, null);
return sslContext;
}
private static void setPostParams(HttpPost httpPost, Map<String, Object> params) {
List<NameValuePair> nameValuePairs = new ArrayList<>();
params.forEach((key, value) -> nameValuePairs.add(new BasicNameValuePair(key, value.toString())));
try {
httpPost.setEntity(new UrlEncodedFormEntity(nameValuePairs, DEFAULT_CHARSET));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
/**
* post请求，默认编码格式为UTF-8
* @param url 请求地址
* @param params 请求参数
* @return 响应正文
*/
public static String doPost(String url, Map<String, Object> params) {
return doPost(url, params, DEFAULT_CHARSET);
}
/**
* post请求
* @param url 请求地址
* @param params 请求参数
* @param charset 字符编码
* @return 响应正文
*/
public static String doPost(String url, Map<String, Object> params, String charset) {
HttpPost httpPost = new HttpPost(url);
config(httpPost);
setPostParams(httpPost, params);
return getResponse(url, httpPost, charset);
}
/**
* get请求，默认编码UTF-8
* @param url 请求地址
* @return 响应正文
*/
public static String doGet(String url) {
return doGet(url, DEFAULT_CHARSET);
}
/**
* get请求
* @param url 请求地址
* @param charset 字符编码
* @return 响应正文
*/
public static String doGet(String url, String charset) {
HttpGet httpGet = new HttpGet(url);
config(httpGet);
return getResponse(url, httpGet, charset);
}
/**
* 发起请求，获取响应
* @param url 请求地址
* @param httpRequest 请求对象
* @param charset 字符编码
* @return 响应正文
*/
private static String getResponse(String url, HttpRequestBase httpRequest, String charset) {
CloseableHttpResponse response = null;
try {
response = getHttpClient(url).execute(httpRequest, HttpClientContext.create());
HttpEntity httpEntity = response.getEntity();
String result = EntityUtils.toString(httpEntity, charset);
EntityUtils.consume(httpEntity);
return result;
} catch (IOException | NoSuchAlgorithmException | KeyManagementException e) {
log.error("网络访问异常！", e);
} finally {
try {
if (response != null) {
response.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
}
