对象的序列化和反序列化
<dependency>
<groupId>org.javatuples</groupId>
<artifactId>javatuples</artifactId>
<version>1.2</version>
</dependency>
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.TreeNode;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.TextNode;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
public final class Json {
private static final Logger LOGGER = LoggerFactory.getLogger(Json.class);
private static final String EMPTY_JSON_OBJECT = "{}";
public static final ObjectMapper MAPPER = new ObjectMapper();
/**
* 日期格式化参数 格式 yyyy-MM-dd HH:mm:ss
*/
public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
static {
SimpleModule simpleModule = new SimpleModule();
simpleModule.addDeserializer(LocalDateTime.class, new LocalDatetimeDeserializer());
simpleModule.addSerializer(LocalDateTime.class, new LocalDatetimeSerializer());
simpleModule.addSerializer(Timestamp.class, new TimestampSerializer());
MAPPER.registerModule(simpleModule);
MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
MAPPER.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE);
}
/**
* 对象转换成Json
*
* @param object Object
* @return String
*/
public static String toJson(Object object) {
StringWriter sw = new StringWriter();
if (Objects.isNull(object)) {
return EMPTY_JSON_OBJECT;
}
try {
MAPPER.writeValue(MAPPER.getFactory().createGenerator(sw), object);
} catch (Exception ex) {
LOGGER.error("Object to json occur error, detail:", ex);
}
return sw.toString();
}
/**
* Json转换成对象
*
* @param jsonString json
* @param tClass 类型
* @param <T> 泛型参数
* @return T
*/
public static <T> T toObject(String jsonString, Class<T> tClass) {
if (Objects.isNull(jsonString) || jsonString.trim().isEmpty()) {
return null;
}
try {
return MAPPER.readValue(jsonString, tClass);
} catch (Exception ex) {
LOGGER.error("Json to object occur error, detail:", ex);
}
return null;
}
/**
* Json转换成对象
*
* @param fileName 文件名称
* @param tClass 类型
* @param <T> 泛型参数
* @return T
*/
public static <T> T fileToObject(String fileName, Class<T> tClass) {
if (Objects.isNull(fileName) || fileName.trim().isEmpty()) {
return null;
}
try {
return MAPPER.readValue(new File(fileName), tClass);
} catch (Exception ex) {
LOGGER.error("Json to object occur error, detail:", ex);
}
return null;
}
/**
* Json转换成对象
*
* @param <T> 泛型
* @param jsonString json
* @param reference 类型引用
* @return T
*/
public static <T> T toObject(String jsonString, TypeReference<T> reference) {
if (Objects.isNull(jsonString) || jsonString.trim().isEmpty()) {
return null;
}
try {
return MAPPER.readValue(jsonString, reference);
} catch (Exception ex) {
LOGGER.error("Json to object occur error, detail:", ex);
}
return null;
}
/**
* 输入流转换成对象
*
* @param <T> 泛型
* @param stream 输入流
* @param reference 类型引用
* @return T
*/
public static <T> T toObject(InputStream stream, TypeReference<T> reference) {
if (Objects.isNull(stream)) {
return null;
}
try {
return MAPPER.readValue(stream, reference);
} catch (Exception ex) {
LOGGER.error("Json to object occur error, detail:", ex);
}
return null;
}
/**
* 输入流转换成对象
*
* @param <T> 泛型
* @param stream 输入流
* @param tClass 类型引用
* @return T
*/
public static <T> T toObject(InputStream stream, Class<T> tClass) {
if (Objects.isNull(stream)) {
return null;
}
try {
return MAPPER.readValue(stream, tClass);
} catch (Exception ex) {
LOGGER.error("Json to object occur error, detail:", ex);
}
return null;
}
/**
* JSON字符串转Map
*
* @param json JSON字符串
* @return map
*/
public static Map<String, Object> toMap(String json) {
if (Objects.nonNull(json) && !json.isEmpty()) {
String tmp = json.trim();
if (tmp.charAt(0) == '{' && tmp.charAt(tmp.length() - 1) == '}') {
return toObject(json, new TypeReference<Map<String, Object>>() {
});
}
}
return null;
}
private final static Map<String, Map<String, Pair<Field, Method>>> dynamicTypeFieldPairsCache = new HashMap<>();
/**
* LocalDatetime类型序列化
*/
public static class LocalDatetimeSerializer extends JsonSerializer<LocalDateTime> {
/**
* 序列化LocalDateTime
*
* @param localDateTime LocalDateTime
* @param jsonGenerator JsonGenerator
* @param serializerProvider SerializerProvider
* @throws IOException IOException
*/
@Override
public void serialize(LocalDateTime localDateTime, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeString(DATE_TIME_FORMATTER.format(localDateTime));
}
/**
* 获取LocalDateTime class
*
* @return LocalDateTime.class
*/
@Override
public Class<LocalDateTime> handledType() {
return LocalDateTime.class;
}
}
/**
* LocalDatetime类型反序列化
*/
public static class LocalDatetimeDeserializer extends JsonDeserializer<LocalDateTime> {
/**
* LocalDateTime 反序列化
*
* @param jsonParser JsonParser
* @param deserializationContext DeserializationContext
* @return LocalDateTime
* @throws IOException IOException
*/
@Override
public LocalDateTime deserialize(JsonParser jsonParser, DeserializationContext deserializationContext)
throws IOException {
TreeNode tree = jsonParser.getCodec().readTree(jsonParser);
if (tree instanceof TextNode) {
TextNode node = (TextNode) tree;
String value = node.textValue();
return LocalDateTime.parse(value, DATE_TIME_FORMATTER);
}
return null;
}
}
/**
* LocalDatetime类型序列化
*/
public static class TimestampSerializer extends JsonSerializer<Timestamp> {
/**
* 序列化 Timestamp
*
* @param timestamp Timestamp
* @param jsonGenerator JsonGenerator
* @param serializerProvider SerializerProvider
* @throws IOException IOException
*/
@Override
public void serialize(Timestamp timestamp, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
LocalDateTime localDateTime = null;
if (Objects.nonNull(timestamp)) {
localDateTime = timestamp.toLocalDateTime();
}
jsonGenerator.writeString(DATE_TIME_FORMATTER.format(localDateTime));
}
/**
* 获取 Timestamp class
*
* @return Timestamp.class
*/
@Override
public Class<Timestamp> handledType() {
return Timestamp.class;
}
}
}
有了这个，你就可以做到json转对象，json转Hashmap，hashmap转对象，这在序列化和反序列化中比较实用。
时间戳、time和时间字符串的转换和处理
public class DatetimeUtil {
public static LocalDateTime getDateTimeOfTimestamp(long timestamp) {
Instant instant = Instant.ofEpochMilli(timestamp);
ZoneId zone = ZoneId.systemDefault();
return LocalDateTime.ofInstant(instant, zone);
}
public static String getViewStrOfDatetime(LocalDateTime localDateTime) {
return localDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
}
public static String getStrofTimestamp(long timestamp) {
return getViewStrOfDatetime(getDateTimeOfTimestamp(timestamp));
}
public static Long getTimestampOfDatetime(LocalDateTime localDateTime) {
return localDateTime.toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
}
public static Long getTimeStampOfViewStr(String value) {
return getTimestampOfDatetime(LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
}
public static LocalDateTime getLocalDatetimeByStr(String value){
return DatetimeUtil.getDateTimeOfTimestamp(DatetimeUtil.getTimeStampOfViewStr(value));
}
}类似的，也有时间戳，date和日期字符串的转换和处理
public class DateUtil {
public static LocalDate getDateTimeOfTimestamp(long timestamp) {
return Instant.ofEpochMilli(timestamp).atZone(ZoneId.systemDefault()).toLocalDate();
}
public static String getViewStrOfDatetime(LocalDate localDateTime) {
return localDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
}
public static String getStrofTimestamp(long timestamp) {
return getViewStrOfDatetime(getDateTimeOfTimestamp(timestamp));
}
public static Long getTimestampOfDatetime(LocalDate localDateTime) {
return localDateTime.atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli();
}
public static Long getTimeStampOfViewStr(String value) {
try {
return getTimeStampOfViewStrWithFormat(value, "yyyy-MM-dd");
}catch (Exception e){
// e.printStackTrace();
}
try {
return getTimeStampOfViewStrWithFormat(value, "yyyy/MM/dd");
}catch (Exception e){
// e.printStackTrace();
}
try {
return getTimeStampOfViewStrWithFormat(value, "yyyy年MM月dd日");
}catch (Exception e){
}
try {
if (value.length() == 4){
StringBuilder x = new StringBuilder(value).append("-12-30");
return getTimeStampOfViewStrWithFormat(x.toString(), "yyyy-MM-dd");
}
}catch (Exception e){
}
return getTimeStampOfViewStrWithFormat(value, "yyyy年");
}
public static Long getTimeStampOfViewStrWithFormat(String value, String format) {
return getTimestampOfDatetime(LocalDate.parse(value, DateTimeFormatter.ofPattern(format)));
}
public static LocalDate getLocalDatetimeByStr(String value) {
return DateUtil.getDateTimeOfTimestamp(DateUtil.getTimeStampOfViewStr(value));
}
public static Integer getWeekWithDate(LocalDate localDate){
return localDate.get(WeekFields.of(DayOfWeek.MONDAY, 1).weekOfYear());
}
}同时，必不可少的，我们的time和date之间的对象转换也是必须的。
public static LocalDate time2Date(LocalDateTime localDateTime){
return getLocalDatetimeByStr(DatetimeUtil.getViewStrOfDatetime(localDateTime).split(" ")[0]);
}
public static LocalDateTime date2Time(LocalDate localDate){
return DatetimeUtil.getLocalDatetimeByStr(getViewStrOfDatetime(localDate)+" 00:00:00");
}有了这些方法，你在时间领域的转换就可以游刃有余。简单快捷。
字符串和数组间的转换和处理
在实际编码的过程中，我们有时候会用字符串来表示数组的值，也可能会转换回去。
public class ArrayStrUtil {
public static List<String> str2Array(String str) {
List<String> ans = new ArrayList<>();
if (Objects.nonNull(str)) {
String[] tmp = str.split(",");
for (String x : tmp) {
if (Objects.nonNull(x) && StringUtils.hasText(x)) {
ans.add(x);
}
}
}
return ans;
}
public static List<Long> str2LArray(String str) {
List<Long> ans = new ArrayList<>();
List<String> tmp = str2Array(str);
for (String x : tmp) {
ans.add(Long.parseLong(x));
}
return ans;
}
public static List<Integer> str2IArray(String str) {
List<Integer> ans = new ArrayList<>();
List<String> tmp = str2Array(str);
for (String x : tmp) {
ans.add(Integer.parseInt(x));
}
return ans;
}
public static Set<String> str2Set(String str) {
Set<String> ans = new HashSet<>();
if (Objects.nonNull(str)) {
String[] tmp = str.split(",");
for (String x : tmp) {
if (Objects.nonNull(x)) {
ans.add(x);
}
}
}
return ans;
}
public static String slist2Str(List<String> a, String b) {
StringBuilder ans = new StringBuilder();
int cnt = 0;
for (String x : a) {
if (cnt == 0) {
ans.append(x);
} else {
ans.append(b).append(x);
}
cnt++;
}
return ans.toString();
}
public static String llist2Str(List<Long> a, String b) {
StringBuilder ans = new StringBuilder();
int cnt = 0;
for (Long x : a) {
if (cnt == 0) {
ans.append(x);
} else {
ans.append(b).append(x);
}
cnt++;
}
return ans.toString();
}
public static String ilist2Str(List<Integer> a, String b) {
StringBuilder ans = new StringBuilder();
int cnt = 0;
for (Integer x : a) {
if (cnt == 0) {
ans.append(x);
} else {
ans.append(b).append(x);
}
cnt++;
}
return ans.toString();
}
}这个工具，可以根据自己的需求进行补充，减少代码的编写，从而增加效率。
处理域名和ip
public final class DomainUtil {
public static String getTopDomain(String url) {
String RE_TOP =
"[\\w-]+\\.(com.cn|net.cn|gov.cn|org\\.nz|org.cn|com|net|org|gov|cc|biz|info|cn|co|edu.cn|ac.cn|mil.cn)\\b()*";
Pattern pattern = Pattern.compile(RE_TOP, Pattern.CASE_INSENSITIVE);
String result = url;
try {
Matcher matcher = pattern.matcher(url);
if (matcher.find()) {
result = matcher.group();
}
} catch (Exception e) {
System.out.println("[getTopDomain ERROR]====>");
e.printStackTrace();
}
return result;
}
public static Boolean isIpFormat(String ip) {
String rexp =
"([1-9]|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])(\\.(\\d|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])){3}";
Pattern pat = Pattern.compile(rexp);
Matcher mat = pat.matcher(ip);
return mat.find();
}
public static long ip2Int(String ipString) {
// 取 ip 的各段
String[] ipSlices = ipString.split("\\.");
long rs = 0;
for (int i = 0; i < ipSlices.length; i++) {
// 将 ip 的每一段解析为 int，并根据位置左移 8 位
long intSlice = Long.parseLong(ipSlices[i]) << 8 * i;
// 或运算
rs = rs | intSlice;
}
return rs;
}
public static String getDomain(String url){
return url.split("//")[1].split("/")[0];
}
}这些方法在爬虫场景中，用得比较多。你要提取主域名，子域名，判断当前提取出来的url是不是ip型等等。
md5的编码
这个有现成的库，不过也可以自己写，相当于是自己去定义md5的算法参数，更安全
public class MD5Utils {
public static String compMd5(String plainText) {
String re_md5 = new String();
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(plainText.getBytes());
byte b[] = md.digest();
int i;
StringBuffer buf = new StringBuffer("");
for (int offset = 0; offset < b.length; offset++) {
i = b[offset];
if (i < 0)
i += 256;
if (i < 16)
buf.append("0");
buf.append(Integer.toHexString(i));
}
re_md5 = buf.toString();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
return re_md5;
}
public static String compMd5WithByte(byte[] x) {
String re_md5 = new String();
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(x);
byte b[] = md.digest();
int i;
StringBuffer buf = new StringBuffer("");
for (int offset = 0; offset < b.length; offset++) {
i = b[offset];
if (i < 0)
i += 256;
if (i < 16)
buf.append("0");
buf.append(Integer.toHexString(i));
}
re_md5 = buf.toString();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
return re_md5;
}
}非spring容器中，获取spring容器中的component
@Component
public class SpringContextUtil1 implements ApplicationContextAware {
private static ApplicationContext applicationContext;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
SpringContextUtil1.applicationContext = applicationContext;
}
public static Object getBean(String beanName) {
return applicationContext.getBean(beanName);
}
public static <T> T getBean(Class<T> beanClass) {
return applicationContext.getBean(beanClass);
}
public static <T> T getBean(String beanName, Class<T> beanClass) {
return applicationContext.getBean(beanName, beanClass);
}
}
这个用得比较多，比如你的代码没有使用@Component、@Server这类注解，此时你使用@Autoware是不生效的，那么你就可以使用此工具类获取spring容器中的内容。
计算log a为底n
public class MathUtils {
/**
* 计算 log a为底n
*
* @param a
* @param n
* @return
*/
public static Double getLogAN(Integer a, Integer n) {
return Math.log10(n * 1.0) / Math.log10(a * 1.0);
}
}这个方法主要用于当你需要计算指数型分数的场景中。
文件处理
递归创建文件
public static void createFileRecursion(String fileName, Integer height) throws IOException {
Path path = Paths.get(fileName);
if (Files.exists(path)) {
// TODO: 2021/11/13 如果文件存在
return;
}
if (Files.exists(path.getParent())) {
// TODO: 2021/11/13 如果父级文件存在，直接创建文件
if (height == 0) {
Files.createFile(path);
} else {
Files.createDirectory(path);
}
} else {
createFileRecursion(path.getParent().toString(), height + 1);
// TODO: 2021/11/13 这一步能保证path的父级一定存在了，现在需要把自己也建一下
createFileRecursion(fileName, height);
}
}下面是完整的
public class GenerateFile {
public static void createFile(String folder, String fileName, String value) {
fileName = folder + "/" + fileName;
try {
createFileRecursion(fileName, 0);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(fileName, false));
bufferedOutputStream.write(value.getBytes());
bufferedOutputStream.flush();
bufferedOutputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void createFileRecursion(String fileName, Integer height) throws IOException {
Path path = Paths.get(fileName);
if (Files.exists(path)) {
// TODO: 2021/11/13 如果文件存在
return;
}
if (Files.exists(path.getParent())) {
// TODO: 2021/11/13 如果父级文件存在，直接创建文件
if (height == 0) {
Files.createFile(path);
} else {
Files.createDirectory(path);
}
} else {
createFileRecursion(path.getParent().toString(), height + 1);
// TODO: 2021/11/13 这一步能保证path的父级一定存在了，现在需要把自己也建一下
createFileRecursion(fileName, height);
}
}
public static void createFileWithRelativePath(String folder, String fileName, String value) {
File directory = new File(".");
try {
fileName = directory.getCanonicalPath() + "\\" + folder + "/" + fileName;
createFileRecursion(fileName, 0);
} catch (IOException e) {
e.printStackTrace();
}
try {
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(fileName, false));
bufferedOutputStream.write(value.getBytes());
bufferedOutputStream.flush();
bufferedOutputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void appendFileWithRelativePath(String folder, String fileName, String value) {
File directory = new File(".");
try {
fileName = directory.getCanonicalPath() + "\\" + folder + "/" + fileName;
createFileRecursion(fileName, 0);
} catch (IOException e) {
e.printStackTrace();
}
try {
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(fileName, true));
bufferedOutputStream.write(value.getBytes());
bufferedOutputStream.flush();
bufferedOutputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}发送请求
最简单的可以使用resttemplate。这里给一个我用得比较多的，细节只有掌握在自己手中，才有好的结果。
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpHost;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.*;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.Map;
import java.util.Objects;
public class SendReq {
private static HttpClient createHttpClient() {
Protocol easyhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
Protocol.registerProtocol("https", easyhttps);
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
connectionManager.getParams().setDefaultMaxConnectionsPerHost(1000);
connectionManager.getParams().setMaxTotalConnections(1000);
HttpClient httpclient = new HttpClient(connectionManager);
return httpclient;
}
public static ResBody sendReq(String url, String method, Map<String, Object> params, Map<String, String> requestHeaders) {
GetMethod httpget = null;
HeadMethod httphead = null;
PostMethod httpPost = null;
DeleteMethod httpDelete = null;
PutMethod httpPut = null;
HttpClient httpClient = null;
String rawHeader = "";
String responce = "";
String rawResponce = "";
String responseHeader = "";
String responseBody = "";
int code = 0;
try {
String line;
if (method.equalsIgnoreCase("HEAD")) {
httphead = new HeadMethod(url);
for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
httphead.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
}
httphead.setFollowRedirects(true);
httpClient = createHttpClient();
code = httpClient.executeMethod(httphead);
httphead.releaseConnection();
} else if (method.equalsIgnoreCase("GET")) {
httpget = new GetMethod(url);
httpget.setFollowRedirects(true);
for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
httpget.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
}
httpClient = createHttpClient();
code = httpClient.executeMethod(httpget);
BufferedReader input = new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));
rawHeader = httpget.getStatusLine() + "\r\n";
Header[] headers = httpget.getResponseHeaders();
StringBuffer buf = new StringBuffer();
for (int a = 0; a < headers.length; ++a) {
buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
}
rawHeader = rawHeader + buf.toString();
responseHeader = rawHeader;
buf = new StringBuffer();
while ((line = input.readLine()) != null) {
buf.append("\r\n" + line);
}
responce = buf.toString();
responseBody = responce;
input.close();
rawResponce = rawHeader + responce;
// TODO: 2021/3/17 responce 就是最后的值
Header contentType = httpget.getResponseHeader("Content-Type");
if (contentType != null && contentType.getValue().startsWith("text")) {
// TODO: 2021/3/17 解析页面，默认先不解析
}
Thread.sleep(10L);
httpget.releaseConnection();
} else if (method.equalsIgnoreCase("POST") || method.equalsIgnoreCase("PUT") || method.equalsIgnoreCase("DELETE")) {
BufferedReader input = null;
Header[] headers = null;
if (method.equalsIgnoreCase("POST")) {
httpPost = new PostMethod(url);
for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
httpPost.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
}
String toJson = Json.toJson(params);
RequestEntity se = new StringRequestEntity (toJson ,"application/json" ,"UTF-8");
httpPost.setRequestEntity(se);
httpClient = createHttpClient();
code = httpClient.executeMethod(httpPost);
input = new BufferedReader(new InputStreamReader(httpPost.getResponseBodyAsStream()));
rawHeader = httpPost.getStatusLine() + "\r\n";
headers = httpPost.getResponseHeaders();
} else if (method.equalsIgnoreCase("PUT")) {
httpPut = new PutMethod(url);
for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
httpPut.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
}
// httpPut.setFollowRedirects(true);
String toJson = Json.toJson(params);
RequestEntity se = new StringRequestEntity (toJson ,"application/json" ,"UTF-8");
httpPut.setRequestEntity(se);
httpClient = createHttpClient();
code = httpClient.executeMethod(httpPut);
input = new BufferedReader(new InputStreamReader(httpPut.getResponseBodyAsStream()));
rawHeader = httpPut.getStatusLine() + "\r\n";
headers = httpPut.getResponseHeaders();
} else if (method.equalsIgnoreCase("DELETE")) {
httpDelete = new DeleteMethod(url);
for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
httpDelete.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
}
// httpDelete.setFollowRedirects(true);
httpClient = createHttpClient();
code = httpClient.executeMethod(httpDelete);
input = new BufferedReader(new InputStreamReader(httpDelete.getResponseBodyAsStream()));
rawHeader = httpDelete.getStatusLine() + "\r\n";
headers = httpDelete.getResponseHeaders();
}
StringBuffer buf = new StringBuffer();
for (int a = 0; a < headers.length; ++a) {
buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
}
rawHeader = rawHeader + buf.toString();
responseHeader = rawHeader;
buf = new StringBuffer();
while ((line = input.readLine()) != null) {
buf.append("\r\n" + line);
}
responce = buf.toString();
responseBody = responce;
input.close();
rawResponce = rawHeader + responce;
Thread.sleep(10L);
if (Objects.nonNull(httpDelete)) httpDelete.releaseConnection();
if (Objects.nonNull(httpPost)) httpPost.releaseConnection();
if (Objects.nonNull(httpPut)) httpPut.releaseConnection();
}
} catch (Exception e) {
e.printStackTrace();
}
return new ResBody(rawHeader, responce, rawResponce, responseHeader, responseBody, code);
}
public static class ResBody {
private String rawHeader;
private String responce;
private String rawResponce;
private String responseHeader;
private String responseBody;
private Integer code;
public ResBody(String rawHeader, String responce, String rawResponce, String responseHeader, String responseBody, Integer code) {
this.rawHeader = rawHeader;
this.responce = responce;
this.rawResponce = rawResponce;
this.responseHeader = responseHeader;
this.responseBody = responseBody;
this.code = code;
}
public Integer getCode() {
return code;
}
public void setCode(Integer code) {
this.code = code;
}
public String getRawHeader() {
return rawHeader;
}
public String getRawResponce() {
return rawResponce;
}
public String getResponce() {
return responce;
}
public String getResponseBody() {
return responseBody;
}
public String getResponseHeader() {
return responseHeader;
}
public void setRawHeader(String rawHeader) {
this.rawHeader = rawHeader;
}
public void setRawResponce(String rawResponce) {
this.rawResponce = rawResponce;
}
public void setResponce(String responce) {
this.responce = responce;
}
public void setResponseBody(String responseBody) {
this.responseBody = responseBody;
}
public void setResponseHeader(String responseHeader) {
this.responseHeader = responseHeader;
}
}
}
其中
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
public class EasyX509TrustManager implements X509TrustManager {
private X509TrustManager standardTrustManager = null;
public EasyX509TrustManager(KeyStore keystore) throws NoSuchAlgorithmException, KeyStoreException {
TrustManagerFactory factory = TrustManagerFactory.getInstance("SunX509");
factory.init(keystore);
TrustManager[] trustmanagers = factory.getTrustManagers();
if (trustmanagers.length == 0) {
throw new NoSuchAlgorithmException("SunX509 trust manager not supported");
} else {
this.standardTrustManager = (X509TrustManager) trustmanagers[0];
}
}
public X509Certificate[] getAcceptedIssuers() {
return null;
}
public void checkClientTrusted(X509Certificate[] certs, String authType) {
}
public void checkServerTrusted(X509Certificate[] certs, String authType) {
}
public boolean isServerTrusted(X509Certificate[] certs) {
return true;
}
public boolean isClientTrusted(X509Certificate[] certs) {
return true;
}
}
建立https传输信道
import org.apache.commons.httpclient.ConnectTimeoutException;
import org.apache.commons.httpclient.HttpClientError;
import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
import javax.net.SocketFactory;
import javax.net.ssl.KeyManager;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import java.io.IOException;
import java.net.*;
import java.security.KeyStore;
import java.security.SecureRandom;
public class EasySSLProtocolSocketFactory implements SecureProtocolSocketFactory {
private SSLContext sslcontext = null;
private static SSLContext createEasySSLContext() {
try {
SSLContext context = SSLContext.getInstance("SSL");
context.init((KeyManager[]) null, new TrustManager[]{new EasyX509TrustManager((KeyStore) null)}, (SecureRandom) null);
return context;
} catch (Exception var1) {
throw new HttpClientError(var1.toString());
}
}
private SSLContext getSSLContext() {
if (this.sslcontext == null) {
this.sslcontext = createEasySSLContext();
}
return this.sslcontext;
}
public Socket createSocket(String host, int port, InetAddress clientHost, int clientPort) throws IOException, UnknownHostException {
return this.getSSLContext().getSocketFactory().createSocket(host, port, clientHost, clientPort);
}
public Socket createSocket(String host, int port, InetAddress localAddress, int localPort, HttpConnectionParams params) throws IOException, UnknownHostException, ConnectTimeoutException {
if (params == null) {
throw new IllegalArgumentException("Parameters may not be null");
} else {
int timeout = params.getConnectionTimeout();
SocketFactory socketfactory = this.getSSLContext().getSocketFactory();
if (timeout == 0) {
return socketfactory.createSocket(host, port, localAddress, localPort);
} else {
Socket socket = socketfactory.createSocket();
SocketAddress localaddr = new InetSocketAddress(localAddress, localPort);
SocketAddress remoteaddr = new InetSocketAddress(host, port);
socket.bind(localaddr);
socket.connect(remoteaddr, timeout);
return socket;
}
}
}
public Socket createSocket(String host, int port) throws IOException, UnknownHostException {
return this.getSSLContext().getSocketFactory().createSocket(host, port);
}
public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException, UnknownHostException {
return this.getSSLContext().getSocketFactory().createSocket(socket, host, port, autoClose);
}
public boolean equals(Object obj) {
return obj != null && obj.getClass().equals(EasySSLProtocolSocketFactory.class);
}
public int hashCode() {
return EasySSLProtocolSocketFactory.class.hashCode();
}
}
playwrite-自动化测试或web2.0爬虫
@Service
public class PlayWriteMachineService implements InitializingBean {
public List<String> getUrlAndParseHtml(String url, String selector, String functionStr) {
Playwright playwright = Playwright.create();
Browser browser = createBrowser("chromium", playwright);
BrowserContext context = createContext(browser);
Page page = context.newPage();
System.out.println(page.title());
try {
List<String> values = getOnePage(url, page, selector, functionStr);
return values;
} catch (Exception e) {
e.printStackTrace();
} finally {
page.close();
context.close();
browser.close();
playwright.close();
}
return new ArrayList<>();
}
@Override
public void afterPropertiesSet() throws Exception {
}
private BrowserContext createContext(Browser browser) {
BrowserContext context = browser.newContext(new Browser.NewContextOptions()
.setIgnoreHTTPSErrors(true)
.setJavaScriptEnabled(true)
.setViewportSize(1920, 1080)
.setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.130 Safari/537.36"));
return context;
}
private Browser createBrowser(String name, Playwright playwright) {
try {
switch (name) {
case "firefox":
return playwright.firefox().launch();
case "chromium":
return playwright.chromium().launch(new BrowserType.LaunchOptions().setHeadless(true).setTimeout(120 * 1000));
case "webkit":
return playwright.webkit().launch();
}
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public List<String> getOnePage(String nodeUrl, Page page, String selector, String functionStr) {
// TODO: 2021/4/19 创建page，然后前往链接
// TODO: 2021/4/19 向页面注入一些东西，以求达到某个目标
Set<String> pageAllUrls = new ConcurrentSkipListSet<>();
Map<String, String> postDatas = new ConcurrentHashMap<>();
page.setDefaultNavigationTimeout(110 * 1000);
page.onRequest(request -> {
if (Objects.nonNull(request)) {
String method = request.method();
String url = request.url();
if (method.toLowerCase().equals("post") && Objects.nonNull(request.postData())) {
postDatas.put(url, request.postData());
}
pageAllUrls.add(request.url());
if (request.isNavigationRequest()) {
try {
if (!url.equals(nodeUrl)) {
request.response();
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
});
page.onResponse(response -> {
if (response.status() >= 300 && response.status() < 400) {
pageAllUrls.add(response.headers().getOrDefault("location", ""));
}
});
page.onFrameNavigated(frame -> {
pageAllUrls.add(frame.url());
});
page.onDialog(Dialog::dismiss);
page.navigate(nodeUrl, new Page.NavigateOptions());
page.waitForTimeout(5000);
// TODO: 2021/4/19 eval
List<String> values = evalAndGetValue(page, selector, functionStr);
return values;
}
private static List<String> evalAndGetValue(Page page, String selector, String functionStr) {
System.out.println(selector);
System.out.println(functionStr);
List<String> ans = (List<String>) page.evalOnSelectorAll(selector, functionStr);
List<String> res = new ArrayList<>();
for (String x : ans) {
if (StringUtils.hasText(x)) {
res.add(x);
}
}
return res;
}
}
这个例子只是我自身应用场景的代码，这是一个简单使用的例子。很好用，不过也有坑。
官网地址：https://github.com/microsoft/playwright-java
api地址：https://playwright.dev/docs/api/class-playwright
maven
<dependency>
<groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId>
<version>1.10.0</version>
</dependency>
利用正则表达式提取关键信息
for (Pattern pattern : patterns) {
Matcher matcher = pattern.matcher(html);
while (matcher.find()) {
tmp.add(matcher.group());
System.out.println("begin = " + matcher.start() + ", end = " + matcher.end());
}
}
// 其中，Pattern可以通过
Pattern.compile("<!--[\\s\\S]*?-->");
得到find一个，再调用group就得到一个。当然，还有更好的方法，你可以在正则表达式中打个括号，然后输入变量名，当正则表达式找到了，就会通过matcher.group(“变量名”)来实现。
缓存
当你没法用redis，又需要缓存来加快速度的场景
@Service
public class LruCacheWithTTL {
/**
* 缓存键规则：查询条件为键
* 唯一缓存标记（用于区分不同的业务场景） + 条件（条件1：值（如果有）+ 条件2：值（如果有）+ 条件3：值（如果有））
* 缓存值规则：返回的值的Json为值
*/
private LRUMap<String, String> cache = new LRUMap<>(20, 10000);
private ConcurrentHashMap<String, Long> ttls = new ConcurrentHashMap<>();
public String getCacheJson(Integer bizCode, Map<String, Object> condition) {
String key = bizCode + "+" + getSortedConditionStr(condition);
if (ttls.containsKey(key)) {
if (DatetimeUtil.getDateTimeOfTimestamp(ttls.get(key)).plusSeconds(5).isBefore(LocalDateTime.now())) {
// TODO: 2021/9/29 如果当前时间没有超时，直接返回结果
ttls.remove(key);
}else {
// TODO: 2021/9/29 删除缓存
return cache.get(key);
}
}
return null;
}
private String getSortedConditionStr(Map<String, Object> condition) {
Set<String> keys = condition.keySet();
List list = new ArrayList(keys);
Object[] ary = list.toArray();
Arrays.sort(ary);
list = Arrays.asList(ary);
String str = "";
for (int i = 0; i < list.size(); i++) {
str += list.get(i) + "=" + condition.get(list.get(i) + "") + "&";
}
return str;
}
public Boolean putCacheJson(Integer bizCode, Map<String, Object> condition, String value) {
try {
String key = bizCode + "+" + getSortedConditionStr(condition);
ttls.put(key, DatetimeUtil.getTimestampOfDatetime(LocalDateTime.now()));
cache.put(key, value);
return true;
}catch (Exception e){
e.printStackTrace();
}
return false;
}
}
调用命令行
Process process = Runtime.getRuntime().exec("ping www.qq.com");传输文件
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.util.Map;
import java.util.Objects;
@Service
public class UploadFileService {
private static Log logger = LogFactory.getLog(UploadFileService.class);
@Autowired
private RestTemplate restTemplate;
@Autowired
private TConfigRepository tConfigRepository;
public void demo(MultipartFile file) {
if (tConfigRepository.findByConfigNameAndDeleteMarkFalse("oos_url").isPresent()) {
String oosUrl = tConfigRepository.findByConfigNameAndDeleteMarkFalse("oos_url").get().getConfigValue();
if (StringUtils.hasText(oosUrl)) {
MultiValueMap<String, Object> param = new LinkedMultiValueMap<>();
param.add("file", new FileSystemResource(convert(file)));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
HttpEntity<MultiValueMap<String, Object>> httpEntity = new HttpEntity<MultiValueMap<String, Object>>(param, headers);
ResponseEntity<FileRes> resResponseEntity = restTemplate.postForEntity(oosUrl + "/v1/upload", httpEntity, FileRes.class);
if (resResponseEntity.getStatusCode().is2xxSuccessful()) {
FileRes res = resResponseEntity.getBody();
if (Objects.nonNull(res)) {
logger.info(res.getMessage());
logger.info(res.getCode());
if (res.getCode().equals("00000")) {
logger.info(String.format("%s %s", res.getInfo().get("fileName"), res.getInfo().get("uuid")));
}
}
logger.info("null");
}
}
}
}
public File convert(MultipartFile file) {
File convFile = new File("temp_file", file.getOriginalFilename());
if (!convFile.getParentFile().exists()) {
logger.info("mkdir:" + convFile.getParentFile().mkdirs());
}
try {
convFile.createNewFile();
FileOutputStream fos = new FileOutputStream(convFile);
fos.write(file.getBytes());
fos.close();
} catch (IOException e) {
e.printStackTrace();
}
return convFile;
}
}
java中使用多元组
<dependency>
<groupId>org.javatuples</groupId>
<artifactId>javatuples</artifactId>
<version>1.2</version>
</dependency>
Unit (1 element)
Pair<A,B> (2 elements)
Triplet<A,B,C> (3 elements)
Quartet<A,B,C,D> (4 elements)
Quintet<A,B,C,D,E> (5 elements)
Sextet<A,B,C,D,E,F> (6 elements)
Septet<A,B,C,D,E,F,G> (7 elements)
Octet<A,B,C,D,E,F,G,H> (8 elements)
Ennead<A,B,C,D,E,F,G,H,I> (9 elements)
Decade<A,B,C,D,E,F,G,H,I,J> (10 elements)
springboot无端口模式启动
spring.main.web-application-type=none
// 然后启动系统任务即可，即有线程不死掉。但是如果想要稳定的停止的话，就必须监听端口
cron的写法
校验时间
https://www.bejson.com/othertools/cronvalidate/ 生成cron
https://www.bejson.com/othertools/cron/
