爬取公众号文章思路
使用fiddler拦截电脑端微信公众号的历史文章列表
构造请求获取文章详情链接
fiddler下载链接:https://www.telerik.com/download/fiddler
java抓取代码:
package com.mybatis.plus.utils;
import cn.hutool.core.lang.Console;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
public class test1 {
public static void main(String[] args) {
String biz = "MzIxOTc0MzQzMg==";
String uin = "MjYyMjc0NzA0Mw%3D%3D";
// 这个是用的的我的微信号 动态改变的只有key
String key = "af4c4474d22b7e510ffb9a836217934142013a224ea9f909f2762f52f45f5062868454601519f489fc119061433823dc131c7ab44c519fd0bba40b284935ff53a721808cb1eb9e04027e28eb2ce97e327ca2dac3a97cbb9313f032850d66335fc3b0223c26ebb15a26f99879aa9643745069960c3bb252c253419feefe9e86e8";
int offset = 0;
int count = 10;
int countFlag = 1;
while (true) {
// HttpRequest httpRequest = HttpUtil.createGet("").header("", "");
// httpRequest.execute().body();
String response = HttpUtil.get("https://mp.weixin.qq.com/mp/profile_ext?" +
"action=getmsg&" +
"__biz=" + biz +
"&f=json&offset=" + offset +
"&count=10" +
"&uin=" + uin +
"&key=" + key);
offset = offset + count;
// 解析数据返回内容
JSONObject jsonObject = JSONObject.parseObject(response);
String general_msg_list = jsonObject.getString("general_msg_list");
if (StringUtils.isEmpty(general_msg_list)) {
Console.log("====访问秘钥已过期,停止采集====");
break;
}
JSONObject jsonObject1 = JSONObject.parseObject(general_msg_list);
JSONArray list = jsonObject1.getJSONArray("list");
if (list.isEmpty()) {
Console.log("=======该公众号没有更多文章了,停止采集====");
break;
}
for (Object o : list) {
String toJSONString = JSONObject.toJSONString(o);
JSONObject jsonObject2 = JSONObject.parseObject(toJSONString);
JSONObject app_msg_ext_info = jsonObject2.getJSONObject("app_msg_ext_info");
String title = app_msg_ext_info.getString("title");
String content_url = app_msg_ext_info.getString("content_url");
Console.log("获取到第:" + countFlag++ + " 条数据");
Console.log("获取到文章标题:" + title);
Console.log("获取到文章url:" + content_url);
MyUtil.saveAsFileWriter("获取到文章标题:" + title + "\n" + "获取到文章url:" + content_url + "\n");
}
}
}
}
抓取效果:
爬虫弊端:由于key会很快失效 需要从fiddler监控软件中获取到key