from selenium import webdriver
import time
import json
import random
import requests
import re
import pymysql
from urllib import parse
gzlist = []
def getListWxName():
db = pymysql.connect("172.16.37.50", "root", "zxsoftoa", "zx_crawler_db")
cursor = db.cursor()
sql = "SELECT w.wxName FROM `wx_account` w;"
try:
cursor.execute(sql)
results = cursor.fetchall()
for row in results:
gzlist.append(row[0])
print(gzlist)
except Exception as inst:
print(inst)
db.rollback()
# 登录微信公众号，获取登录之后的cookies信息，并保存到本地文本中
def wechat_login():
# 用webdriver启动谷歌浏览器
print("启动浏览器，打开微信公众号登录界面")
driver = webdriver.Chrome(r"E:\chromedriver.exe")
driver.get("https://mp.weixin.qq.com/")
time.sleep(2)
print("正在输入微信公众号登录账号和密码......")
time.sleep(20)
print("登录成功")
# 获取cookies
cookie_items = driver.get_cookies()
post = {}
# 获取到的cookies是列表形式，将cookies转成json形式并存入本地名为cookie的文本中
for cookie_item in cookie_items:
post[cookie_item['name']] = cookie_item['value']
cookie_str = json.dumps(post)
with open('cookie.txt', 'w+', encoding='utf-8') as f:
f.write(cookie_str)
print("cookies信息已保存到本地")
driver.quit()
# 爬取微信公众号文章，并存在本地文本中
def get_content(query):
# query为要爬取的公众号名称
# 公众号主页
url = 'https://mp.weixin.qq.com'
# 设置headers
header = {
"HOST": "mp.weixin.qq.com",
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
}
from requests.packages import urllib3
urllib3.disable_warnings() # 关闭警告
# 读取上一步获取到的cookies
with open('cookie.txt', 'r', encoding='utf-8') as f:
cookie = f.read()
cookies = json.loads(cookie)
# 增加重试连接次数
session = requests.Session()
session.keep_alive = False
# 增加重试连接次数
session.adapters.DEFAULT_RETRIES = 511
time.sleep(5)
# 登录之后的微信公众号首页url变化为：https://mp.weixin.qq.com/cgi-bin/home?t=home/index&lang=zh_CN&token=1849751598，从这里获取token信息
response = session.get(url=url, cookies=cookies, verify=False)
token = re.findall(r'token=(\d+)', str(response.url))[0]
time.sleep(2)
# 搜索微信公众号的接口地址
search_url = 'https://mp.weixin.qq.com/cgi-bin/searchbiz?'
# 搜索微信公众号接口需要传入的参数，有三个变量：微信公众号token、随机数random、搜索的微信公众号名字
query_id = {
'action': 'search_biz',
'token': token,
'lang': 'zh_CN',
'f': 'json',
'ajax': '1',
'random': random.random(),
'query': query,
'begin': '0',
'count': '5'
}
# 打开搜索微信公众号接口地址，需要传入相关参数信息如：cookies、params、headers
search_response = session.get(
search_url,
cookies=cookies,
headers=header,
params=query_id)
# 取搜索结果中的第一个公众号
lists = search_response.json().get('list')[0]
print(lists)
# 获取这个公众号的fakeid，后面爬取公众号文章需要此字段
fakeid = lists.get('fakeid')
# 微信公众号文章接口地址
appmsg_url = 'https://mp.weixin.qq.com/cgi-bin/appmsg?'
# 搜索文章需要传入几个参数：登录的公众号token、要爬取文章的公众号fakeid、随机数random
query_id_data = {
'token': token,
'lang': 'zh_CN',
'f': 'json',
'ajax': '1',
'random': random.random(),
'action': 'list_ex',
'begin': '0', # 不同页，此参数变化，变化规则为每页加5
'count': '5',
'query': '',
'fakeid': fakeid,
'type': '9'
}
# 打开搜索的微信公众号文章列表页
appmsg_response = session.get(
appmsg_url,
cookies=cookies,
headers=header,
params=query_id_data)
# 获取文章总数
max_num = appmsg_response.json().get('app_msg_cnt')
# 每页至少有5条，获取文章总的页数，爬取时需要分页爬
num = int(int(max_num) / 5)
# 起始页begin参数，往后每页加5
begin = 0
seq = 0
while num + 1 > 0:
query_id_data = {
'token': token,
'lang': 'zh_CN',
'f': 'json',
'ajax': '1',
'random': random.random(),
'action': 'list_ex',
'begin': '{}'.format(str(begin)),
'count': '5',
'query': '',
'fakeid': fakeid,
'type': '9'
}
print('正在翻页：--------------', begin)
time.sleep(5)
# 获取每一页文章的标题和链接地址，并写入本地文本中
query_fakeid_response = requests.get(
appmsg_url,
cookies=cookies,
headers=header,
params=query_id_data)
fakeid_list = query_fakeid_response.json().get('app_msg_list')
if fakeid_list:
for item in fakeid_list:
content_link = item.get('link')
content_title = item.get('title')
fileName = query + '.txt'
qs = parse.parse_qs(content_link)
articleId = qs['mid'][0]
seq += 1
with open(fileName, 'a', encoding='utf-8') as fh:
fh.write(str(seq) + "|" + articleId + "|" + content_title + "|" + content_link + "\n")
num -= 1
begin = int(begin)
begin += 5
if __name__ == '__main__':
getListWxName()
# 登录微信公众号，获取登录之后的cookies信息，并保存到本地文本中
wechat_login()
query = "XXX"
print("开始爬取公众号：" + query)
get_content(query)
print("爬取完成")
# 登录之后，通过微信公众号后台提供的微信公众号文章接口爬取文章
for query in gzlist:
# 爬取微信公众号文章，并存在本地文本中
print("开始爬取公众号：" + query)
get_content(query)
print("爬取完成")
/*
Navicat MySQL Data Transfer
Source Server : 172.16.37.50
Source Server Version : 50717
Source Host : 172.16.37.50:3306
Source Database : zx_crawler_db
Target Server Type : MYSQL
Target Server Version : 50717
File Encoding : 65001
Date: 2021-09-03 15:39:51
*/
SET FOREIGN_KEY_CHECKS=0;
-- ----------------------------
-- Table structure for wx_article
-- ----------------------------
DROP TABLE IF EXISTS `wx_article`;
CREATE TABLE `wx_article` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`wxCode` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
`wxBiz` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`articleId` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`articleTitle` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`articleUrl` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
`articleDigest` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`coverUrl` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`articleSouUrl` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`readNum` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`goodNum` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`companyName` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`creditCode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`regisCode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`cityCode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`publishTime` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`createTime` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`updateTime` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`) USING BTREE,
UNIQUE KEY `articleUrl` (`articleUrl`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=15459 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
故乡明
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
