爬取python标准库
想要把Python的标准库文档趴下来,试过直接存成html,但是简单的存储css的样式等都会丢失,遂想存为pdf。
需要一个工具pdfkit,结合之前的selenium爬下来。
首先需要
pip install pdfkit
# coding:utf-8
import urllib
from urllib import request
import os,time
from os import path
from selenium import webdriver
import pdfkit
import re
#去掉尖括号之间的内容
def transname(name):
pattern = '<.*?>'
res = re.compile(pattern).sub("",name)
return res
url_root = 'https://docs.python.org/3/library/'
url_index = url_root + 'index.html'
result_dir = path.join(os.getcwd(),'result')
if not path.exists(result_dir):
os.makedirs(result_dir)
#pdfkit.from_url(url_index,path.join(result_dir,'index.pdf'))
driver = webdriver.PhantomJS()
# driver = webdriver.Firefox()
driver.get(url_index)
html_index = driver.page_source
pattern = '<li class="toctree-l[12]"><a class="reference internal" href="(.+?)">(.+?)</a>'
res = re.compile(pattern,re.S).findall(html_index)
print(res)
ct = 0
amt = len(res)
for i,x in enumerate(res):
if i<127:
continue
addr = url_root + x[0]
if re.compile('.*\.html$').match(addr):
name = re.compile(os.sep).sub(r'-',x[1]) #去掉系统转换的符号,防止误把路径分割了
name = path.join(result_dir,name+'.pdf')
name = transname(name)
ct = ct + 1
print(ct,'/',i+1,'/',amt,addr,name)
pdfkit.from_url(addr,name)
else:
amt = amt - 1
结果