Python25行代码爬取豆瓣排行榜数据

只需要用到requests, re ,csv 三个库即可。

code

import re
import requests
import csv
url = 'https://movie.douban.com/top250'
obj = re.compile(r'<span class="title">(?P<name>.*?)</span>.*?<br>.*?(?P<year>.*?)&nbsp.*?'
                 r'<span class="rating_num" property="v:average">(?P<score>.*?)</span>.*?'
                 r'<span>(?P<person>.*?)人评价', re.S)

headers = { # 这里使用自己电脑浏览器的user-agent即可
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"
}

r = requests.get(url, headers=headers)
c = r.text
it = obj.finditer(c)
# 默认csv写入时写一行会自动空一行,设置newline即可
f = open("data.csv", mode="w", encoding="utf-8", newline='')
cw = csv.writer(f)

for i in it:
    dic = i.groupdict()
    dic['year'] = dic['year'].strip()
    cw.writerow(dic.values())

r.close()
f.close()
# print(i.group("name"))
# print(i.group("year").strip())
# print(i.group("score"))
# print(i.group("person"))

data.csv内容

肖申克的救赎,1994,9.7,2336915
霸王别姬,1993,9.6,1737860
阿甘正传,1994,9.5,1759121
这个杀手不太冷,1994,9.4,1935854
泰坦尼克号,1997,9.4,1721133
美丽人生,1997,9.5,1083466
千与千寻,2001,9.4,1836993
辛德勒的名单,1993,9.5,898293
盗梦空间,2010,9.3,1700346
忠犬八公的故事,2009,9.4,1167713
星际穿越,2014,9.3,1373371
楚门的世界,1998,9.3,1288132
海上钢琴师,1998,9.3,1381565
三傻大闹宝莱坞,2009,9.2,1552495
机器人总动员,2008,9.3,1092632
放牛班的春天,2004,9.3,1075957
大话西游之大圣娶亲,1995,9.2,1255541
疯狂动物城,2016,9.2,1517255
无间道,2002,9.2,1043103
熔炉,2011,9.3,762596
教父,1972,9.3,763465
当幸福来敲门,2006,9.1,1247494
龙猫,1988,9.2,1040400
怦然心动,2010,9.1,1473508
控方证人,1957,9.6,363652