文章目录
- 一 步骤
- 二 代码
- 三 遇到的问题及解决办法
- 1.缺少模块
- 2. 报错
- 3 报错
- 四 更正
一 步骤
1 创建"web服务",获取key
2 网上找到Python爬虫代码
3 运行,调试,直至能得到结果
二 代码
注意:不是自己写的,从网上找的
from urllib.parse import quote
import urllib.request
import urllib
import pandas as pd
import xlwt
import json
class getpoi:
output_path="C:/out/" ##########输出文件夹名称###########
path_class ="C:/out/amap_poicode.xlsx" ##########输入的POI分类编码表(用以指示需要爬取的POI类型)###########
amap_web_key = 'key' ##########高德API的密钥###########
poi_search_url = "https://restapi.amap.com/v3/place/text?" ##########不需要更改##########
cityname = '重庆市' ############目标的城市(与城市编码表中的名称需要一致)#############
areas = ['重庆市市辖区','万州区','涪陵区','渝中区','大渡口区','江北区','沙坪坝区','九龙坡区','南岸区','北碚区'] ############目标城市的次级行政范围(与城市编码表中的名称需要一致)#############
totalcontent = {}
def __init__(self):
data_class = self.getclass()
for type_class in data_class:
for area in self.areas:
page = 1;
if type_class['type_num'] / 10000 < 10:
classtype = str('0') + str(type_class['type_num'])
else:
classtype = str(type_class['type_num'])
while True:
if classtype[-4:] =="0000":
break;
poidata = self.get_poi(classtype, area, page);
poidata = json.loads(poidata)
if poidata['count'] == "0":
break;
else:
poilist = self.hand(poidata)
print("area:" + area + " type:" + classtype + " page:第" + str(page) + "页 count:" + poidata['count'] + "poilist:")
page += 1
for pois in poilist:
if classtype[0:2] in self.totalcontent.keys():
pois['bigclass'] = type_class['bigclass']
pois['midclass'] = type_class['midclass']
pois['smallclass'] = type_class['smallclass']
list_total = self.totalcontent[classtype[0:2]]
list_total.append(pois)
else:
self.totalcontent[classtype[0:2]] = []
pois['bigclass'] = type_class['bigclass']
pois['midclass'] = type_class['midclass']
pois['smallclass'] = type_class['smallclass']
self.totalcontent[classtype[0:2]].append(pois)
for content in self.totalcontent:
self.writeexcel(self.totalcontent[content], content)
def writeexcel(self, data, classname):
book = xlwt.Workbook(encoding='utf-8', style_compression=0)
sheet = book.add_sheet(classname, cell_overwrite_ok=True)
# 第一行(列标题)
sheet.write(0, 0, 'x')
sheet.write(0, 1, 'y')
sheet.write(0, 2, 'count')
sheet.write(0, 3, 'name')
sheet.write(0, 4, 'adname')
sheet.write(0, 5, 'smallclass')
sheet.write(0, 6, 'typecode')
sheet.write(0, 7, 'midclass')
classname = data[0]['bigclass']
for i in range(len(data)):
sheet.write(i + 1, 0, data[i]['lng'])
sheet.write(i + 1, 1, data[i]['lat'])
sheet.write(i + 1, 2, 1)
sheet.write(i + 1, 3, data[i]['name'])
sheet.write(i + 1, 4, data[i]['adname'])
sheet.write(i + 1, 5, data[i]['smallclass'])
sheet.write(i + 1, 6, data[i]['classname'])
sheet.write(i + 1, 7, data[i]['midclass'])
book.save(self.output_path + self.cityname + '_' + classname + '.xls')
def hand(self, poidate):
pois = poidate['pois']
poilist = []
for i in range(len(pois)):
content = {}
content['lng'] = float(str(pois[i]['location']).split(",")[0])
content['lat'] = float(str(pois[i]['location']).split(",")[1])
content['name'] = pois[i]['name']
content['adname'] = pois[i]['adname']
content['classname'] = pois[i]['typecode']
poilist.append(content)
return poilist
def readfile(self, readfilename, sheetname):
data = pd.read_excel(readfilename, sheet_name=sheetname)
return data
def getclass(self):
readcontent = self.readfile(self.path_class, "amap_poicode")
data = []
for num in range(readcontent.shape[0]):
content = {}
content['type_num'] = readcontent.iloc[num]['NEW_TYPE']
content['bigclass'] = readcontent.iloc[num]['大类']
content['midclass'] = readcontent.iloc[num]['中类']
content['smallclass'] = readcontent.iloc[num]['小类']
data.append(content)
return data
def get_poi(self, keywords, city, page):
poiurl = self.poi_search_url +"key="+self.amap_web_key+ "&extensions=all&keywords=" + quote(keywords) + '&city=' +quote(city)+"&citylimit=true"+"&offset=25"+"&page=" +str(page)+"&output=json"
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36"}
req = urllib.request.Request(url=poiurl, headers=headers)
data = ''
with urllib.request.urlopen(req) as f:
data = f.read().decode('utf8')
return data
if __name__ == "__main__":
gp = getpoi()
三 遇到的问题及解决办法
1.缺少模块
解决办法: 在cmd模式下,执行命令
pip install XXXX
安装需要的模块
2. 报错
报错:module ‘urllib’ has no attribute ‘request’
解决办法:改一下导入模块
import urllib.request
3 报错
报错:http.client.RemoteDisconnected: Remote end closed connection without response
原因:服务器根据U-A判断拒绝爬虫
解决办法:修改headers
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36"}
req = urllib.request.Request(url=poiurl, headers=headers)
四 更正
上文的代码不对,爬出来的数据不合理,比如说重庆的汽车服务只有几百个,还有医药服务甚至只有一个,太离谱了
所以我又去网上找了一个代码,这次爬出来的数据是对的了
还是说明:不是自己写的,网上找到的
import requests
import pandas as pd
import json
import time
import math
#各下级行政区的代码,若是嫌逐个复制麻烦可以通过读取文件的方式实现,此处不进行讲解
arr=['500104','500105','500106','500107','500108']
#API的URL,在这里进行了结构化处理
url1="https://restapi.amap.com/v3/place/text?keywords=汽车服务&city="
url2="&output=JSON&offset=20&key=key&extensions=all&page="
#用于储存数据
x=[]
#用于计数
num=0
#循环各下级行政区进行POI检索
for i in range(0,len(arr)):
#当前行政区
city=arr[i]
#因为官方对API检索进行了45页限制,所以只要检索到45页即可
for page in range(1,46):
#若该下级行政区的POI数量达到了限制,则警告使用者,之后考虑进行POI类型切分
if page==45:
print("警告!!POI检索可能受到限制!!")
#构造URL
thisUrl=url1+city+url2+str(page)
#获取POI数据
data=requests.get(thisUrl)
#转为JSON格式
s=data.json()
#解析JSON
aa = s["pois"]
#若解析的JSON为空,即当前行政区的数据不够45页(即没有达到限制),返回
if len(aa)==0:
break
#对每条POI进行存储
for k in range(0,len(aa)):
s1=aa[k]["name"]
s2=aa[k]["type"]
if("address" in aa[k]):
s3 = aa[k]["address"]
else:
s3 = ""
s4=aa[k]["adname"]
s5=aa[k]["location"].split(",")
x.append([s1,s2,s3,s4,float(s5[0]),float(s5[1])])
num+=1
print("爬取了 "+str(num)+" 条数据")
#将数据结构化存储至规定目录的CSV文件中
c = pd.DataFrame(x)
c.to_csv('C:/out/toilet.csv',encoding='utf-8-sig')
尤其是不会受到1000个的限额,爱了爱了
应该写到这里结束了,我要去搞数据了,baibai~