本文基于python3.4的selenium库打开浏览器，并将浏览器中的登陆cookie保存到本地，那么下次登陆就可以直接利用cookie了：
1 # !/usr/bin/python3.4
2 # -*- coding: utf-8 -*-
3
4 from selenium import webdriver
5 import time
6 import requests
7 from bs4 import BeautifulSoup
8 import os
9 import re
10 import random
11 import xlsxwriter
12
13
14 # 找出文件夹下所有xml后缀的文件，可选择递归
15 def listfiles(rootdir, prefix='.xml', iscur=False):
16 file = []
17 for parent, dirnames, filenames in os.walk(rootdir):
18 if parent == rootdir:
19 for filename in filenames:
20 if filename.endswith(prefix):
21 file.append(filename)
22 if not iscur:
23 return file
24 else:
25 if iscur:
26 for filename in filenames:
27 if filename.endswith(prefix):
28 file.append(filename)
29 else:
30 pass
31 return file
32
33
34 # 抓取dp的正则表达式
35 def getdp(string):
36 reg = r'(http.+?/dp/)(.+)'
37 all = re.compile(reg)
38 alllist = re.findall(all, string)
39 return alllist[0][1]
40
41
42 # 抓取filter的正则表达式
43 # https://sellercentral.amazon.com/productsearch?filter=grocery&q=fish
44 def getfilter(string):
45 reg = r'(https.+?filter=)(.+?)(&)'
46 all = re.compile(reg)
47 alllist = re.findall(all, string)
48 return alllist[0][1]
49
50
51 # 抓取最大页数的正则
52 def getpagenum(string):
53 reg = r'(.+?\()(\d+)(\))'
54 all = re.compile(reg)
55 alllist = re.findall(all, string)
56 return alllist[0][1]
57
58
59 # 创建文件夹
60 def createjia(path):
61 try:
62 os.makedirs(path)
63 except:
64 pass
65
66
67 def timetochina(longtime, formats='{}天{}小时{}分钟{}秒'):
68 day = 0
69 hour = 0
70 minutue = 0
71 second = 0
72 try:
73 if longtime > 60:
74 second = longtime % 60
75 minutue = longtime // 60
76 else:
77 second = longtime
78 if minutue > 60:
79 hour = minutue // 60
80 minutue = minutue % 60
81 if hour > 24:
82 day = hour // 24
83 hour = hour % 24
84 return formats.format(day, hour, minutue, second)
85 except:
86 raise Exception('时间非法')
87
88
89 # 打开浏览器抓取cookie
90 def openbrowser(url):
91 # 打开谷歌浏览器
92 # Firefox() Chrome()
93 browser = webdriver.Chrome()
94 # browser = webdriver.Chrome(executable_path='C:/Python34/chromedriver.exe')
95 # 输入网址
96 browser.get(url)
97 # 打开浏览器时间
98 # print("等待10秒打开浏览器...")
99 # time.sleep(10)
100
101 # 找到id="ap_email"的对话框
102 # 清空输入框
103 browser.find_element_by_id("ap_email").clear()
104 browser.find_element_by_id("ap_password").clear()
105
106 # 输入账号密码
107 inputemail = input("请输入账号：")
108 inputpassword = input("请输入密码：")
109 browser.find_element_by_id("ap_email").send_keys(inputemail)
110 browser.find_element_by_id("ap_password").send_keys(inputpassword)
111
112 # 点击登陆sign in
113 # id="signInSubmit"
114 browser.find_element_by_id("signInSubmit").click()
115
116 # 等待登陆10秒
117 # print('等待登陆10秒...')
118 # time.sleep(10)
119 print("等待网址加载完毕...")
120
121 select = input("请观察浏览器网站是否已经登陆(y/n)：")
122 while 1:
123 if select == "y" or select == "Y":
124 print("登陆成功！")
125 # 获取cookie
126 cookie = [item["name"] + ":" + item["value"] for item in browser.get_cookies()]
127 cookiestr = ';'.join(item for item in cookie)
128 print("正在复制网页cookie...")
129
130 # 写入本地txt
131 if "jp" in url:
132 path = "../data/Japcookie.txt"
133 else:
134 path = "../data/Amecookie.txt"
135
136 filecookie = open(path, "w")
137 filecookie.write(cookiestr)
138 filecookie.close()
139
140 time.sleep(1)
141 print("准备关闭浏览器...")
142 browser.quit()
143 # print(cookiestr)
144 break
145
146 elif select == "n" or select == "N":
147 selectno = input("账号密码错误请按0，验证码出现请按1...")
148 # 账号密码错误则重新输入
149 if selectno == "0":
150
151 # 找到id="ap_email"的对话框
152 # 清空输入框
153 browser.find_element_by_id("ap_email").clear()
154 browser.find_element_by_id("ap_password").clear()
155
156 # 输入账号密码
157 inputemail = input("请输入账号：")
158 inputpassword = input("请输入密码：")
159 browser.find_element_by_id("ap_email").send_keys(inputemail)
160 browser.find_element_by_id("ap_password").send_keys(inputpassword)
161 # 点击登陆sign in
162 # id="signInSubmit"
163 browser.find_element_by_id("signInSubmit").click()
164
165 elif selectno == "1":
166 # 验证码的id为id="ap_captcha_guess"的对话框
167 input("请在浏览器中输入验证码并登陆...")
168 select = input("请观察浏览器网站是否已经登陆(y/n)：")
169
170 else:
171 print("请输入“y”或者“n”！")
172 select = input("请观察浏览器网站是否已经登陆(y/n)：")
173
174 return cookiestr
175
176
177 def gethtml(url):
178 # 读取cookie
179 # 写入字典
180 mycookie = {}
181 if "jp" in url:
182 path = "../data/Japcookie.txt"
183 else:
184 path = "../data/Amecookie.txt"
185
186 try:
187 filecookie = open(path, "r")
188 cookies = filecookie.read().split(";")
189 for items in cookies:
190 item = items.split(":")
191 mycookie[item[0]] = item[1]
192 # print(mycookie)
193 filecookie.close()
194 except:
195 print("cookie为空...")
196
197 if "jp" in url:
198 referer = "https://sellercentral.amazon.co.jp/"
199 host = "www.amazon.co.jp"
200 else:
201 referer = "https://sellercentral.amazon.com/"
202 host = "www.amazon.com"
203
204 # 制作头部
205 header = {
206 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0',
207 'Referer': referer,
208 'Host': host,
209 'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
210 'Connection': 'keep-alive',
211 'Upgrade-Insecure-Requests': '1',
212 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
213 'Accept-Encoding': 'gzip, deflate, br'
214 }
215
216 htmlget = requests.get(url=url, headers=header, cookies=mycookie, timeout=60)
217 htmlcontent = htmlget.content.decode("UTF-8", "ignore")
218
219 return htmlcontent
220
221
222 def getinfo(html, Loginurl):
223 # BeautifulSoup解析需要的东西
224 soups = BeautifulSoup(html, "html.parser")
225 # 筛选出商品的div
226 sellyours = soups.find_all("div", attrs={"class": "a-box product"})
227 information = []
228 for item in sellyours:
229 # 一个一个商品筛选
230 # 第一次筛选，得到有“出售您的”的商品列表
231 temp = item.find("a", attrs={"class", "a-button-text"})
232
233 if temp != None:
234 if "sellYoursClick" in temp["data-csm"]:
235 # 第二次筛选得到“无数字、无新品”字样的商品列表
236 temp = item.find("span", attrs={"class", "offerCountDetails"})
237 if temp == None:
238 temp = item.find("div", attrs={"class", "a-fixed-right-grid-col description a-col-left"})
239
240 # 得到详情页网址
241 hrefurl = temp.find('a').get('href')
242 # 得到所有当前class下的文本信息
243 # 包括title、UPC、EAN、Rank
244 try:
245 spans = temp.get_text()
246 except:
247 spans = "Nothing"
248 # 将得到的文本信息写入数组里面
249 temparr = spans.strip().split("\n")
250 # 正则得到Asin
251 asin = getdp(hrefurl)
252 temparr.append(asin)
253 temparr.append(hrefurl)
254
255 # 这里记录一份副本到txt中，防止程序中断什么都没保存
256 txtcontent = ' '.join(temparr)
257 filename = time.strftime('%Y%m%d', time.localtime())
258 path = "../xls/" + filename
259 createjia(path)
260 file = open(path + "/" + filename + ".txt", "a")
261 file.write("\n" + txtcontent)
262 file.close()
263
264 # 这里是解析详情页，如果详情页有price，就抓取review下来
265 # 并且将抓取的东西储存到数组，并写入excel中
266 # 解析详情页
267 htmldetail = gethtml(hrefurl)
268
269 if 'id="words"' in htmldetail or 'ap_email' in htmldetail or "Amazon.com Page Not Found" in htmldetail:
270 print("抓取得太快！需要重新登陆...")
271 openbrowser(Loginurl)
272 htmldetail = gethtml(hrefurl)
273
274 # BeautifulSoup解析需要的东西
275 soups = BeautifulSoup(htmldetail, "html.parser")
276 # 筛选出商品的centerCol列表
277 centerCols = soups.findAll('div', attrs={'id': "centerCol"})
278 if centerCols:
279 for item in centerCols:
280 temp = item.find("td", attrs={"id": "priceblock_ourprice_lbl"})
281 if temp == None:
282 # 得到评分等级
283 star = item.find("a", attrs={"id": "reviewStarsLinkedCustomerReviews"}).get_text()
284 # 得到评分人数
285 reviews = item.find("span", attrs={"id": "acrCustomerReviewText"}).get_text()
286 # 将抓取的东西写入数组
287 if star:
288 temparr.append(star.strip().replace(" out of 5 stars", ""))
289 else:
290 temparr.append("")
291 if reviews:
292 temparr.append(reviews.strip().replace(" customer reviews", ""))
293 else:
294 temparr.append("")
295
296 information.append(temparr)
297 print(information)
298 else:
299 temparr.append("")
300 temparr.append("")
301 information.append(temparr)
302 print(information)
303 return information
304
305
306 def begin():
307 taoyanbai = '''
308 -----------------------------------------
309 | 欢迎使用后台爬虫系统 |
310 | 时间：2016年10月21日 |
311 | 出品：技术部 |
312 -----------------------------------------
313 '''
314 print(taoyanbai)
315
316
317 if __name__ == "__main__":
318
319 a = time.clock()
320
321 while 1:
322 try:
323 LoginWhere = int(input("抓取美国请按0，日本请按1："))
324 if LoginWhere == 0:
325 Loginurl = "https://sellercentral.amazon.com/"
326 break
327 elif LoginWhere == 1:
328 Loginurl = "https://sellercentral.amazon.co.jp/"
329 break
330 except:
331 print("请正确输入0或1！！")
332 LoginWhere = int(input("抓取美国请按0，日本请按1："))
333
334 keywords = input("请输入查找的关键词：")
335 keyword = keywords.replace(" ", "+")
336
337 print("正在检查登陆状态...")
338
339 if "jp" in Loginurl:
340 seekurl = "https://sellercentral.amazon.co.jp/productsearch?q=" + str(keyword)
341 else:
342 seekurl = "https://sellercentral.amazon.com/productsearch?q=" + str(keyword)
343
344 try:
345 htmlpage = gethtml(seekurl)
346 except Exception as err:
347 input("网络似乎有点问题...")
348 print(err)
349 exit()
350
351 while 1:
352 if 'ap_email' in htmlpage or "Amazon.com Page Not Found" in htmlpage or "<title>404" in htmlpage:
353 print("cookie已经过期，需要重新登陆...")
354 print("等待网页打开...")
355 openbrowser(Loginurl)
356 htmlpage = gethtml(seekurl)
357 else:
358 print("直接使用cookie登陆...")
359 break
360
361 # BeautifulSoup解析需要的东西
362 soups = BeautifulSoup(htmlpage, "html.parser")
363 # 筛选出类别及其网址
364 categorys = soups.findAll('ul', attrs={'class': "a-nostyle a-vertical"})
365 categoryurl = []
366 categoryname = ""
367 pagenum = []
368 filtername = []
369
370 for item in categorys:
371 for temp in item.find_all("a"):
372 hrefurl = temp.get('href')
373 categoryurl.append(hrefurl)
374
375 for temp in item.find_all("span", attrs={"class", "a-color-tertiary"}):
376 spantext = temp.get_text()
377 pagenum.append(getpagenum(spantext))
378 for i in range(0, len(categoryurl)):
379 name = getfilter(categoryurl[i])
380 filtername.append(name)
381 categoryname = categoryname + "抓取(" + str(name) + ")请按" + str(i) + ","
382
383 # 选择抓取的类型
384 try:
385 print(categoryname)
386 selectcategory = int(input("请选择你要抓取类型的数字号码："))
387 except:
388 print("请正确输入前面的数字!!!")
389 print(categoryname)
390 selectcategory = int(input("请选择你要抓取类型的数字编码："))
391
392 filter = filtername[selectcategory]
393 mustpage = int(pagenum[selectcategory]) // 10
394
395 try:
396 print("温馨提醒：（1）后台仅仅展现1000页...（2）你要抓取的类型大约有" + str(mustpage) + "页...")
397 page = int(input("请问你要抓取多少页？（默认15页）："))
398 if page > 1000:
399 print("后台最多只能看到1000页！！！")
400 page = int(input("后台仅仅展现1000页！！！你要抓取的类型大约有" + str(mustpage) + "页！！！请问你要抓取多少页？（默认15页）："))
401 except:
402 page = 15
403
404 # 储存抓取到的东西
405 information = []
406 temparr = []
407
408 for i in range(0, page):
409 try:
410 if "jp" in Loginurl:
411 # https://sellercentral.amazon.co.jp/productsearch?filter=sporting&q=空気入れ&page=2
412 openurl = "https://sellercentral.amazon.co.jp/productsearch?filter=" + str(filter) + "&q=" + str(
413 keyword) + "&page=" + str(i + 1)
414 else:
415 # https://sellercentral.amazon.com/productsearch?filter=pets&q=dog
416 openurl = "https://sellercentral.amazon.com/productsearch?filter=" + str(filter) + "&q=" + str(
417 keyword) + "&page=" + str(i + 1)
418
419 print("开始抓取：" + str(openurl))
420 openhtml = gethtml(openurl)
421
422 # BeautifulSoup解析需要的东西
423 soups = BeautifulSoup(openhtml, "html.parser")
424 # 筛选出商品的div
425 sellyours = soups.findAll('div', attrs={'class': "product"})
426
427 if 'ap_email' in openhtml or "Amazon.com Page Not Found" in openhtml:
428 print("抓取得太快！需要重新登陆...")
429 openbrowser(Loginurl)
430 openhtml = gethtml(openurl)
431
432 elif sellyours == None:
433 print("已经翻到最后一页了...")
434 break
435 temparr = getinfo(openhtml, Loginurl)
436 except Exception as err:
437 print(err)
438 print("访问抓取过程中出现小错误...")
439 print("暂停20秒记录bug并尝试自我修复...")
440 time.sleep(20)
441
442 if temparr:
443 information.append(temparr[0])
444 loadtime = random.randint(5, 10)
445 print("防止反爬虫设定暂停" + str(loadtime) + "秒...")
446 time.sleep(loadtime)
447
448 print("抓到的列表如下：")
449 print(information)
450
451 # 这里写入excel
452 # 创建文件夹
453 filename = time.strftime('%Y%m%d', time.localtime())
454 path = "../xls/" + filename
455 createjia(path)
456
457 # 写入excel
458 timename = time.strftime('%Y%H%M%S', time.localtime())
459 with xlsxwriter.Workbook(path + "/" + timename + '.xlsx') as workbook:
460 # workbook = xlsxwriter.Workbook(path + "/" + timename + '.xlsx')
461 worksheet = workbook.add_worksheet()
462
463 first = ['title', 'UPC', 'EAN', 'Rank', 'Nothing', 'ASIN', 'DetailUrl', 'Star', 'Reviews']
464 # 写入第一行
465 for i in range(0, len(first)):
466 worksheet.write(0, i, first[i])
467 # 写入后面几行
468 for m in range(0, len(information)):
469 for n in range(0, len(information[m])):
470 insert = str(information[m][n]).replace("UPC: ", "").replace("EAN: ", "").replace("Sales Rank:",
471 "").replace(
472 "customer reviews", "").replace("out of 5 stars", "")
473 worksheet.write(m + 1, n, insert)
474 workbook.close()
475
476 b = time.clock()
477 print('运行时间：' + timetochina(b - a))
478 input('请关闭窗口') ##防止运行完毕后窗口直接关闭而看不到运行时间
由于selenium库支持低版本的浏览器，例如本文的谷歌浏览器需要下载插件，并将插件放到目录C:\Python34即可：
插件为chromedriver.exe，自己搜索，网上很多哒
