代码如下：
# 爬取国家粮食数据
import os
import time
import requests
SPICE_URL = 'https://comtrade.un.org/api/get?type=C&freq=M&head=H&fmt=csv&rg=ALL&px=HS&' # 需要拼接的字符串
#需要修改的部分
R_VAL = 'r=' #发送端国家代码
P_VAL = 'p=' #接收端国家代码
PS_VAL= 'ps=' #时间代码
CC_VAL = 'cc=' #商品代码
# csv命名格式：商品代码+时间+R+P 保证唯一性
def downloadFile(name, url):
'''
:param name:下载保存的名称
:param url: 下载链接
:return:
'''
headers = {'Proxy-Connection': 'keep-alive'}
r = requests.get(url, stream=True, headers=headers)
length = float(r.headers['content-length'])
f = open(name, 'wb')
count = 0
count_tmp = 0
time1 = time.time()
for chunk in r.iter_content(chunk_size=512):
if chunk:
f.write(chunk)
count += len(chunk)
if time.time() - time1 > 2:
p = count / length * 100
speed = (count - count_tmp) / 1024 / 1024 / 2
count_tmp = count
print(name + ': ' + formatFloat(p) + '%' + ' Speed: ' + formatFloat(speed) + 'M/S')
time1 = time.time()
f.close()
def formatFloat(num):
return '{:.2f}'.format(num)
RCOUNTRY_PATH = 'country.txt' # RCountry读取路径
PCOUNTRY_PATH = 'pcountry.txt' #PCountry读取路径
CC_PATH = 'shop.txt'
PS_PATH = 'ps.txt'
# 开始读取拼接
r_file = open(RCOUNTRY_PATH)
p_file = open(PCOUNTRY_PATH)
cc_file = open(CC_PATH)
ps_file = open(PS_PATH)
for r_line in r_file:
r_str = r_line.strip()
for p_line in p_file:
p_str = p_line.strip()
for cc_line in cc_file:
cc_str = cc_line.strip()
for ps_line in ps_file:
ps_str = ps_line.strip()
csv_url = SPICE_URL + CC_VAL + cc_str + '&' + R_VAL + r_str + '&' + P_VAL + p_str+'&'+PS_VAL+ps_str
print(csv_url)
csv_name = ps_str+cc_str+p_str+r_str+'.csv'
downloadFile(csv_name,csv_url)
time.sleep(3)
#csv_url = SPICE_URL+'&'+CC_VAL+cc_str+'&'+R_VAL+r_str+'&'+P_VAL+p_str
cc_file.close()
ps_file.close()
p_file.close()
r_file.close()
