python异步并发多url请求 python 异步io并发

# 事件循环+回调（驱动生成器）+epool（IO多路复用）
# asyncio 是python用于解决异步io编程的一整套方案
# tornado，gevent，twisted，(scrapy,django channels)
# tornado:协程+事件循环 实现了高并发。实现了web服务器，django+flask
# tornado可以直接部署。真正部署时，还是会采用nginx+tornado

# 使用asyncio
import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    await asyncio.sleep(2)   # 一个阻塞语句，需要添加await，否则会报警，RuntimeWarning: coroutine 'sleep' was never awaited
# 阻塞函数，如time.sleep()不要直接写在协程里面。
    print("end get url")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()   # asyncip已经实现事件循环，只需要调用asyncio.get_event_loop()就可以完成select的操作
    loop.run_until_complete(get_html("http://www.baidu.com"))
    print(time.time()-start_time)

start get url
end get url
2.0040769577026367

# 使用asyncio
import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    await time.sleep(2)   # 如果此处使用了time.sleep(),会报“TypeError: object NoneType can't be used in 'await' expression”
    # time.sleep()无返回，即返回为None，await后面跟的必须是一个awaitable对象
    print("end get url")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()   # asyncip已经实现事件循环，只需要调用asyncio.get_event_loop()就可以完成select的操作
    loop.run_until_complete(get_html("http://www.baidu.com"))
    print(time.time()-start_time)

start get url
Traceback (most recent call last):
  File "C:/Users/Amber/PycharmProjects/test0/Chapter13/loop_test.py", line 20, in <module>
    loop.run_until_complete(get_html("http://www.baidu.com"))
  File "C:\Users\Amber\Anaconda3\lib\asyncio\base_events.py", line 579, in run_until_complete
    return future.result()
  File "C:/Users/Amber/PycharmProjects/test0/Chapter13/loop_test.py", line 13, in get_html
    await time.sleep(2)   # 如果此处使用了time.sleep(),会报“TypeError: object NoneType can't be used in 'await' expression”
TypeError: object NoneType can't be used in 'await' expression

# 使用asyncio
import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    #time.sleep(2)   # time.sleep()未报错，但它是阻塞io，不可用。为什么？
    await asyncio.sleep(2)
    print("end get url")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()   # asyncip已经实现事件循环，只需要调用asyncio.get_event_loop()就可以完成select的操作
    tasks = [get_html("http://www.baidu.com") for i in range(10)]
    loop.run_until_complete(asyncio.wait(tasks))
    print(time.time()-start_time)

start get url
start get url
start get url
start get url
start get url
start get url
start get url
start get url
start get url
start get url
end get url
end get url
end get url
end get url
end get url
end get url
end get url
end get url
end get url
end get url
2.0032663345336914

有上可以，采用asyncio，提交10个任务，瞬间10个任务并发执行完毕，返回结果2秒。

# 使用asyncio
import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    time.sleep(2)   # time.sleep()是阻塞io，不可用。为什么？ 如果使用它，就无法并行，time.sleep()是同步的操作方式。
    # 总执行时间20秒。属于单线程逻辑，与顺序执行无区别
    # await asyncio.sleep(2) # 会返回future，会立即返回，下次返回会去看前面有无完成。
    print("end get url")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()   # asyncip已经实现事件循环，只需要调用asyncio.get_event_loop()就可以完成select的操作
    tasks = [get_html("http://www.baidu.com") for i in range(10)]
    loop.run_until_complete(asyncio.wait(tasks))
    print(time.time()-start_time)

start get url
end get url
start get url
end get url
start get url
end get url
start get url
end get url
start get url
end get url
start get url
end get url
start get url
end get url
start get url
end get url
start get url
end get url
start get url
end get url
20.008148908615112

如果用asyncio，传统的requests库是不可用的。

# 获取返回值

import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    await asyncio.sleep(2) # 会返回future，会立即返回，下次返回会去看前面有无完成。
    print("end get url")
    return ("Tom")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()   # asyncip已经实现事件循环，只需要调用asyncio.get_event_loop()就可以完成select的操作
    get_future = asyncio.ensure_future(get_html("http://www.baidu.com"))  # 等效未loop.create_task()
    loop.run_until_complete(get_future)
    print(get_future.result())  # Tom

start get url
end get url
Tom

# 获取返回值

import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    await asyncio.sleep(2) # 会返回future，会立即返回，下次返回会去看前面有无完成。
    print("end get url")
    return ("Tom")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()   # asyncip已经实现事件循环，只需要调用asyncio.get_event_loop()就可以完成select的操作
    #get_future = asyncio.ensure_future(get_html("http://www.baidu.com"))  # 等效未loop.create_task()
    task = loop.create_task(get_html("http://www.baidu.com"))
    loop.run_until_complete(task)
    print(task.result())  # Tom

import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    await asyncio.sleep(2) # 会返回future，会立即返回，下次返回会去看前面有无完成。
    print("end get url")
    return ("Tom")

def call_back(future):  # 如果只写call_back()，会报错，TypeError: call_back() takes 0 positional arguments but 1 was given，
    # 默认会将future传递进来。所以此处应该为call_back(future)
    print("send mail to Bom")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()   # asyncip已经实现事件循环，只需要调用asyncio.get_event_loop()就可以完成select的操作
    #get_future = asyncio.ensure_future(get_html("http://www.baidu.com"))  # 等效未loop.create_task()
    task = loop.create_task(get_html("http://www.baidu.com"))
    task.add_done_callback(call_back)
    loop.run_until_complete(task)
    print(task.result())  # Tom

start get url
end get url
send mail to Bom
Tom

# 获取返回值

import asyncio # 将asyncio理解为协程池
import time
from functools import partial  # 可以将函数包装为另一个函数，实现参数传递

async def get_html(url):
    print("start get url")
    await asyncio.sleep(2) # 会返回future，会立即返回，下次返回会去看前面有无完成。
    print("end get url")
    return ("Tom")

def call_back(url,future):  # 如果只写call_back()，会报错，TypeError: call_back() takes 0 positional arguments but 1 was given，
    # 默认会将future传递进来。所以此处应该为call_back(future)
    print(url)
    print("send mail to Bom")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()   # asyncip已经实现事件循环，只需要调用asyncio.get_event_loop()就可以完成select的操作
    #get_future = asyncio.ensure_future(get_html("http://www.baidu.com"))  # 等效未loop.create_task()
    task = loop.create_task(get_html("http://www.baidu.com"))
    task.add_done_callback(partial(call_back,"http://www.baidu.com")) # 返回是函数名称，不是调用
    loop.run_until_complete(task)
    print(task.result())  # Tom

start get url
end get url
http://www.baidu.com
send mail to Bom
Tom

# 使用wait
import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    await asyncio.sleep(2) 
    print("end get url")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()   
    tasks = [get_html("http://www.baidu.com") for i in range(10)]
    loop.run_until_complete(asyncio.wait(tasks))
    print(time.time()-start_time)

# 使用gather
import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    await asyncio.sleep(2)
    print("end get url")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()
    tasks = [get_html("http://www.baidu.com") for i in range(10)]

# gather与wait的区别
# gather更加高层，出来完成上述功能外，还可以将task分组。
    group1 = [get_html("http://www.sohu.com") for i in range(2)]
    group2 = [get_html("http://www.163.com") for i in range(2)]
    loop.run_until_complete(asyncio.gather(*group1,*group2))  # 只需要等待group完成即可
    print(time.time()-start_time)

start get url
start get url
start get url
start get url
end get url
end get url
end get url
end get url
2.002568006515503
sys:1: RuntimeWarning: coroutine 'get_html' was never awaited

# 使用gather
import asyncio # 将asyncio理解为协程池
import time

async def get_html(url):
    print("start get url")
    await asyncio.sleep(2)
    print("end get url")

if __name__=="__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()
    tasks = [get_html("http://www.baidu.com") for i in range(10)]

# gather与wait的区别
# gather更加高层，出来完成上述功能外，还可以将task分组。
    group1 = [get_html("http://www.sohu.com") for i in range(2)]
    group2 = [get_html("http://www.163.com") for i in range(2)]
    group1=asyncio.gather(*group1)
    group2=asyncio.gather(*group2)
    loop.run_until_complete(asyncio.gather(group1,group2))
    print(time.time()-start_time)

start get url
start get url
start get url
start get url
end get url
end get url
end get url
end get url
2.00278377532959
sys:1: RuntimeWarning: coroutine 'get_html' was never awaited

# gather更加高层，使用更灵活，只有在有些定制性特别强的场合使用wait 可以用group.cancel()批量取消任务