python多线程限制文件数 python多线程读写文件

IO操作会造成程序阻塞耗时，因此采用 QUEUE + 多线程方式让IO操作不再阻塞主程序的运行，这样可以减少主程序的等待时间

from concurrent.futures import ThreadPoolExecutor
import time
import threading

data_queue=[]  # event_id: [data]   | event_id: [data1, data2, ... ]

def call_write_data(data):
    print('push writer data to queue')
    print('call writer process to write data' + str(data))
    queue_done = push2queue(data)
    print('pushed to queue done: ' + queue_done)

def push2queue(data):
    data_queue.append(data)
    return 'OK. write done: ' + str(data)


def do_call_internal(seconds_internal=5):
    for i in range (10):
        call_write_data(i)
        time.sleep(seconds_internal)

    time.sleep(100)
    for i in range(20,25):
        call_write_data(i)
        time.sleep(seconds_internal)


def do_writer():
    print('start to do writer')
    while True:
        while len(data_queue) > 0:
            print('this is to write data : ' + str(data_queue[0]))
            print(data_queue)
            time.sleep(6)
            data_queue.pop(0)
            print('write done')
            print(data_queue)

        while len(data_queue) <= 0:
            print('no data in queue, will sleep 3 seconds . . . ')
            time.sleep(3)


def binary_main():
    # for i in range(5):
    #     call_write_function(i)
    #     t_writer = threading.Thread(target=do_writer)
    #     print(t_writer.name)
    #     t_writer.start()
    #     print(threading.active_count())
    #     t_writer.join()

    # t_push = threading.Thread(target=call_write_function, args = (10,))
    t_push = threading.Thread(target=do_call_internal, args = (3,))
    t_writer = threading.Thread(target=do_writer)

    print(t_push.name)
    print(t_writer.name)
    t_push.start()
    t_writer.start()
    print(threading.active_count())
    t_push.join()
    t_writer.join()

binary_main()

将所有的要读的、要写的文件放到task list中，这样所有文件的读写都是异步的，速度比较快 

async def grpc_interface(opt_type, source_file, target_dir = './logs'):
    if opt_type =='read':
        print('read file ' + source_file)
        await do_read(source_file)
        print('done read file ' + source_file)
    else:
        print('write file ' + source_file)
        await do_write(source_file, target_dir)
        print('done write file ' + source_file)

    return

async def do_read(file):
    with open(file, 'rb') as fr:
        content = fr.read()
        await asyncio.sleep(5)
    return

async def do_write(source_file, target_dir):
    with open(source_file,'rb') as fr:
        content = fr.read()

    with open(os.path.join(target_dir, 'test.writer'), 'wb') as fw:
        fw.write(content)
        await asyncio.sleep(10)

    return

source_dir ='/home/lc/视频/dup_sub'
files = os.listdir(source_dir)
ioloop = asyncio.get_event_loop()

# tasks中也可以使用asyncio.ensure_future(gr1())..
tasks = [
    # grpc_interface('read', os.path.join(source_dir, file)) for file in files
    asyncio.ensure_future(grpc_interface('read', os.path.join(source_dir, file))) for file in files
]
tasks.extend([asyncio.ensure_future(grpc_interface('write', os.path.join(source_dir,file))) for file in files])
ioloop.run_until_complete(asyncio.wait(tasks))
ioloop.close()

读写folder： 只是读folder 和 写 folder是异步，读folder内的文件 和 写folder内的文件 是单线程，非异步，因此较上一种肉眼可见的慢

async def grpc_interface(opt_type, source_dir, target_dir = './logs'):
    if opt_type =='read':
        await do_read(source_dir)
    else:
        await do_write(source_dir, target_dir)
    return

async def do_read(file_dir):
    files = os.listdir(file_dir)
    for file in files:
        print('read file ' + file)
        with open(os.path.join(file_dir, file), 'rb') as fr:
            content = fr.read()
            await asyncio.sleep(5)
        print('done read file ' + file)
    return

async def do_write(source_dir, target_dir):
    files = os.listdir(source_dir)
    with open(os.path.join(target_dir, 'test.writer'), 'wb') as fw:
        for file in files:
            print('write file' + file)
            with open(os.path.join(source_dir, file), 'rb') as fr:
                content = fr.read()
                fw.write(content)
                await asyncio.sleep(10)
        print('done write file ' + file)
    return

source_dir ='/home/lc/视频/dup_sub'
files = os.listdir(source_dir)
ioloop = asyncio.get_event_loop()

# tasks中也可以使用asyncio.ensure_future(gr1())..
tasks = [
    # grpc_interface('read', os.path.join(source_dir, file)) for file in files
    asyncio.ensure_future(grpc_interface('read', source_dir)),
    asyncio.ensure_future(grpc_interface('write', source_dir))
]
ioloop.run_until_complete(asyncio.wait(tasks))
ioloop.close()