python:python的多进程

https://docs.python.org/3.6/library/multiprocessing.html#examples

从这里改造而来:

import time
from urllib import request
from multiprocessing import Process, Queue, current_process, freeze_support,cpu_count


def worker2(input, output):
    for func, args in iter(input.get, 'STOP'):
        result = nety(func, args)
        output.put(result)
def nety(func, args):
    result = func(args)
    return 'process: %s  done func:%s url:%s' % \
        (current_process().name, func.__name__, args)


def getUrl(url):
    res=request.urlopen(url)
    #return res.read().decode('UTF-8')
    return res


def test():
    NUMBER_OF_PROCESSES = cpu_count()*2
    urllist=['http://115.239.211.112/']*100
    TASKS3=[(getUrl, url) for url in urllist]

    # Create queues
    task_queue = Queue()
    done_queue = Queue()

    # Submit tasks
    '''
    for task in TASKS1:
        task_queue.put(task)
    '''
    for task in TASKS3:
        task_queue.put(task)
    # Start worker processes
    for i in range(NUMBER_OF_PROCESSES):
        Process(target=worker2, args=(task_queue, done_queue)).start()

    # Get and print results
    print('Unordered results:')
    for i in range(len(TASKS3)):
        print('\t', done_queue.get())
        
    # Tell child processes to stop
    for i in range(NUMBER_OF_PROCESSES):
        task_queue.put('STOP')


if __name__ == '__main__':
    freeze_support()
    startTime=time.time()
    test()
    endTime=time.time()
    print(endTime-startTime)

改进版本:

import time
from urllib import request
from multiprocessing import Process, Queue, current_process,cpu_count,freeze_support
import os

def worker2(queueIn, queueOut):
    for func, args in iter(queueIn.get, 'STOP'):
        result = netty(func, args)
        queueOut.put(result)


def netty(func, args):
    result = func(args)
    #return ('process: %s  done func:%s url:%s' % (current_process().name, func.__name__, args),result)
    return (args, result) #返回传入的参数和输出的结果

def getUrl(url):
    res=request.urlopen(url)
    #return res.read().decode('UTF-8')
    return res.read()


def test():
    NUMBER_OF_PROCESSES = cpu_count()*1
    #urllist=['http://192.168.199.119:8080/']*10
    urllist = []
    urlFileName = 'urls\\urls.txt'
    with open(urlFileName,'r') as f:
        for line in f.readlines():
            urllist.append(line.replace('\n',''))
    urllist = urllist*5



    TASKS3 = [(getUrl, url) for url in urllist]

    # Create queues
    task_queue = Queue()
    done_queue = Queue()

    # Submit tasks
    for task in TASKS3:
        task_queue.put(task)
    # Start worker processes
    for i in range(NUMBER_OF_PROCESSES):
        Process(target=worker2, args=(task_queue, done_queue)).start()

    # Get and print results
    print('Unordered results:')
    for i in range(len(TASKS3)):
        # print(done_queue.get())
        res=done_queue.get()
        url=res[0]
        data=res[1]
        #fileName = 'DATA\\'+url[7:].replace('/', '').replace(':', '_')+'_'+str(i)+'.txt'
        #fileName='DATA\\'+'data'+str(i)+'.data'
        fileName='DATA\\'+os.path.basename(url).replace('?','_')+'_'+str(i)+'.data'
        with open(fileName,'wb') as f:
            f.write(data)
    # Tell child processes to stop
    for i in range(NUMBER_OF_PROCESSES):
        task_queue.put('STOP')


if __name__ == '__main__':
    freeze_support()
    startTime = time.time()
    test()
    endTime = time.time()
    print(endTime-startTime)