Python通过future处理并发

 1 import os
 2 import time
 3 import sys
 4 
 5 import requests
 6 
 7 
 8 POP20_CC = (
 9     "CN IN US ID BR PK NG BD RU JP MX PH VN ET EG DE IR TR CD FR"
10 ).split()
11 
12 
13 BASE_URL = 'http://flupy.org/data/flags'
14 
15 DEST_DIR = 'downloads/'
16 
17 
18 def save_flag(img,filename):
19     path = os.path.join(DEST_DIR,filename)
20     with open(path,'wb') as fp:
21         fp.write(img)
22 
23 
24 def get_flag(cc):
25     url = "{}/{cc}/{cc}.gif".format(BASE_URL,cc=cc.lower())
26     resp = requests.get(url)
27     return resp.content
28 
29 
30 def show(text):
31     print(text,end=" ")
32     sys.stdout.flush()
33 
34 
35 def download_many(cc_list):
36     for cc in sorted(cc_list):
37         image = get_flag(cc)
38         show(cc)
39         save_flag(image,cc.lower()+".gif")
40 
41     return len(cc_list)
42 
43 
44 def main(download_many):
45     t0 = time.time()
46     count = download_many(POP20_CC)
47     elapsed = time.time()-t0
48     msg = "\n{} flags downloaded in {:.2f}s"
49     print(msg.format(count,elapsed))
50 
51 
52 if __name__ == '__main__':
53     main(download_many)

 1 from concurrent import futures
 2 
 3 from flags import save_flag, get_flag, show, main
 4 
 5 
 6 MAX_WORKERS = 20
 7 
 8 
 9 def download_one(cc):
10     image = get_flag(cc)
11     show(cc)
12     save_flag(image, cc.lower()+".gif")
13     return cc
14 
15 
16 def download_many(cc_list):
17     workers = min(MAX_WORKERS,len(cc_list))
18     with futures.ThreadPoolExecutor(workers) as executor:
19         res = executor.map(download_one, sorted(cc_list))
20 
21     return len(list(res))
22 
23 
24 if __name__ == '__main__':
25     main(download_many)

 1 from concurrent import futures
 2 
 3 from flags import save_flag, get_flag, show, main
 4 
 5 
 6 MAX_WORKERS = 20
 7 
 8 
 9 def download_one(cc):
10     image = get_flag(cc)
11     show(cc)
12     save_flag(image, cc.lower()+".gif")
13     return cc
14 
15 
16 def download_many(cc_list):
17     cc_list = cc_list[:5]
18     with futures.ThreadPoolExecutor(max_workers=3) as executor:
19         to_do = []
20         for cc in sorted(cc_list):
21             future = executor.submit(download_one,cc)
22             to_do.append(future)
23             msg = "Secheduled for {}:{}"
24             print(msg.format(cc,future))
25 
26         results = []
27         for future in futures.as_completed(to_do):
28             res = future.result()
29             msg = "{}result:{!r}"
30             print(msg.format(future,res))
31             results.append(res)
32 
33     return len(results)
34 
35 
36 if __name__ == '__main__':
37     main(download_many)