也是按自己的想法去实现的。
每个线程下载完分配的文件之后，就退出来。
感觉是比单个下载快，但如何提高了性能，现在只能是看时间。
至于在下载什么。。。真是不好细说，哈哈哈。。私用，私用。
from BeautifulSoup import BeautifulSoup import urllib2,urllib import os,sys,time,re from threading import Thread local_proxies = {'http': 'http://131.139.58.200:8080'} class AxelPython(Thread, urllib.FancyURLopener): '''Multi-thread downloading class. run() is a vitural method of Thread. ''' def __init__(self, threadname, url, ranges=0, proxies={}): Thread.__init__(self, name=threadname) urllib.FancyURLopener.__init__(self, proxies) self.name = threadname self.url = url #self.filename = filename self.ranges = ranges self.downloaded = 0 def run(self): '''vertual function in Thread''' downImg(self.url,save_path) def grabHref(url,localfile): html = urllib2.urlopen(url).read() html = unicode(html,'gb2312','ignore').encode('utf-8','ignore') content = BeautifulSoup(html).findAll('a') myfile = open(localfile,'a') pat = re.compile(r'href="([^"]*)"') pat1 = re.compile(r'movie') pat2 = re.compile(r'http') for item in content: h = pat.search(str(item)) href = h.group(1) if not(pat1.search(href)): continue if pat2.search(href): ans = href else: ans = url+href myfile.write(ans) myfile.write('\r\n') #print ans grabImg(ans) break myfile.close() def grabImg(url): req = urllib2.Request(url); resp = urllib2.urlopen(req); respHtml = resp.read(); songtasteHtmlEncoding = "GB2312"; soup = BeautifulSoup(respHtml, fromEncoding=songtasteHtmlEncoding); foundClassH1user = soup.find(attrs={"class":"bigImage"}); #print "foundClassH1user=%s",foundClassH1user; for item in foundClassH1user: pat = re.compile(r'src="([^"]*)"') h = pat.search(str(item)) href = h.group(1) #downImg(href) #print href #downImg(href,save_path) #filename = href.split("/")[-1] paxel(href,save_path, blocks=2, proxies={} ) def downImg(imgurl,save_path): filename = imgurl.split("/")[-1] dist = os.path.join(save_path, filename) connection = urllib2.build_opener().open(urllib2.Request(imgurl)) urllib.urlretrieve(imgurl, dist,None) print "Done: ", filename #time.sleep(1) def SpliteBlocks(totalsize, blocknumber): blocksize = totalsize/blocknumber ranges = [] for i in range(0, blocknumber-1): ranges.append((i*blocksize, i*blocksize +blocksize - 1)) ranges.append(( blocksize*(blocknumber-1), totalsize -1 )) return ranges def islive(tasks): for task in tasks: if task.isAlive(): return True return False def paxel(href,save_path, blocks=6, proxies=local_proxies): ''' paxel ''' output = href.split("/")[-1] dist = os.path.join(save_path, output) ranges = SpliteBlocks( size, blocks ) threadname = [ "thread_%d" % i for i in range(0, blocks) ] #filename = [ "tmpfile_%d" % i for i in range(0, blocks) ] tasks = [] for i in range(0,blocks): task = AxelPython(threadname[i], href, ranges[i]) task.setDaemon( True ) task.start() tasks.append( task ) time.sleep(0.2) while islive(tasks): sys.stdout.flush() time.sleep(0.1) def main(): localfile = sys.argv[1]+'.txt' if not os.path.exists(save_path): os.mkdir(save_path) for iHref in xrange(int(sys.argv[2]),int(sys.argv[3])): url = "http://www.XXX.XXX/search/" + sys.argv[1] +"-" + str('%0.3d'%(iHref)) grabHref(url,localfile) if __name__=="__main__": save_path = os.path.abspath("./" + sys.argv[1]) size = int(sys.argv[3]) - int(sys.argv[2]) main()
