<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html><head><meta http-equiv="Cont-阿里云开发者社区

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html><head><meta http-equiv="Cont

2019-06-01 695

版权

本文内容由阿里云实名注册用户自发贡献，版权归原作者所有，阿里云开发者社区不拥有其著作权，亦不承担相应法律责任。具体规则请查看《阿里云开发者社区用户服务协议》和《阿里云开发者社区知识产权保护指引》。如果您发现本社区中有涉嫌抄袭的内容，填写侵权投诉表单进行举报，一经查实，本社区将立刻删除涉嫌侵权内容。

本文涉及的产品

转发路由器TR，750小时连接 100GB跨地域

简介： #!C:\Python27\python.exe#coding=utf8import osimport pdfkitimport urllib2from bs4 import BeautifulSoupfr...

#!C:\Python27\python.exe
#coding=utf8

import os
import pdfkit
import urllib2
from bs4 import BeautifulSoup
from multiprocessing import Pool


import socket
socket.setdefaulttimeout(60)

import sys
reload(sys)
sys.setdefaultencoding('utf-8')


def url_open(url):
    user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36'
    headers = {'User-Agent': user_agent}
    request = urllib2.Request(url=url, headers=headers)
    try:
        page = urllib2.urlopen(request, timeout=60)
    except urllib2.HTTPError as e:
        return 1
    contents = page.read()
    # print contents
    soup = BeautifulSoup(contents.decode('utf-8','ignore'), "lxml")
    return soup


def retrieve_pdf(dir, link):
    savedStderr = sys.stderr
    with open('errlog.txt', 'w+') as file:
        sys.stderr = file
        try:
            pdfkit.from_url(link, dir)
            pass
        finally:
            pass
    sys.stderr = savedStderr


def strip_char(string):
    char = ['*', '/', '\\', ':', '"', '?', '<', '>', '|']
    processed = []
    for i in string:
        if i not in char:
            processed.append(i)
    return ''.join(processed)


def crawler(root, url, num):
        # print url
        if url_open(url) != 1:
            soup = url_open(url)
            # print soup
            for tr in soup.find_all("tr"):
                # print tr
                td = tr.find_all('td')
                if list(td) == None:
                    continue
                if len(td) > 0:
                    if td[0].get_text() == u"提交时间":
                        continue
                    date = td[0].get_text()
                    title = td[1].get_text()
                    dir = title + '.pdf'
                    type = td[2].get_text()
                    poster = td[3].get_text()
                    print date + "  " + title + "   " + type + "    " + poster
                    link = root + '.'.join(tr.get('onclick').split('\'')[1].split('.')[1:])
                    print link
                    print "Retrieving PDF..."
                    print dir
                    dir = strip_char(dir).encode('utf-8').decode('utf-8')
                    temp_name = 'temp' + str(num) + '.pdf'
                    try:
                        retrieve_pdf(temp_name, link)
                    except Exception:
                        if os.path.exists(temp_name):
                            print "Retrieved Successfully!"
                            os.rename(temp_name, dir)
                        else:
                            print 'Retrieve failed!'
                        continue


def single_func(num):
    root = 'http://cb.drops.wiki'
    url = "http://cb.drops.wiki/search.php?kind=drops&keywords=&page=" + str(num)
    crawler(root, url, num)


if __name__ == '__main__':
    # single_func(1) #func test
    # for page in range(1, 86):
    #     single_func(page)
    pool = Pool(processes=4)
    for i in range(1, 86):
        result = pool.apply_async(single_func, (i,))
    pool.close()
    pool.join()

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html><head><meta http-equiv="Cont

热门文章

最新文章

相关课程

相关电子书

相关实验场景

热门

活动广场

任务中心

开发者评测

高校计划

乘风者计划

训练营

阿里云MVP

话题

直播

下载

镜像站

技术资料

插件

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html><head><meta http-equiv="Cont

热门文章

最新文章

相关课程

相关电子书

相关实验场景