# -*- coding:utf8 -*-
import string
import urllib2
import re
import time
import random
class CSDN_Spider:
    def __init__(self,url):
        self.myUrl = url
        self.datas = []
        print u"爬虫已启动...."
    def csdn(self):
        url = self.myUrl + "?viewmode=list"
        user_agents = [
        'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
        'Opera/9.25 (Windows NT 5.1; U; en)',
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
        'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
        'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
        'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
        "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
        "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 ",
        ]
        agent = random.choice(user_agents)
        req = urllib2.Request(url)
        req.add_header('User-Agent', agent)
        req.add_header('Host', 'blog.csdn.net')
        req.add_header('Accept', '*/*')
        req.add_header('Referer', 'http://blog.csdn.net/djd1234567?viewmode=contents')
        req.add_header('GET', url)
        mypage = urllib2.urlopen(req).read().decode("utf8")
        #print mypage
        Pagenum = self.page_counter(mypage)
        #print Pagenum
        self.find_data(self.myUrl,Pagenum)
    def page_counter(self,mypage):#<a href="/yangshangwei/article/list/11">尾页</a>
        myMatch = re.search(u'/article/list/(\d+?)">尾页</a>',mypage,re.S)
        if myMatch:
            Pagenum = int(myMatch.group(1))
            print u"爬虫报告:发现目录一共%d页" %Pagenum
        else:
            Pagenum = 0
            print u"爬虫报告:没找到页面的数量"
        return Pagenum
    def find_data(self,myurl,Pagenum):
        name = myurl.split("/")
        f = open(name[-1] + '.txt','w+')
        for i in range(1,Pagenum+1):
            print i
            print u"爬虫报告:第%d页正在加载中......" % i
            url = myurl + "/article/list/" + str(i)
            user_agents = [
            'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
            'Opera/9.25 (Windows NT 5.1; U; en)',
            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
            'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
            'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
            'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
            "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
            "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 ",
            ]
            agent = random.choice(user_agents)
            req = urllib2.Request(url)
            req.add_header('User-Agent', agent)
            req.add_header('Host', 'blog.csdn.net')
            req.add_header('Accept', '*/*')
            req.add_header('Referer', url)
            req.add_header('GET', url)
            mypage = urllib2.urlopen(req).read()
            myItems = re.findall(u'"><a href="/' + myurl.split("/")[-1] + '/article/details/(\d+?)" title="',mypage,re.S)
            print myItems
            for item in myItems:
                self.datas.append("http://blog.csdn.net/yangshangwei/article/details/" + item+"\n")
            #time.sleep(1)
        f.writelines(self.datas)
        f.close()
        print self.datas
        print u"爬虫报告:txt文件生成,请在当前目录查看"
url = "http://blog.csdn.net/yangshangwei"
mySpider = CSDN_Spider(url)
mySpider.csdn()