Python网络爬虫（Xpath解析, lxml库, selenium）

    sudo apt-get install libssl-dev
       sudo apt-get install libffi-dev 
       sudo apt-get install python3-dev
       sudo apt-get install build-essential
       sudo apt-get install libxml2
       sudo apt-get install libxml2-dev
       sudo apt-get install libxslt1-dev
       sudo apt-get install zlib1g-dev
    sudo pip3 install Scrapy

        //book/title[@lang="en"]

        /bookstore/book[2]/title/text()

           //price  、  /bookstore/book//price

           //title[@lang="en"]

      python -m pip install lxml
      conda install lxml

# 构造解析对象
parseHtml = etree.HTML(html)
# 利用解析对象调用xpath匹配
r1 = parseHtml.xpath('//a/@href')
print(r1)

# 获取 /
r2 = parseHtml.xpath('//a[@id="channel"]/@href')
print(r2)

# 获取非 /
r3 = parseHtml.xpath('//ul[@id="nav"]//a/@href')
print(r3)
# 获取所有 a 节点的文本内容
r4 = parseHtml.xpath('//a/text()')
print(r4)
# 获取 图片、军事 ... 
r5 = parseHtml.xpath('//ul[@id="nav"]//a')
for i in r5:
    print(i.text)

'''02_百度贴吧图片抓取案例.py'''
import requests
from lxml import etree
import time

class BaiduImageSpider:
    def __init__(self):
        self.headers = {"User-Agent":"Mozilla/5.0"}
        self.baseurl = "http://tieba.baidu.com"
        self.pageurl = "http://tieba.baidu.com/f?"
        
    # 获取所有帖子URL列表
    def getPageUrl(self,params):
        res = requests.get(self.pageurl,params=params,headers=self.headers) 
        res.encoding = "utf-8"
        html = res.text
        # 构建解析对象
        parseHtml = etree.HTML(html)
        # 帖子链接列表
        t_list = parseHtml.xpath('//div[@class="t_con cleafix"]/div/div/div/a/@href')
        # t_list : ['/p/233432','/p/2039820',..]
        print(t_list)
        for t_link in t_list:
            # 拼接帖子完整链接
            t_link = self.baseurl + t_link
            self.getImageUrl(t_link)
    
    # 获取帖子中图片URL列表
    def getImageUrl(self,t_link):
        res = requests.get(t_link,headers=self.headers)
        res.encoding = "utf-8"
        html = res.text
        # 构造解析对象
        parseHtml = etree.HTML(html)
        img_list = parseHtml.xpath('//img[@class="BDE_Image"]/@src')
        print(img_list)
        for img_link in img_list:
            self.writeImage(img_link)
    
    # 保存到本地
    def writeImage(self,img_link):
        # 获取图片的bytes
        res = requests.get(img_link,headers=self.headers)
        res.encoding = "utf-8"
        html = res.content
        # filename
        filename = img_link[-12:]
        with open(filename,"wb") as f:
            f.write(html)
            time.sleep(0.5)
            print("%s下载成功" % filename)
    
    # 主函数
    def workOn(self):
        name = input("请输入贴吧名:")
        begin = int(input("请输入起始页:"))
        end = int(input("请输入终止页:"))
        
        for n in range(begin,end+1):
            pn = (n-1)*50
            params = {
                    "kw":name,
                    "pn":str(pn)
                }
            self.getPageUrl(params)
            
if __name__ == "__main__":
    spider = BaiduImageSpider()
    spider.workOn()

'''03_糗事百科案例.py'''
import requests
from lxml import etree
import pymongo

class QiuShiSpider:
    def __init__(self):
        self.url = "https://www.qiushibaike.com/8hr/page/1/"
        self.headers = {"User-Agent":"Mozilla/5.0"}
        self.conn = pymongo.MongoClient("localhost",27017)
        self.db = self.conn.Baikedb
        self.myset = self.db.baikeset
        
    def getPage(self):
        res = requests.get(self.url,headers=self.headers)
        res.encoding = "utf-8"
        html = res.text
        self.parsePage(html)
    
    def parsePage(self,html):
        parseHtml = etree.HTML(html)
        # 基准xpath,每个段子的列表
        base_list = parseHtml.xpath('//div[contains(@id,"qiushi_tag_")]')
        # 遍历每个段子的节点对象(base)
        for base in base_list:
            # 用户昵称
            username = base.xpath('./div/a/h2')
            if len(username) == 0:
                username = "匿名用户"
            else:
                username = username[0].text
            # 段子内容
            content = base.xpath('.//div[@class="content"]/span')[0].text
            # 好笑数量
            # [<element.好笑数量>,<eleme.评论>,<element...>]
            laughNum = base.xpath('.//i')[0].text 
            # 评论数量
            pingNum = base.xpath('.//i')[1].text
            
            d = {
                  "username":username.strip(),
                  "content":content.strip(),
                  "laughNum":laughNum.strip(),
                  "pingNum":pingNum.strip()
              }
            self.myset.insert(d)
            print("存入数据库成功")


if __name__ == "__main__":
    spider = QiuShiSpider()
    spider.getPage()