用xpath、bs4、re爬取B站python数据

import requests,re
from lxml import etree
from bs4 import BeautifulSoup

def get_page(page):
    try:
        #这里要加上cookie
        headers = {
            'Cookie': 'LIVE_BUVID=AUTO5015218915110407; sid=4oag5i0u; fts=1521891539; pgv_pvi=3655305216; UM_distinctid=16257cdfffd2e4-032750a28294b2-3b60450b-100200-16257cdfffe2a0; buvid3=7B94813D-1039-4A88-A1EE-9AEFDF54BE05140244infoc; rpdid=kxsliqpkisdosikxllmww; CURRENT_QUALITY=80; finger=edc6ecda',
            'Host': 'search.bilibili.com',
            'Referer': 'https://www.bilibili.com/',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
        }
        url = 'https://search.bilibili.com/all?keyword=python&from_source=banner_search&spm_id_from=333.334.banner_link.1&page={}'.format(str(page))
        response = requests.get(url,headers=headers)
        if response.status_code == 200:
            return response.text
    except Exception:
        return None

#用xpath解析网页,获取属性值用符号@
def xpath_parse_page(html):
    data = etree.HTML(html)
    items = data.xpath('//*[@class="video-contain clearfix"]/li')
    for item in items:
        yield {
        'video_image' : item.xpath('./a/div/div[1]/img/@src')[0],
        'video_title' : item.xpath('./div/div[1]/a/text()')[0],
        'video_view' : item.xpath('./div/div[3]/span[1]/text()')[0].strip(),
        'video_updateTime' : item.xpath('./div/div[3]/span[3]/text()')[0].strip(),
        'video_up' : item.xpath('./div/div[3]/span[4]/a/text()')[0]
        }

#用bs4解析网页
def bs4_parse_page(html):
    soup = BeautifulSoup(html,'lxml')
    items = soup.find_all('li',{'class':'video matrix'})
    for item in items:
        yield{
        'video_image'  : item.find('img').get('src'),
        'video_title' : item.find('a',{'class':'title'}).get_text(),
        'video_view': item.find('span',{'class':'so-icon watch-num'}).get_text().strip(),
        'video_updateTime' : item.find('span', {'class': 'so-icon time'}).get_text().strip(),
        'video_up': item.find('span',{'title':'up主'}).get_text()#这里用class的属性得到的结果是播放量，要用title
        }

#用正则解析网页，关键是确定好字段的定位，一般来说定位在字段上面的class属性里
def re_parse_page(html):
    pattern = re.compile('<li.*?info.*?title="(.*?)".*?icon-playtime"></i>(.*?)</span>.*?icon-date"></i>(.*?)</span>.*?up-name">(.*?)</a>',re.S)
    items = re.findall(pattern,html)
    for item in items:
        yield {
            'video_title': item[0],
            'video_view': item[1].strip(),
            'video_updateTime':item[2].strip(),
            'video_up': item[3].strip()
        }

def main():
    #处理翻页
    for page in range(1,2):#这里更改爬取的页数
        html = get_page(page)
        for result in bs4_parse_page(html):#这里更改使用哪一种方式解析网页，xpath、bs4、re
            print(result)

if __name__ == '__main__':
    main()