Beaustiful Soup爬虫案例

import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
import time
import random
import json
import re
import pymysql
from tqdm import tqdm

class mySpider:

    # 初始化url
    def __init__(self, url):
        self.url = url
        # 计数，请求一个页面的次数，初始值为1
        self.count = 1
        # 数据库连接对象
        self.db = pymysql.connect(
            host='localhost',
            port=3306,
            user='root',
            password='logicfeng',
            database='test2')
        # 创建游标对象
        self.cursor = self.db.cursor()

    # 结束断开数据库连接
    def __del__(self):
        self.cursor.close()
        self.db.close()
        print("关闭数据库！")

    # 获取一个header
    def getHeader(self):
        # 实例化ua对象
        ua = UserAgent()
        # 随机获取一个ua
        headers = {
   'User-Agent': ua.random}
        return headers

    # 获取请求body
    def getBody(self, url, send_type, data):
        # 每次请求都随机停顿一些时间
        # time.sleep(random.randint(1, 2))
        # 在超时时间内，对于失败页面尝试请求三次
        if self.count <= 3:
            try:
                if send_type == 'get':
                    res = requests.get(url=url, headers=self.getHeader(), params=data, timeout=2)
                elif send_type == 'post':
                    res = requests.post(url=url, headers=self.getHeader(), data=data, timeout=2)
                else:
                    print("未输入send_type，直接返回None")
                    res = None
                return res
            except Exception as e:
                print(e)
                self.count += 1
                print(f"第{self.count}次，发起请求")
                # 再次调用自己,并把值返回，（注意要加return）
                return self.getBody(url, send_type, data)

    # 解析body
    def parseData(self, dataList):
        # 循环查看详情
        for row in tqdm(dataList, desc='爬取进度'):
            # 请求详情页url
            urlDetail = f"https://www.baidu.com/CTMDS/pub/PUB010100.do?method=handle04&compId={row['companyId']}"
            # 发起请求
            # 每次请求都初始化一次self.count
            self.count = 1
            res = self.getBody(url=urlDetail, send_type='get', data={
   })
            if res is not None:
                # 解析html
                self.parseHtml(row=row, htmlText=res.text)
            else:
                print(f"{urlDetail}请求失败！")

    # 解析页面
    def parseHtml(self, row, htmlText):
        soup = BeautifulSoup(htmlText, 'html.parser')
        # 获取备案信息
        divList = soup.find_all('div', class_=['col-md-8'])
        divtextList = [re.sub(r'\s+', '', div.text) for div in divList]

        # 获取其他机构地址
        divListOther = soup.find_all('div', class_=['col-sm-8'])
        divtextListOther = [re.sub(r'\s+', '', div.text) for div in divListOther]
        otherOrgAdd = ','.join(divtextListOther)

        # 插入数据库
        companyId = row['companyId']
        linkTel = row['linkTel']
        recordNo = row['recordNo']
        areaName = row['areaName']
        linkMan = row['linkMan']
        address = row['address']
        compName = row['compName']
        recordStatus = row['recordStatus']
        cancelRecordTime = row.get('cancelRecordTime', '')
        compLevel = divtextList[2]
        recordTime = divtextList[6]
        sql1 = "insert INTO medical_register(company_id,area_name,record_no,comp_name,address,link_man,link_tel,record_status,comp_level,record_time,cancel_record_time,other_org_add) "
        sql2 = f"values('{companyId}','{areaName}','{recordNo}','{compName}','{address}','{linkMan}','{linkTel}','{recordStatus}','{compLevel}','{recordTime}','{cancelRecordTime}','{otherOrgAdd}')"
        sql3 = sql1 + sql2
        # 执行sql
        self.cursor.execute(sql3)
        # 提交
        self.db.commit()

        # 获取备案专业和主要研究者信息
        tbody = soup.find('tbody')
        trList = tbody.find_all('tr')
        # 对tr循环获取td
        for tr in trList:
            tdList = tr.find_all('td')
            tdTextList = [td.text for td in tdList]
            tdTextList.insert(0, companyId)
            # print(tdTextList)
            # 插入数据库
            sql4 = "insert into medical_register_sub (company_id,professional_name,principal_investigator,job_title) values(%s,%s,%s,%s)"
            self.cursor.execute(sql4, tdTextList)
            # 提交到数据库
            self.db.commit()

    # 入口函数
    def run(self):
        try:
            # 拿第一页的数据
            data = {
   'pageSize': 1350, 'curPage': 1}
            # 每次请求都初始化一次self.count
            self.count = 1
            res = self.getBody(url=self.url, send_type='post', data=data)
            if res is not None:
                # 加载为json
                jsonRes = json.loads(res.text)
                # 查看响应状态码
                status = jsonRes['success']
                # 如果状态为True
                if status == True:
                    # 获取数据
                    dataList = jsonRes['data']
                    # 处理数据
                    self.parseData(dataList=dataList)
            else:
                print(f"{self.url}请求失败")
        except Exception as e:
            print('发生错误！', e)


if __name__ == '__main__':
    spider = mySpider('https://www.百度.com/CTMDS/pub/PUB010100.do?method=handle05')
    spider.run()