汽车之家评论

import csv
import os
import requests
from lxml import etree
import re
class Spider():
    def __init__(self):
        self.headers = {
            "authority": "www.autohome.com.cn",
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
            "cache-control": "no-cache",
            "pragma": "no-cache",
            "referer": "https://www.autohome.com.cn/beijing/",
            "sec-ch-ua": "\"Chromium\";v=\"122\", \"Not(A:Brand\";v=\"24\", \"Microsoft Edge\";v=\"122\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Windows\"",
            "sec-fetch-dest": "document",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "same-origin",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0"
        }
        self.cookies = {
            "fvlid": "1708592005308QmeCfkEcQf",
            "__ah_uuid_ng": "",
            "sessionuid": "be9bf153-8fda-41f7-99e3-9fbc5172d455",
            "ASP.NET_SessionId": "bmfejflbhqwqxmdwkf5jfuya",
            "ahsids": "5714_5998",
            "historybbsName4": "c-5998%7C%E5%A5%A5%E8%BF%AAA7L%2Cc-5714%7C%E5%AE%8F%E5%85%89MINIEV",
            "historyseries": "5714%2C5998",
            "ahpvno": "21",
            "pvidchain": "2112108,6830286,6861598,2042204,2042204,101075,6830286,6861598,6861421,3454440",
            "ahrlid": "1716956158466JPRl4Pm0jp-1716956304138"
        }
        self.price = ''
        self.count = 0
    def spider_list(self):
        proce_list = ['_0_5','_5_8','_8_15','_15_20','_20_30','_30_50','_50_100','_100_9000']
        proce_list = ['_30_9000']
        for self.price in proce_list:
            url = f"https://www.autohome.com.cn/price/ev/price{self.price}"
            response = requests.get(url, headers=self.headers, cookies=self.cookies).text
            home_html = etree.HTML(response)
            links = home_html.xpath("//li[@class='tw-group tw-relative tw-cursor-pointer tw-overflow-hidden tw-rounded tw-bg-[#F7FAFE] tw-pb-4 tw-text-center tw-text-[#111E36] hover:tw-shadow-[0_8px_32px_0_rgba(17,30,54,0.1)]']/div[@class='tw-mt-1 tw-px-4']/a/@href")[:5]
            for index , link in enumerate(links):
                # 'https://www.autohome.com.cn/5714/#pvareaid=6861421'
                match = re.search(r'www.autohome.com.cn/(\d+)/#pvareaid', link)
                if match:
                    seriesId = match.group(1)  # group(1)捕获第一个括号内的内容
                    pageIndex = 1
                    self.spider_subdata(seriesId , pageIndex , index)
    def spider_subdata(self,seriesId,pageIndex,index):
        url = "https://koubeiipv6.app.autohome.com.cn/pc/series/list"
        params = {
            "pm": "3",
            "seriesId": f"{seriesId}",
            "pageIndex": f"{pageIndex}",
            "pageSize": "20",
            "yearid": "0",
            "ge": "0",
            "seriesSummaryKey": "0",
            "order": "0"
        }
        response = requests.get(url, headers=self.headers, cookies=self.cookies, params=params).json()
        print(response['result']['list'])
        comment_list = response['result']['list']
        for comments in comment_list:
            showId = comments['showId']
            self.spider_detail(showId)
        if response['result']['list'] != []:
            print(f'车型{index+1} : 第{pageIndex}页爬取完毕')
            pageIndex += 1
            self.spider_subdata(seriesId,pageIndex,index)
    def spider_detail(self ,showId):
        url = f"https://k.autohome.com.cn/detail/view_{showId}.html"
        response = requests.get(url, headers=self.headers, cookies=self.cookies).text
        html = etree.HTML(response)
        data = html.xpath("//div[@class='space kb-item']/p/text()")
        # 满意
        satisfied = ",".join(html.xpath("//div[@class='satisfied kb-item']/p/text()")).replace('\n','').strip()
        # 不满意
        unsatis = ",".join(html.xpath("//div[@class='unsatis kb-item']/p/text()")).replace('\n','').strip()
        space = self.check_comment(data , 0)# 空间
        feel = self.check_comment(data , 1)# 驾驶感受
        endurance = self.check_comment(data , 2)# 续航
        appearance = self.check_comment(data , 3)# 外观
        trim = self.check_comment(data , 4)# 内饰
        costPerformance = self.check_comment(data , 5)# 性价比
        intelligentize = self.check_comment(data , 6)# 智能化
        data_list = [satisfied,unsatis,space,feel,endurance,appearance,trim,costPerformance,intelligentize]
        if len(data) == 7:
            self.count += 1
            print(f"爬取数量{self.count} , 数据 : {data_list}")
            self.save_data_to_csv(data_list)
    def check_comment(self , data , count):
        try:
            result = data[count].replace('\n','')
        except:
            result = ''
        return result
    def save_data_to_csv(self,data_list):
        filename = f'{self.price}.csv'
        name_headers = [ '最满意', '最不满意', '空间', '驾驶感受', '续航', ' 外观', '内饰', '性价比', '智能化' ]
        if not os.path.isfile(filename):
            with open(f'{filename}', 'a', encoding='utf-8-sig', newline='')as f:
                csv_write = csv.DictWriter(f, fieldnames=name_headers)
                csv_write.writeheader()
        else:
            with open(f'{filename}', 'a', encoding='utf-8', newline='')as f:
                csv_write = csv.DictWriter(f, fieldnames=data_list)
                csv_write.writeheader()
if __name__ == '__main__':
    spider = Spider()
    spider.spider_list()