什么你还不知道招聘信息，小唐来教你——最新2021爬取拉勾网招聘信息（一）

import re
import xlwt
import requests
from lxml import etree
from xlrd import open_workbook
from xlutils.copy import copy

import re
import xlwt
import requests
from lxml import etree
from xlrd import open_workbook
from xlutils.copy import copy
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36'}
# url_q="https://www.lagou.com/wn/jobs?px=new&gj=%E5%9C%A8%E6%A0%A1%2F%E5%BA%94%E5%B1%8A&pn="
# url_h="&kd=Java&city=%E5%85%A8%E5%9B%BD"
# url_q="https://www.lagou.com/wn/jobs?px=new&gj=%E5%9C%A8%E6%A0%A1%2F%E5%BA%94%E5%B1%8A&pn="
# url_h="&kd=Python&city=%E5%85%A8%E5%9B%BD"
# url_q="https://www.lagou.com/wn/jobs?px=new&gj=%E5%9C%A8%E6%A0%A1%2F%E5%BA%94%E5%B1%8A&pn="
# url_h="&kd=C%2B%2B&city=%E5%85%A8%E5%9B%BD"
# url_q="https://www.lagou.com/wn/jobs?px=new&gj=%E5%9C%A8%E6%A0%A1%2F%E5%BA%94%E5%B1%8A&pn="
# url_h="&kd=html5&city=%E5%85%A8%E5%9B%BD"
url_q="https://www.lagou.com/wn/jobs?px=new&gj=%E5%9C%A8%E6%A0%A1%2F%E5%BA%94%E5%B1%8A&pn="
url_h="&kd=Python&city=%E5%85%A8%E5%9B%BD"
wb_name='Python.xls'
def spider(base_url,headers):
    response = requests.get(base_url, headers)#构建服务器
    response.encoding = response.apparent_encoding#获取编码格式
    response_text = response.text   # txt化页面内容
    selector = etree.HTML(response_text)#html化
    try:
        job_list=selector.xpath("//div[@class='list__YibNq']/div")
        print(job_list)
        try:
            for job in job_list:
                #岗位
                job_name=job.xpath("//div[@class='p-top__1F7CL']/a/text()[1]")
                # print(job_name)
                #位置
                job_add=job.xpath("//div[@class='p-top__1F7CL']/a/text()[2]")
                # print(job_add)
                #公司名
                job_cp=job.xpath("//div[@class='company-name__2-SjF']/a/text()")
                # print(job_cp)
                #公司现状
                job_cpinfo=job.xpath("//div[@class='industry__1HBkr']/text()")
                # print(job_cpinfo)
                #钱
                job_money=job.xpath("//div[@class='p-bom__JlNur']/span/text()")
                # print(job_money)
                #学历
                job_xl=job.xpath("//div[@class='p-bom__JlNur']/text()")
                # print(job_xl)
                #描述
                job_mess=job.xpath("//div[@class='il__3lk85']/text()")
                # print(job_mess)
                # 需求
                job_ir= job.xpath("//div[@class='ir___QwEG']/span/text()")
            print("解析成功——————开始存储数据")
            save(job_name,job_add,job_cp,job_cpinfo,job_money,job_xl,job_mess,job_ir)
        except BaseException as e:
            print(e)
            print(e.__traceback__.tb_lineno)
            print("本条解析失败，跳过")
    except BaseException as e:
        print(e)
        print(e.__traceback__.tb_lineno)
        print("本页爬取失败，跳过")
def save(name,add,cp,cpinfo,money,xl,mess,ir):
    try:
        r_xls = open_workbook(wb_name)  # 读取excel文件
    except BaseException as e:
        print(e)
        print(e.__traceback__.tb_lineno)
        #没有读取到我们就创建
        file = xlwt.Workbook()
        sheet1 = file.add_sheet(u'表1', cell_overwrite_ok=True)
        sheet1.write(0, 0, "岗位")
        sheet1.write(0, 1, "位置")
        sheet1.write(0, 2, "公司名")
        sheet1.write(0, 3, "公司现状")
        sheet1.write(0, 4, "工资")
        sheet1.write(0, 5, "学历")
        sheet1.write(0, 6, "描述")
        sheet1.write(0, 7, "需求")
        file.save(wb_name)
    try:
        r_xls = open_workbook(wb_name)
        row = r_xls.sheets()[0].nrows  # 获取已有的行数
        excel = copy(r_xls)  # 将xlrd的对象转化为xlwt的对象
        worksheet = excel.get_sheet(0)  # 获取要操作的sheet
        for i in range(0, len(name)):
            worksheet.write(row + i, 0, name[i])
            worksheet.write(row + i, 1, add[i])
            worksheet.write(row + i, 2, cp[i])
            worksheet.write(row + i, 3, cpinfo[i])
            worksheet.write(row + i, 4, money[i])
            worksheet.write(row + i, 5, xl[i])
            worksheet.write(row + i, 6, mess[i])
            worksheet.write(row + i, 7, ir[i])
        excel.save(wb_name)
        print("存储成功")
    except BaseException as e:
        print(e)
        print(e.__traceback__.tb_lineno)
        print("存储失败")
def run():
    for page in range(1,31):
        print("正在解析"+str(page)+"页")
        url=url_q+str(page)+url_h
        print(url)
        spider(url, headers)
run()