想成为优秀的斗鱼主播，首先得掌握优秀的自拍技能；这次写个有意思的, 爬取斗鱼小姐姐的自拍头像...

效果图:

001

002

003

004

005

分析频道

频道API

获取关键参数

分析参数

查看Json

请求API, 爬虫负责翻页,https://www.douyu.com/gapi/rkc/directory/2_201/1

脚本运行界面

脚本运行

源码():

4月13日10时更新: 可按照主播人气, 对图片进行排序, 并实现了图片去重

import requests
from lxml import etree
import json
import os
import time

def getResponse(url):
    headers = {
        # 设置用户代理头(为狼披上羊皮)
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
    }
    response = requests.get(url, headers = headers)
    return response

def getAllChannelMark(response):
    data_etree = etree.HTML(response.content)
    title_list = data_etree.xpath('//div[@class="leftnav-cate"]//li/a')
    title_mark_list = []
    for title in title_list:
        title_name = title.xpath('@title')
        title_mark = title.xpath('@data-rk')
        if title_name and title_mark:
            tmp_title = {"title_name": title_name, "title_mark": title_mark}
            title_mark_list.append(tmp_title)

    return title_mark_list

def getChanneTitleMark(title_mark_list):
    for index, title_mark in enumerate(title_mark_list):
        print("编号:",index,"=>",title_mark["title_name"], end="")
        if index%4 == 0:
            print()

    checkNumPass = True
    while checkNumPass:
        try:
            channelNum = int(input("请输入主题对应的编号(例如: 33):"))
            checkNumPass = False
        except:
            print("输入的编号格式有误")

    ChanneTitleMark = title_mark_list[channelNum]["title_mark"]
    return ChanneTitleMark

def checkNumFormat(message):
    canPass = False
    num = 0
    while not canPass:
        try:
            num = int(input(message))
            canPass = True
        except:
            print("输入的格式有误请重新输入!")
    return num


def getSourceJson(ChanneTitleMark):
    num = checkNumFormat("请输入需要爬取的主播图片数量(例如: 200):")
    # 用于生产url的变量
    url_index = 0
    # 设置去重列表
    name_list = []
    while num > 0:
        JsonUrl = "https://www.douyu.com/gapi/rkc/directory/"+str(ChanneTitleMark[0])+"/" + str(url_index)
        SourceJson = getResponse(JsonUrl).content
        # 获取多个主播的信息
        anchors = json.loads(SourceJson)["data"]["rl"]

        # # 计算本轮获取的主播数量
        # anchor_num = len(anchors)
        # # 计算出待获取的图片数量
        # last_num = num
        # num = num - anchor_num
        # # 如果本次信息过量,则截取部分json信息
        # if num <= 0:
        #     anchors = anchors[0:last_num]
        groupAnchorInfoList = []
        for anchor in anchors:
            tmp_anchor_info = {}
            # 主播照片
            tmp_anchor_info["anchor_img"] = anchor["rs1"]
            # 主播名
            tmp_anchor_info["anchor_name"] = anchor["nn"]
            # 直播房间id
            tmp_anchor_info["anchor_rid"] = anchor["rid"]
            # 主题
            tmp_anchor_info["anchor_rn"] = anchor["rn"]
            # 即时热度(人气)
            tmp_anchor_info["anchor_ol"] = str(anchor["ol"])
            # 将人气补齐到百万级别
            if len(str(anchor["ol"])) < 7:
                ol_tmp = "0000000" + str(anchor["ol"])
                tmp_anchor_info["anchor_ol"] = ol_tmp[-7:]

            # 频道名
            tmp_anchor_info["channelName"] = anchor["c2name"]

            # 如果已经存在此主播图片, 则不添加
            if tmp_anchor_info["anchor_name"] not in name_list:

                groupAnchorInfoList.append(tmp_anchor_info)
                name_list.append(tmp_anchor_info["anchor_name"])

        # 获取一页, 保存一次
        url_index += 1

        num = saveImage(groupAnchorInfoList, num)

def saveImage(groupAnchorInfoList, num):
    # 延迟0.2秒
    time.sleep(0.2)
    for AnchorInfo in groupAnchorInfoList:
        if num > 0:
            # 建立文件夹
            try:
                os.makedirs("./images/%s"%(AnchorInfo["channelName"]))
            except Exception as e:
                pass

            # 写入图片
            file_path = "./images/%s/%s"%(AnchorInfo["channelName"], AnchorInfo["anchor_ol"]+"_"+AnchorInfo["anchor_name"]+"_"+AnchorInfo["anchor_rn"]+".jpg")
            file_data = getResponse(AnchorInfo["anchor_img"]).content

            try:
                with open(file_path, "wb+") as f:

                    f.write(file_data)
                    print(">",file_path, "下载成功", "剩余", num, "张")
            except Exception as e:
                pass
        num = num - 1
    return num

def main():
    response = getResponse("https://www.douyu.com/directory/all")
    title_mark_list = getAllChannelMark(response)
    ChanneTitleMark = getChanneTitleMark(title_mark_list)
    getSourceJson(ChanneTitleMark)



if __name__ == '__main__':
    main()

由于分析获取了API, 所以爬虫效率很高, 斗鱼的"颜值"(第33个)频道大概有940个主播, 耗时1分钟全部爬完...