爬取熊猫TV,javascript,selenium,模拟点击-阿里云开发者社区

开发者社区> 开发与运维> 正文

爬取熊猫TV,javascript,selenium,模拟点击

简介:
from selenium import webdriver
import csv


def get_pages_numger(browser):
    res = browser.find_elements_by_xpath('//div[@class="page-component"]/a[7]')
    return int(res.text)
    
def get_next_page_buttun(browser):
    button = browser.find_elements_by_xpath()
    return button

def get_rooms_number_in_a_page(browser):
    res = []
    for li in browser.find_elements_by_xpath('//li[@data-id]'):
        id = li.get_attribute("data-id")
        nickname = li.find_element_by_xpath('//span[@class="video-nickname"]')
        number = li.find_element_by_xpath('//span[@class="video-number"]') # 观众数
        cate = li.find_element_by_xpath('//span[@class="video-cate"]')
        res.append([id, nickname, cate, number])
    return res

def get_rooms_number_in_all_pages(browser, pages_number):
    res = [] 
    for i in range(pages_number):
        print('第{}页'.format(i+1))
        # 抓取
        res.extend(get_rooms_number_in_a_page(browser))
        
        # 点击进入下一页
        next_page_button = get_next_page_buttun(browser)
        next_page_button.click()
    
    browser.close()
    
    return res
    
    
def save_to_csv(rooms_number):
    with open('live_rooms_number.csv', 'w') as f:
        writer = csv.writer(f, lineterminator='\n')
        writer.writerow(['id','nickname','cate', 'number']) # 表头
        writer.writerows(rooms_number)
        
            
def read_from_csv():
    with open('live_rooms_number.csv', 'r') as f:
        reader = csv.reader(f)
        your_list = list(reader)[1:] # 去掉表头
    #print(your_list)
    return your_list
        

    
    
def get_rooms_number():
    browser = webdriver.Firefox()

    browser.get('http://www.panda.tv/all')
    assert '熊猫TV' in browser.title
    
    pages_number = get_pages_numger(browser)
    
    all_live_rooms_number = get_rooms_number_in_all_pages(browser, pages_number)
    
    return all_live_rooms_number
    
    
    
if __name__ == '__main__':
    
    rooms_number = get_rooms_number()
    save_to_csv(rooms_number)
    
    #rooms_number = read_from_csv()
    
    
    
本文转自罗兵博客园博客,原文链接:http://www.cnblogs.com/hhh5460/p/5708802.html,如需转载请自行联系原作者

版权声明:本文首发在云栖社区,遵循云栖社区版权声明:本文内容由互联网用户自发贡献,版权归用户作者所有,云栖社区不为本文内容承担相关法律责任。云栖社区已升级为阿里云开发者社区。如果您发现本文中有涉嫌抄袭的内容,欢迎发送邮件至:developer2020@service.aliyun.com 进行举报,并提供相关证据,一经查实,阿里云开发者社区将协助删除涉嫌侵权内容。

分享:
开发与运维
使用钉钉扫一扫加入圈子
+ 订阅

集结各类场景实战经验,助你开发运维畅行无忧

其他文章