废话
本文旨在记录selenium的一些操作实例，没啥技术含量，也没有太多概念的内容。
安装selenium
pip install selenium
下载驱动(chromedriver)
下载前需要确认适配自己的浏览器版本
https://chromedriver.storage.googleapis.com/index.html
selenium的基本用法
基本用法看官方文档，讲述的很清楚了，推荐
https://selenium-python.readthedocs.io/installation.html
常用的实例
设置浏览器不加载图片
from selenium import webdriver # 设置不加载图片 chrome_opt = webdriver.ChromeOptions() prefs = { "profile.managed_default_content_settings.images": 2 } chrome_opt.add_experimental_option("prefs", prefs) driver = webdriver.Chrome(chrome_options=chrome_opt) driver.get("https://www.taobao.com")
点击弹窗与点击下拉列表
from selenium import webdriver from selenium.webdriver.support.select import Select driver = webdriver.Chrome() # 点击接受弹窗 driver.switch_to.alert.accept() # 点击下拉列表 sel = driver.find_element_by_id("nr") Select(sel).select_by_index(2)
切换窗口
from selenium import webdriver driver = webdriver.Chrome() first_win = driver.current_window_handle all_win = driver.current_window_handle for win in all_win: if win != first_win: driver.switch_to.window(win)
自动下拉列表(以开源中国的博客栏目为例)
很多页面是下拉加载更多信息，我们如何模拟这个下拉操作：
from selenium import webdriver import time driver = webdriver.Chrome() driver.get('https://www.oschina.net/blog') time.sleep(5) # 实现自动下拉刷新 下拉三页 for i in range(3): driver.execute_script('window.scrollTo(0,document.body.scrollHeight); var lenOfPage=document.body.scrollHeight; return lenOfPage;') time.sleep(3)
如何模拟手机访问？
from selenium import webdriver # 模拟手机 mobilesetting = {"deviceName":"iPhone 6 Plus"} options = webdriver.ChromeOptions() options.add_experimental_option("mobileEmulation", mobilesetting) driver = webdriver.Chrome(chrome_options=options) # 设置大小 driver.set_window_size(400, 800) # driver.maximize_window() driver.get("https://www.taobao.com") # 后退 driver.back() # 前进 driver.forward() # 刷新 driver.refresh()
如何为selenium设置代理？(连接无用户名密码认证的代理)
# 设置代理 from selenium import webdriver options = webdriver.ChromeOptions() options.add_argument("--proxy-server=http://ip:port") driver = webdriver.Chrome(chrome_options=options) driver.get("http://httpbin.org/ip") print(driver.page_source)
如何为selenium设置代理？(有用户名和密码的连接)
推荐几篇文章：
scrapy + selenium 模拟登录csdn
其实，没啥技术含量。只是简单运用，敲一遍加深印象。
关于selenium的基础用法强烈建议用的时候看下文档就好，技术含量不高，无需过多费心。
spider.py
# -*- coding: utf-8 -*- import scrapy from selenium import webdriver class CsdnSpider(scrapy.Spider): name = 'csdn' allowed_domains = ['csdn.net'] start_urls = ['https://passport.csdn.net/account/login','https://i.csdn.net/#/account/index'] def __init__(self): # mobilsetting = {"deviceName":"iPhone 6 Plus"} # options = webdriver.ChromeOptions() # options.add_experimental_option("mobileEmulation", mobilsetting) self.browser = None self.cookies = None # self.browser.set_window_size(400,800) super(CsdnSpider, self).__init__() def spider_closed(self, response): print("spider close") self.brower.close() def parse(self, response): print(response.url) print(response.body.decode("utf-8","ignore"))
middlewares.py
from scrapy import signals from selenium import webdriver from scrapy.http import HtmlResponse import time import requests class LoginMiddleware(object): def process_request(self, request, spider): if spider.name == "csdn": if request.url.find("login") != -1: spider.browser = webdriver.Chrome() spider.browser.get(request.url) switch = spider.browser.find_element_by_xpath('//a[@class="login-code__open js_login_trigger login-user__active"]') if switch.text == '账号登录': switch.click() time.sleep(3) username = spider.browser.find_element_by_id('username') password = spider.browser.find_element_by_id('password') time.sleep(2) username.send_keys("") time.sleep(1) password.send_keys("") time.sleep(2) click = spider.browser.find_element_by_class_name("logging") time.sleep(2) click.click() time.sleep(8) spider.cookies = spider.browser.get_cookies() return HtmlResponse( url=spider.browser.current_url, body=spider.browser.page_source, encoding="utf-8" ) else: req = requests.session() for cookie in spider.cookies: req.cookies.set(cookie['name'], cookie['value']) req.headers.clear() newpage = req.get(request.url) print(request.url) print(newpage.text) return HtmlResponse( url=request.url, body=newpage.text, encoding="utf-8" )
