说明：
本文参考了官网文档，以及stackoverflow的几个问题
注意：
下面这个爬虫不能实际运行！我只是用它来展示登录，以及之后如何处理。
方式一：FormRequest
import scrapy
from myprojct.items import ExampleItem
class ExampleSpider(scrapy.Spider):
name = 'example'
allowed_domains = ["example.com"]
start_urls = [
'http://www.example.com/articals/',
'http://www.example.com/blogs/',
'http://www.example.com/news/',
]
# 先登录
def start_requests(self):
return [scrapy.FormRequest("http://www.example.com/login",
formdata={'user': 'john', 'pass': 'secret'},
callback=self.login_check)]
# 检查登录。
def login_check(self, response):
if "Login failed" not in response.body: # 如果登录成功，则
for url in self.start_urls:
yield scrapy.Request(url, callback=self.parse_page)
# 解析页面
def parse_page(self, response):
for tablerow in response.css('table.basictable tr'):
item = ExampleItem()
item["name"] = tablerow.xpath('td[1]').extract()
item["handicap"] = tablerow.xpath('td[2]').extract()
item["exact"] = tablerow.xpath('td[3]').extract()
item["category"] = tablerow.xpath('td[4]').extract()
yield item
方式二：FormRequest.from_response
import scrapy
class LoginSpider(scrapy.Spider):
name = 'example.com'
start_urls = ['http://www.example.com/users/login.php']
def parse(self, response):
return scrapy.FormRequest.from_response(
response,
formdata={'username': 'john', 'password': 'secret'},
callback=self.after_login
)
def after_login(self, response):
# check login succeed before going on
if "authentication failed" in response.body:
self.logger.error("Login failed")
return
# continue scraping with authenticated session...
