Scrapy入门到放弃03：理解settings配置，监控scrapy引擎|8月更文挑战

# Scrapy settings for ScrapyDemo project
# 自动生成的配置，无需关注，不用修改
BOT_NAME = 'ScrapyDemo'
SPIDER_MODULES = ['ScrapyDemo.spiders']
NEWSPIDER_MODULE = 'ScrapyDemo.spiders'
# 设置UA，但不常用，一般都是在MiddleWare中添加
USER_AGENT = 'ScrapyDemo (+http://www.yourdomain.com)'
# 遵循robots.txt中的爬虫规则，很多人喜欢False，当然我也喜欢....
ROBOTSTXT_OBEY = True
# 对网站并发请求总数，默认16
CONCURRENT_REQUESTS = 32
# 相同网站两个请求之间的间隔时间，默认是0s。相当于time.sleep()
DOWNLOAD_DELAY = 3
# 下面两个配置二选一，但其值不能大于CONCURRENT_REQUESTS，默认启用PER_DOMAIN
# 对网站每个域名的最大并发请求，默认8
CONCURRENT_REQUESTS_PER_DOMAIN = 16
# 默认0，对网站每个IP的最大并发请求，会覆盖上面PER_DOMAIN配置，
# 同时DOWNLOAD_DELAY也成了相同IP两个请求间的间隔了
CONCURRENT_REQUESTS_PER_IP = 16
# 禁用cookie，默认是True，启用
COOKIES_ENABLED = False
# 请求头设置，这里基本上不用
DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}
# 配置启用Spider MiddleWares，Key是class，Value是优先级
SPIDER_MIDDLEWARES = {
    'ScrapyDemo.middlewares.ScrapydemoSpiderMiddleware': 543,
}
# 配置启用Downloader MiddleWares
DOWNLOADER_MIDDLEWARES = {
    'ScrapyDemo.middlewares.ScrapydemoDownloaderMiddleware': 543,
}
# 配置并启用扩展，主要是一些状态监控
EXTENSIONS = {
    'scrapy.extensions.telnet.TelnetConsole': None,
}
# 配置启用Pipeline用来持久化数据
ITEM_PIPELINES = {
   'ScrapyDemo.pipelines.ScrapydemoPipeline': 300,
}
# 禁止控制台使用telnet连接scrapy获取状态，默认是启用。我们使用默认值即可
TELNETCONSOLE_ENABLED = False
# Telnet终端使用的端口范围。默认: [6023, 6073],如果设置为 None 或 0 ， 则使用动态分配的端口
# TELNETCONSOLE_PORT
# telnet账号，默认：scrapy
TELNETCONSOLE_USERNAME = None
# telnet密码：默认会自动生成一个密码
TELNETCONSOLE_PASSWORD
# Telnet终端监听的接口(interface)。默认: '127.0.0.1'
TELNETCONSOLE_HOST = '127.0.0.1'
# AutoThrottle是限速节流算法
# 让爬虫程序自适应download_delay和concurrent并发
AUTOTHROTTLE_ENABLED = True
# 爬虫程序启动时，开始对网站发起请求的延迟
AUTOTHROTTLE_START_DELAY = 5
# 请求到响应的最大允许的延迟时间，必须大于download_delay
AUTOTHROTTLE_MAX_DELAY = 60
# 并行发送到每个远程服务器的平均请求数，小于CONCURRENT_REQUESTS_PER_DOMAIN和CONCURRENT_REQUESTS_PER_IP
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# 为每个响应启用显示限制统计信息
AUTOTHROTTLE_DEBUG = False
# HttpCache主要是将每次的请求和响应缓存到本地，可以离线进行处理
# 配置启用HTTP Cache，默认不启用
HTTPCACHE_ENABLED = True
# 缓存的过期时间，0为永不过期
HTTPCACHE_EXPIRATION_SECS = 0
# 缓存目录名称
HTTPCACHE_DIR = 'httpcache'
# 设置不需要缓存的状态码请求
HTTPCACHE_IGNORE_HTTP_CODES = []
# 此类将缓存保存到本地文件系统，还可以使用其他类保存到数据库
HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

custom_settings = {
        'DOWNLOAD_DELAY': 10
    }

scrapy crawl [spiderName] -s DOWNLOAD_DELAY=10

import scrapy
class DouLuoDaLuSpider(scrapy.Spider):
    name = 'DouLuoDaLu'
    allowed_domains = ['v.qq.com']
    start_urls = ['https://v.qq.com/detail/m/m441e3rjq9kwpsc.html']
    # 自定义配置
    custom_settings = {
        'DOWNLOAD_DELAY': 10
    }
    def parse(self, response):
        # 获取配置
        delay = self.crawler.settings.get('DOWNLOAD_DELAY')
        print(delay)

scrapy crawl DouLuoDaLu -s DOWNLOAD_DELAY=11

import scrapy
from scrapy import Request
class DouLuoDaLuSpider(scrapy.Spider):
    name = 'DouLuoDaLu'
    allowed_domains = ['v.qq.com']
    start_urls = ['https://v.qq.com/detail/m/m441e3rjq9kwpsc.html']
    custom_settings = {
        'DOWNLOAD_DELAY': 6,
        'CONCURRENT_REQUESTS': 32,
        'CONCURRENT_REQUESTS_PER_DOMAIN': 8,
        # 账号默认就是scrapy
        # 'TELNETCONSOLE_USERNAME' = 'scrapy'
        'TELNETCONSOLE_PASSWORD': 'scrapy'
    }
    def start_requests(self):
        for i in range(0, 100):
            yield Request(url=self.start_urls[0], callback=self.parse, dont_filter=True)
    def parse(self, response):
        for i in range(0, 10):
            yield Request(url=self.start_urls[0], callback=self.parse, dont_filter=True)

# telnet终端监听端口配置为*TELNETCONSOLE_PORT*，默认为6023。
telnet localhost 6023

engine.pause()
engine.unpause()
engine.stop()