# Scrapy 内置的 Downloader Middleware 为 Scrapy 供了基础的功能， # 定义一个类，其中(object)可以不写，效果一样 classSimpleProxyMiddleware(object): 
# 声明一个数组          proxyList= ['你购买的HTTP代理地址'] 
# Downloader Middleware的核心方法，只有实现了其中一个或多个方法才算自定义了一个 Downloader Middleware def process_request(self, request, spider): # 随机从其中选择一个，并去除左右两边空格 proxy=random.choice(self.proxyList).strip() 
# 打印结果出来观察 print("this is request ip:"+proxy) 
# 设置request的proxy属性的内容为代理ip request.meta['proxy'] =proxy# Downloader Middleware的核心方法，只有实现了其中一个或多个方法才算自定义了一个Downloader Middleware def process_response(self, request, response, spider): # 请求失败不等于200 ifresponse.status!=200: 
# 重新选择一个代理ip proxy=random.choice(self.proxyList).strip() 
print("this is response ip:"+proxy) 
# 设置新的代理ip内容 request.mete['proxy'] =proxyreturnrequestreturnresponse

#DDWNLQADER_MIDDLEWARES = {#        *images.middlewares.ImagesDownloaderMiddleware": 543，#    *images middlewares，LocalProxyMiddleware*: 100#}

#DDWNLQADER_MIDDLEWARES = {'scrapydownloadertest.middlewares.SimpleProxyMiddleware': 100,
|}

# -*- coding: utf-8 -*-importscrapyclassHttpproxyipSpider(scrapy.Spider):
name='httpProxyIp'allowed_domains= ['icanhazip.com']
start_urls= ['http://icanhazip.com/']
defparse(self, response):
pass

# -*- coding: utf-8 -*-importscrapyfromscrapy.cmdlineimportexecuteclassHttpproxyipSpider(scrapy.Spider):
# spider 任务名name='httpProxyIp'# 允许访问的域名allowed_domains= ['icanhazip.com']
# 起始爬取的urlstart_urls= ['http://icanhazip.com/']
# spider 爬虫解析的方法，关于内容的解析都在这里完成; self表示实例的引用，response爬虫的结果defparse(self, response):
print('代理后的ip: ', response.text)
# 这个是main函数也是整个程序入口的惯用写法if__name__=='__main__':
execute(['scrapy', 'crawl', 'httpbin'])