DOWNLOADER_MIDDLEWARES= {
'dbtop.middlewares.DbtopDownloaderMiddleware': 543,
    }
ITEM_PIPELINES= {
'dbtop.pipelines.DbtopPipeline': 300,
    }
USER_AGENT='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36 Edg/105.0.1643.42'关闭注释

importscrapyclassScrapyDbtop250Item(scrapy.Item):
# 定义以下变量# define the fields for your item here like:# name = scrapy.Field()# 排名ord=scrapy.Field()
# 电影名称name=scrapy.Field()
# 导演director=scrapy.Field()
# 编剧scriptwriters=scrapy.Field()
# 主演star=scrapy.Field()
# 类型type=scrapy.Field()
# 地区region=scrapy.Field()
# 语言language=scrapy.Field()
# 上映时间releaseDate=scrapy.Field()
# 片长length=scrapy.Field()
# 简介synopsis=scrapy.Field()
# 图片地址img=scrapy.Field()

importscrapyfromscrapy.linkextractorsimportLinkExtractorfromscrapy.spidersimportCrawlSpider, Rulefromscrapy_dbtop250_02.itemsimportScrapyDbtop25002ItemclassDbSpider(CrawlSpider):
name='db'allowed_domains= ['movie.douban.com']
start_urls= ['https://movie.douban.com/top250?start=0&filter=']
rules= (
Rule(LinkExtractor(allow=r'\?start=\d+&filter='), callback='parse_item',
follow=True),
    )
defparse_item(self, response):
videos_a=response.xpath('//div[@class="hd"]/a/@href')
forv_ainvideos_a:
url=v_a.extract()
# 对链接发起访问yieldscrapy.Request(url=str(url), callback=self.parse_second)
# 去除空格与\ndefdislodge_blank(self, src):
price= [sy.strip() forsyinsrcifsy.strip() !='']
returnpricedefparse_second(self, response):
# 排名ord=response.xpath('//div[@id="content"]/div[@class="top250"]/span[1]/text()').extract()[0]
# # 电影名称name=response.xpath('//div[@id="content"]/h1/span')[0].xpath('./text()').extract()[0]
iflen(name) ==0:
name='无电影名'# 导演director=response.xpath('//div[@id="info"]/span')[0].xpath('./span[@class="attrs"]/a/text()').extract()[0]
iflen(director) ==0:
director='无导演'# 编剧try:
response.xpath('//div[@id="info"]/span')[1].xpath('./span[@class="attrs"]/a/text()').extract()[0]
exceptException:
scriptwriters='无编剧信息'else:
scriptwriters= \
response.xpath('//div[@id="info"]/span')[1].xpath('./span[@class="attrs"]/a/text()').extract()[0]
# 主演star='/'.join(response.xpath('//span[@class="actor"]//a/text()').extract())
iflen(star) ==0:
star='无主演'# 类型type='/'.join(response.xpath('//div[@id="info"]/span[@property="v:genre"]/text()').extract())
# 地区region=response.xpath('//span[contains(./text(), "制片国家/地区:")]/following::text()[1]').extract()[0]
iflen(region) ==0:
region='无地区'# 语言language=response.xpath('//span[contains(./text(), "语言:")]/following::text()[1]').extract()[0]
iflen(language) ==0:
language='无语言'# 上映时间releaseDate=self.dislodge_blank(
src=response.xpath('//div[@id="info"]//span[@property="v:initialReleaseDate"]/@content').extract())[0]
# 片上length=self.dislodge_blank(
src=response.xpath('//div[@id="info"]//span[@property="v:runtime"]/@content').extract())[0] +"分钟"# 简介synopsis=''.join(response.xpath('//div[@id="link-report"]/span/text()').extract()).replace(u'\u3000',
u'').replace('\n',
'').replace(
'\r', '').replace(" ", "").replace('"', "'")
# 图片地址img=response.xpath('//div[@id="mainpic"]/a/img/@src').extract()[0]
data=ScrapyDbtop25002Item(ord=ord, name=name, director=director, scriptwriters=scriptwriters, star=star,
type=type, region=region, language=language, releaseDate=releaseDate, length=length,
synopsis=synopsis, img=img)
yielddata

fromitemadapterimportItemAdapterclassScrapyDbtop25002Pipeline:
# 加上这两个方法是为了避免文件重复的打开# 在爬虫文件开始的之前就执行的方法defopen_spider(self, spider):
self.fd=open('book.json', 'w', encoding='utf-8')
# 将item就是yield后面的book对象defprocess_item(self, item, spider):
self.fd.write(str(item))
returnitem# 在爬虫文件执行完成后执行的方法defclose_spider(self, spider):
self.fd.close()
# 加载settings文件fromscrapy.utils.projectimportget_project_settingsimportpymysqlclassMysqlPipelines:
# 开启链接defopen_spider(self, spider):
settings=get_project_settings()
self.host=settings['DB_HOST']
self.user=settings['DB_USER']
self.port=settings['DB_PORT']
self.passwrod=settings['DB_PASSWROD']
self.name=settings['DB_NAME']
self.charset=settings['DB_CHARSET']
self.connect()
defconnect(self):
self.conn=pymysql.connect(
host=self.host,
port=self.port,
user=self.user,
password=self.passwrod,
db=self.name,
charset=self.charset        )
self.cursor=self.conn.cursor()
# 执行sql语句defprocess_item(self, item, spider):
# sql语句sql='insert into dbtop250(ord,name,director,scriptwriters,star,type,region,language,releaseDate,length,synopsis,img) ' \
'values ("{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}");'.format(
item["ord"], item["name"], item["director"], item["scriptwriters"], item["star"], item["type"],
item["region"], item["language"], item["releaseDate"], item["length"], item["synopsis"], item["img"])
print(sql)
# 执行sql语句self.cursor.execute(sql)
# 提交self.conn.commit()
# 关闭连接defclose_spider(self, spider):
self.cursor.close()
self.conn.close()
importosimportcsv# 导出csvclassExcelPipelines:
def__init__(self):
# csv文件的位置,无需事先创建store_file=os.path.dirname(__file__) +'/spiders/csdn.csv'# 打开(创建)文件self.file=open(store_file, 'w')
# csv写法self.writer=csv.writer(self.file, dialect="excel")
self.writer.writerow(
            ["排名", "电影名称", "导演", "编剧", "主演", "类型", "地区", "语言", "上映时间", "片长", "简介",
"图片地址"])
defprocess_item(self, item, spider):
self.writer=csv.writer(self.file, dialect="excel")
line= [item['ord'].encode('utf8', 'ignore'), item['name'].encode('utf8', 'ignore'),
item['director'].encode('utf8', 'ignore'), item['scriptwriters'].encode('utf8', 'ignore'),
item['star'].encode('utf8', 'ignore'), item['type'].encode('utf8', 'ignore'),
item['region'].encode('utf8', 'ignore'),
item['language'].encode('utf8', 'ignore'),
item['releaseDate'].encode('utf8', 'ignore'), item['length'].encode('utf8', 'ignore'),
item['synopsis'].encode('utf8', 'ignore'), item['img'].encode('utf8', 'ignore')]
self.writer.writerow(line)
returnitemdefspider_closed(self, spider):
self.file.close()

ITEM_PIPELINES= {
'scrapy_dbtop.pipelines.DbtopItem': 300,
# 数据库'scrapy_dbtop.pipelines.MysqlPipelines': 301,
# csv'scrapy_dbtop.pipelines.ExcelPipelines': 302,
}
DB_HOST='127.0.0.1'DB_PORT=端口号DB_USER='mysql账号'DB_PASSWROD='mysql密码'DB_NAME='reptilebank'DB_CHARSET='UTF8'