一，项目简介
基于Python实现豆瓣电影数据的抓去，并存入本在数据库。
数据库结构准备：
create table if not exists `categories` ( `id` int(11) NOT NULL PRIMARY KEY, `type` varchar (255) NOT NULL DEFAULT '' ) ENGINE=InnoDB DEFAULT CHARSET=utf8; create table if not exists `movies`( `id` int(11) NOT NULL PRIMARY KEY AUTO_INCREMENT, `cover` varchar (255) NOT NULL DEFAULT '', `title` varchar (50) NOT NULL DEFAULT '', `date` varchar (10) NOT NULL DEFAULT '', `rate` float DEFAULT 0, `director` varchar (100) NOT NULL DEFAULT '', `scriptwriter` varchar(100) NOT NULL DEFAULT '', `actors` text, `district` varchar(255) DEFAULT '', `language` varchar (255) DEFAULT '', `duration` varchar (100) DEFAULT '', `abs` text, UNIQUE (`title`) )ENGINE=InnoDB DEFAULT CHARSET=utf8; create table if not exists `movie-category` ( `id` BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, `mid` int(11) NOT NULL, `cid` int(11) NOT NULL, KEY `fk_on_movie_id` (`mid`), CONSTRAINT `fk_on_movie_id` FOREIGN KEY (`mid`) REFERENCES `movies` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, KEY `fk_on_category_id` (`cid`), CONSTRAINT `fk_on_category_id` FOREIGN KEY (`cid`) REFERENCES `categories` (`id`) ON DELETE CASCADE ON UPDATE CASCADE )ENGINE=InnoDB DEFAULT CHARSET=utf8;
INSERT INTO `categories` VALUES (1,'剧情'); INSERT INTO `categories` VALUES (2,'喜剧'); INSERT INTO `categories` VALUES (3,'动作'); INSERT INTO `categories` VALUES (4,'爱情'); INSERT INTO `categories` VALUES (5,'科幻'); INSERT INTO `categories` VALUES (6,'动画'); INSERT INTO `categories` VALUES (7,'悬疑'); INSERT INTO `categories` VALUES (8,'惊悚'); INSERT INTO `categories` VALUES (9,'恐怖'); INSERT INTO `categories` VALUES (10,'犯罪'); INSERT INTO `categories` VALUES (11,'同性'); INSERT INTO `categories` VALUES (12,'音乐'); INSERT INTO `categories` VALUES (13,'歌舞'); INSERT INTO `categories` VALUES (14,'传记'); INSERT INTO `categories` VALUES (15,'历史'); INSERT INTO `categories` VALUES (16,'战争'); INSERT INTO `categories` VALUES (17,'西部'); INSERT INTO `categories` VALUES (18,'奇幻'); INSERT INTO `categories` VALUES (19,'冒险'); INSERT INTO `categories` VALUES (20,'灾难'); INSERT INTO `categories` VALUES (21,'武侠'); INSERT INTO `categories` VALUES (22,'情色');
二，环境介绍
语言环境：Python3.7+scrapy
数据库：Mysql: mysql5.7
开发工具：IDEA或eclipse
三，核心代码展示
数据模型：items.py
# Define here the models for your scraped items # # See documentation in: # https://docs.scrapy.org/en/latest/topics/items.html import scrapy class DoubanItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() #电影标题 title = scrapy.Field() #导演 director = scrapy.Field() #编剧 scriptwriter = scrapy.Field() #演员 actors = scrapy.Field() #上映日期 date = scrapy.Field() #评分 rate = scrapy.Field() #国家/地区 district = scrapy.Field() #语言 language = scrapy.Field() #封面图片 cover = scrapy.Field() #简介 abs = scrapy.Field() #类型 categories = scrapy.Field() #时长 duration = scrapy.Field()
数据存储工具定义：pipelines.py
# Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html # useful for handling different item types with a single interface from scrapy.exceptions import DropItem from scrapy.http import Request from scrapy.pipelines.images import ImagesPipeline import pymysql import random class DoubanPipeline: def process_item(self, item, spider): return item #根据取得的图片url重新请求，下载图片到本地 class DownloadImagePipeline(ImagesPipeline): default_headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36", #"Cookie":'_vwo_uuid_v2=D65EBF690D9454DE4C13354E37DC5B9AA|3bb7e6e65f20e31141b871b4fea88dc2; __yadk_uid=QBp8bLKHjCn5zS2J5r8xV7327R0wnqkU; douban-fav-remind=1; gr_user_id=0a41d8d1-fe39-4619-827a-17961cf31795; viewed="35013197_10769749_23008813_26282806_34912177_22139960_35003794_30249691_26616244_27035127"; push_noty_num=0; push_doumail_num=0; __utmv=30149280.21320; bid=gplG4aEN4Xc; ll="108288"; ap_v=0,6.0; __utma=30149280.819011260.1572087992.1604448803.1604453561.105; __utmc=30149280; __utmz=30149280.1604453561.105.65.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __gads=ID=eddb65558a1da756-223ab4f88bc400c8:T=1604453562:RT=1604453562:S=ALNI_MZGB_I69qmiL2tt3lm57JVX1i4r2w; __utmb=30149280.4.10.1604453561; dbcl2="213202515:Ip9mjwUAab4"; ck=wxUS; __utma=223695111.897479705.1572088003.1604448803.1604455298.71; __utmb=223695111.0.10.1604455298; __utmc=223695111; __utmz=223695111.1604455298.71.42.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1604455298%2C%22https%3A%2F%2Faccounts.douban.com%2F%22%5D; _pk_ses.100001.4cf6=*; _pk_id.100001.4cf6=e11874c5506d4ab1.1572088003.71.1604455342.1604450364.' } def get_media_requests(self, item, info): #print('到这里来了...') image_url = item['cover'] yield Request( image_url, headers=self.default_headers) #get_media_requests函数返回后执行 def item_completed(self, results, item, info): image_paths = [x['path'] for ok, x in results if ok] if not image_paths: raise DropItem("Item contains no images") #返回的图片地址是full+文件名的格式，由于我是边爬边下载，所以每次只有一张图片，但是返回的是 #数组，函数设计为多张图片，我将‘full’替换成了自己后台接口的地址，方便数据库中的存储 image_paths = str(image_paths[0]).replace('full','http://localhost:8443/api/file') item['cover'] = image_paths return item # 将电影信息存入到数据库中 class DBPipeline(object): def __init__(self): # connection database # 后面三个依次是数据库连接名、数据库密码、数据库名称 self.connect = pymysql.connect(host='127.0.0.1', user='root', password='root', db='fivesix',charset='utf8',port=3306) # get cursor self.cursor_1 = self.connect.cursor() self.cursor_2 = self.connect.cursor() self.type_to_id = { '剧情': 1,'喜剧':2, '动作':3, '爱情': 4, '科幻':5, '动画':6, '悬疑': 7, '惊悚' : 8, '恐怖' : 9, '犯罪': 10, '同性':11, '音乐':12, '歌舞':13, '传记':14,'历史':15, '战争':16, '西部':17, '奇幻':18, '冒险':19, '灾难':20,'武侠':21, '情色':22 } print("连接数据库成功") def process_item(self, item, spider): if item['title'] == '': return # sql语句 insert_movie_sql = """ insert ignore into `movies`(cover,title, director, scriptwriter, actors, district,rate,date,language,duration,abs) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) """ insert_mc_sql = """ insert into `movie-category` (mid,cid) values (%s,%s) """ # 执行插入数据到数据库操作 self.cursor_1.execute(insert_movie_sql, (item['cover'], item['title'], item['director'], item['scriptwriter'], item['actors'],item['district'],item['rate'], item['date'],item['language'],item['duration'],item['abs'])) mid = self.cursor_1.lastrowid #处理标签 cids = [] categories = item['categories'].split('/') for c in categories: if c not in self.type_to_id.keys():continue cids.append(self.type_to_id.get(c)) #插入关联表 print(cids) for cid in cids: self.cursor_2.execute(insert_mc_sql,(mid,cid)) # 提交，不进行提交无法保存到数据库 self.connect.commit() def close_spider(self, spider): # 关闭游标和连接 self.cursor_1.close() self.cursor_2.close() self.connect.close()
爬虫核心代码：movies.py
# -*- coding: utf-8 -*- import scrapy import json import re import time from douban.items import DoubanItem from fake_useragent import UserAgent import random class MovieHotSpider(scrapy.Spider): #爬虫的名称，在命令行可以方便的运行爬虫 name = "movie_hot" allowed_domains = ["movie.douban.com"] #pro = ['139.224.37.83','115.223.7.110','221.122.91.75'] # 拼接豆瓣电影URL BASE_URL = 'https://movie.douban.com/j/search_subjects?type=movie&tag=%s&sort=recommend&page_limit=%s&page_start=%s' MOVIE_TAG = '华语' PAGE_LIMIT = 20 page_start = 0 domains = BASE_URL % (MOVIE_TAG, PAGE_LIMIT, page_start) #伪装浏览器 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36" #,"Cookie":'_vwo_uuid_v2=D65EBF690D9454DE4C13354E37DC5B9AA|3bb7e6e65f20e31141b871b4fea88dc2; __yadk_uid=QBp8bLKHjCn5zS2J5r8xV7327R0wnqkU; douban-fav-remind=1; gr_user_id=0a41d8d1-fe39-4619-827a-17961cf31795; viewed="35013197_10769749_23008813_26282806_34912177_22139960_35003794_30249691_26616244_27035127"; push_noty_num=0; push_doumail_num=0; __utmv=30149280.21320; bid=gplG4aEN4Xc; ll="108288"; ap_v=0,6.0; __utma=30149280.819011260.1572087992.1604448803.1604453561.105; __utmc=30149280; __utmz=30149280.1604453561.105.65.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __gads=ID=eddb65558a1da756-223ab4f88bc400c8:T=1604453562:RT=1604453562:S=ALNI_MZGB_I69qmiL2tt3lm57JVX1i4r2w; __utmb=30149280.4.10.1604453561; dbcl2="213202515:Ip9mjwUAab4"; ck=wxUS; __utma=223695111.897479705.1572088003.1604448803.1604455298.71; __utmb=223695111.0.10.1604455298; __utmc=223695111; __utmz=223695111.1604455298.71.42.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1604455298%2C%22https%3A%2F%2Faccounts.douban.com%2F%22%5D; _pk_ses.100001.4cf6=*; _pk_id.100001.4cf6=e11874c5506d4ab1.1572088003.71.1604455342.1604450364.' } #总共爬取的页数 pages = 100 # 爬虫从此开始 def start_requests(self): print('~~~~爬取列表: '+ self.domains) yield scrapy.Request( url = self.domains, headers=self.headers, callback=self.request_movies ) # 分析列表页 def request_movies(self, response): infos = response.text # 使用JSON模块解析响应结果 infos = json.loads(infos) # 迭代影片信息列表 for movie_info in infos['subjects']: print('~~~爬取电影: ' + movie_info['title'] + '/'+ movie_info['rate']) # 提取影片页面url，构造Request发送请求，并将item通过meta参数传递给影片页面解析函数 yield scrapy.Request( url = str(movie_info['url']), headers = self.headers, callback = self.request_movie, dont_filter=True ) #如果已经爬完pages或者当前标签下没有更多电影时退出 if self.pages > 0 and len(infos['subjects']) == self.PAGE_LIMIT: self.pages -= 1 self.page_start += self.PAGE_LIMIT url = self.BASE_URL % (self.MOVIE_TAG,self.PAGE_LIMIT,self.page_start) time.sleep(5) print('-----爬取列表: ' + url) yield scrapy.Request( url=url, headers=self.headers, callback=self.request_movies, dont_filter=True ) # 分析详情页 def request_movie(self, response): #组装数据 movie_item = DoubanItem() title = response.css('div#content>h1>span:nth-child(1)::text').extract_first() t = re.findall('[\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b\u4e00-\u9fa5_0-9]', title) #获取非info区域数据 movie_item['title'] = ''.join(t) movie_item['date'] = response.css('div#content>h1>span.year::text').extract_first()[1:-1] movie_item['rate'] = response.css('strong.rating_num::text').extract_first() #movie_item['commentCount'] = response.css('div.rating_sum>a.rating_people>span::text').extract_first() #movie_item['start'] = '/'.join(response.css('span.rating_per::text').extract()) #movie_item['better'] = '/'.join(response.css('div.rating_betterthan>a::text').extract()) movie_item['abs'] = response.css('#link-report>span::text').extract_first().strip() movie_item['cover'] = response.css('#mainpic>a>img::attr(src)').extract_first() # 获取整个信息字符串 info = response.css('div.subject div#info').xpath('string(.)').extract_first() # 提取所以字段名 fields = [s.strip().replace(':', '') for s in response.css('div#info span.pl::text').extract()] # 提取所有字段的值 values = [re.sub('\s+', '', s.strip()) for s in re.split('\s*(?:%s):\s*' % '|'.join(fields), info)][1:] # 处理列名称 for i in range(len(fields)): if '导演' == fields[i]: fields[i] = 'director' if '编剧' == fields[i]: fields[i] = 'scriptwriter' if '主演' == fields[i]: fields[i] = 'actors' if '类型' == fields[i]: fields[i] = 'categories' if '制片国家/地区' == fields[i]: fields[i] = 'district' if '语言' == fields[i]: fields[i] = 'language' if '片长' == fields[i]: fields[i] = 'duration' # 将所有信息填入item other_info = list(zip(fields,values)) for field,value in other_info: if field in ['IMDb链接','上映日期','官方网站','又名']: other_info.remove((field,value)) final_info = dict(other_info[:-1]) movie_item.update(final_info) # 处理缺失字段 if not 'director' in movie_item.keys(): movie_item['director'] = '/' if not 'scriptwriter' in movie_item.keys(): movie_item['scriptwriter'] = '/' if not 'actors' in movie_item.keys(): movie_item['actors'] = '/' if not 'categories' in movie_item.keys(): movie_item['categories'] = '/' if not 'district' in movie_item.keys(): movie_item['district'] = '/' if not 'language' in movie_item.keys(): movie_item['language'] = '/' if not 'duration' in movie_item.keys(): movie_item['duration'] = '/' print('~完成爬取电影: ' + movie_item['title'] + '/' + movie_item['rate']) #将数据加入到字典中 yield movie_item
四，项目总结
爬取的数据最终会存到MYSQL服务器的表中，可以写程序将数据展示出来。注意的时会对IP进行限制封号，200条为限，超过IP会被限制，可以换一个IP进行抓去。主要研究爬虫的基本使用规范和语法，相对较为简单，供大家学习参考
