采集笔趣阁小说，使用以下几个核心包：
requests：2.10.0
beautifulsoup4：4.7.1
其中：
1、BaseFrame.__log__("开始采集中国船舶网的数据...") BaseFrame.__log__() 就是我个人封装的日之类替换成print就可以。
2、response = requests.get(self.base_url, timeout=30, headers=UserAgent().get_random_header(self.base_url)) 这个就是封装的一个随机header，防止被认为是机器，每次都模拟一个新的浏览器的header。代码是这个：UserAgent().get_random_header(self.base_url) 随机header类：
3、filepath = BaseConfig().CORPUS_ROOT + os.sep + "equipment_info.xlsx" 这个就是文件的路径BaseConfig().CORPUS_ROOT替换成自己的文件路径就可以了。
4、mmEntity = SpriderEntity() 这个就是一个实体类用来记录采集过的数据，程序可以多次执行。防止数据重复采集的策略。
不废话了直接上代码：
import os import requests from bs4 import BeautifulSoup from access.sprider.SpriderAccess import SpriderAccess from base.BaseConfig import BaseConfig from base.BaseFrame import BaseFrame from business.sprider.UserAgent import UserAgent from object.entity.SpriderEntity import SpriderEntity from plugin.Tools import Tools class QuLa: base_url = "https://www.qu.la/" save_path = BaseConfig().CORPUS_ROOT + os.sep + "QuLa" def __init__(self): Tools.judge_diskpath_exits_create(self.save_path) pass def sprider_story(self): BaseFrame.__log__("开始采集笔趣阁排行榜小说...") self.story_url = self.base_url + "paihangbang" try: response = requests.get(self.story_url, timeout=30, headers=UserAgent().get_random_header(self.story_url)) response.encoding = 'UTF-8' soup = BeautifulSoup(response.text, "html5lib") except Exception as e: BaseFrame.__err__("采集出现错误" + str(e)) pass div_list = soup.findAll('div', attrs={"class": 'topbooks'}) for div in div_list: a_list = div.find_all('a', attrs={"target": '_blank'}) for a in a_list: content_url = self.base_url + a.get("href") txt_title = a.get("title") try: response = requests.get(content_url, timeout=30, headers=UserAgent().get_random_header(content_url)) response.encoding = 'UTF-8' soup = BeautifulSoup(response.text, "html5lib") dl_tag = soup.find('dl') a_list = dl_tag.find_all('a') for a_tag in a_list: href = a_tag.get("href") if "book" in href: url = self.base_url + href chapter = a_tag.text mmEntity = SpriderEntity() mmEntity.sprider_base_url = self.base_url mmEntity.create_datetime = Tools.get_current_datetime() mmEntity.sprider_url = url mmEntity.sprider_pic_title = chapter mmEntity.sprider_pic_index = str(1) if SpriderAccess().query_sprider_entity_by_urlandtitle(url, chapter) is None: SpriderAccess().save_sprider(mmEntity) self.get_content(url, chapter, txt_title) except Exception as e: BaseFrame.__err__("采集" + content_url + "出现错误" + str(e)) pass pass def get_content(self, url, chapter, title): """ 写文件至文本中 :param url:采集的URL :param chapter: 章节名称 :param title: 小说名称 :return: """ try: BaseFrame.__log__("正在采集" + url + "上的小说...") response = requests.get(url, timeout=60, headers=UserAgent().get_random_header(url)) response.encoding = 'UTF-8' soup = BeautifulSoup(response.text, "html5lib") content = soup.find('div', attrs={"id": 'content'}) content = chapter + "\n" + str(content). \ replace('<br/>', '\n'). \ replace("<script>chaptererror();</script>", ""). \ replace("<div id=\"content\">", ""). \ replace("</div>", "") txt_path = self.save_path + os.sep + str(title) + ".txt" Tools.write_string_to_txt(txt_path, content) except Exception as e: BaseFrame.__err__("采集" + chapter + "出现错误" + str(e) + "尝试重新采集.") self.get_content(url, chapter, title) pass if __name__ == '__main__': QuLa().sprider_story() pass
