基于Python采集136个PHP源码

page_count = 1  # 每个栏目开始业务content="text/html; charset=gb2312"
    base_url = "https://down.chinaz.com"  # 采集的网址  https://sc.chinaz.com/tag_ppt/zhongguofeng.html
    save_path = "D:\\Freedom\\Sprider\\ChinaZ\\"
    sprider_count = 136# 采集数量
    haved_sprider_count = 0  # 已经采集的数量
    word_content_list = []
    folder_name = ""
    first_column_name = "ppt"
    sprider_start_count=120 #已经采集完成第136个 debug
    max_pager=16 #每页的数量

def sprider(self, title_name="PHP"):
        """
        采集 https://down.chinaz.com/class/572_5_1.htm
        :return:
        """
        if title_name == "PHP":
            self.folder_name = "PHP源码"
            self.second_column_name = "572_5"
        elif title_name == "Go":
            self.folder_name = "Go源码"
            self.second_column_name = "606_572"
        merchant = int(self.sprider_start_count) // int(self.max_pager) + 1
        second_folder_name = str(self.sprider_count) + "个" + self.folder_name
        self.save_path = self.save_path+ os.sep + "Code" + os.sep +  second_folder_name
        print("开始采集ChinaZCode"+self.folder_name+"...")
        sprider_url = (self.base_url + "/class/{0}_1.htm".format(self.second_column_name))
        #print(sprider_url)
        #sprider_url = (self.base_url + "/" +  self.first_column_name + "/" + second_column_name + ".html")
        response = requests.get(sprider_url, timeout=10, headers=UserAgent().get_random_header(self.base_url))
        response.encoding = 'UTF-8'
        soup = BeautifulSoup(response.text, "html5lib")
        #print(soup)
        div_list =soup.find('div', attrs={"class": 'main'})
        div_list=div_list.find_all('div', attrs={"class": 'item'})
        #print(div_list)
        laster_pager_ul = soup.find('ul', attrs={"class": 'el-pager'})
        laster_pager_li = laster_pager_ul.find_all('li', attrs={"class": 'number'})
        laster_pager_url = laster_pager_li[len(laster_pager_li)-1]
        #<a href="zhongguofeng_89.html"><b>89</b></a>
        page_end_number = int(laster_pager_url.string)
        #print(page_end_number)
        self.page_count = merchant
        while self.page_count <= int(page_end_number):  # 翻完停止
            try:
                if self.page_count == 1:
                    self.sprider_detail(div_list,self.page_count,page_end_number)
                else:
                    if self.haved_sprider_count == self.sprider_count:
                        BaseFrame().debug("采集到达数量采集停止...")
                        BaseFrame().debug("开始写文章...")
                        self.builder_word(self.folder_name, self.save_path, self.word_content_list)
                        BaseFrame().debug("文件编写完毕，请到对应的磁盘查看word文件和下载文件！")
                        break
                    next_url =self.base_url + "/class/{0}_{1}.htm".format(self.second_column_name,self.page_count )
                    response = requests.get(next_url, timeout=10, headers=UserAgent().get_random_header(self.base_url))
                    response.encoding = 'UTF-8'
                    soup = BeautifulSoup(response.text, "html5lib")
                    div_list = soup.find('div', attrs={"class": 'main'})
                    div_list = div_list.find_all('div', attrs={"class": 'item'})
                    self.sprider_detail(div_list, self.page_count,page_end_number)
                    pass
            except Exception as e:
                print("sprider()执行过程出现错误" + str(e))
                pass
            self.page_count = self.page_count + 1  # 页码增加1
    def sprider_detail(self, element_list, page_count,max_page):
        try:
            element_length = len(element_list)
            self.sprider_start_index = int(self.sprider_start_count) % int(self.max_pager)
            index = self.sprider_start_index
            while index < element_length:
                a=element_list[index]
                if self.haved_sprider_count == self.sprider_count:
                    BaseFrame().debug("采集到达数量采集停止...")
                    break
                index = index + 1
                sprider_info = "正在采集第" + str(page_count) + "页的第" + str(index) + "个资源"
                print(sprider_info)
                #title_image_obj = a.find('img', attrs={"class": 'lazy'})
                url_A_obj=a.find('a', attrs={"class": 'name-text'})
                next_url = self.base_url+url_A_obj.get("href")
                coder_title = url_A_obj.get("title")
                response = requests.get(next_url, timeout=10, headers=UserAgent().get_random_header(self.base_url))
                response.encoding = 'UTF-8'
                soup = BeautifulSoup(response.text, "html5lib")
                #print(soup)
                down_load_file_div = soup.find('div', attrs={"class": 'download-list'})
                if down_load_file_div is None:
                    BaseFrame().debug("应该是多版本的暂时不下载因此跳过哦....")
                    continue
                down_load_file_url =self.base_url+down_load_file_div.find('a').get("href")
                #image_obj = soup.find('img', attrs={"class": "el-image__inner"})
                #image_src =self.base_url+image_obj.get("src")
                #print(image_src)
                codeEntity = SpriderEntity()  # 下载过的资源不再下载
                codeEntity.sprider_base_url = self.base_url
                codeEntity.create_datetime = SpriderTools.get_current_datetime()
                codeEntity.sprider_url = next_url
                codeEntity.sprider_pic_title = coder_title
                codeEntity.sprider_pic_index = str(index)
                codeEntity.sprider_pager_index = page_count
                codeEntity.sprider_type = "code"
                if SpriderAccess().query_sprider_entity_by_urlandindex(next_url, str(index)) is None:
                    SpriderAccess().save_sprider(codeEntity)
                else:
                    BaseFrame().debug(coder_title + next_url + "数据采集过因此跳过")
                    continue
                if (DownLoad(self.save_path).down_load_file__(down_load_file_url, coder_title, self.folder_name)):
                    #DownLoad(self.save_path).down_cover_image__(image_src, coder_title)  # 资源的 封面
                    sprider_content = [coder_title,
                                       self.save_path + os.sep + "image" + os.sep + coder_title + ".jpg"]  # 采集成功的记录
                    self.word_content_list.append(sprider_content)  # 增加到最终的数组
                    self.haved_sprider_count = self.haved_sprider_count + 1
                    BaseFrame().debug("已经采集完成第" + str(self.haved_sprider_count) + "个")
            if (int(page_count) == int(max_page)):
                self.builder_word(self.folder_name, self.save_path, self.word_content_list)
                BaseFrame().debug("文件编写完毕，请到对应的磁盘查看word文件和下载文件！")
        except Exception as e:
            print("sprider_detail:" + str(e))
            pass
    def builder_word(self, word_title, save_path, list_files):
        """
        输出产物是word文件
        :param word_title: 文件的标题
        :param save_path: 文件的保存路径
        :param list_files: 文件集合（单个内容）
        :return:
        """
        try:
            self.copy_file(self.save_path)
            print("Create Word"+word_title)
            file_count= len(list_files)
            self.gen_passandtxt(file_count,word_title)
            document = Document()
            document.add_heading(word_title, level=2)
            document.add_paragraph("分享"+str(file_count)+"个"+str(word_title)+"，总有一款适合您\r\n"
                                   "下面是文件的名字，我放了一些图片，文章里不是所有的图主要是放不下...，大家下载后可以看到。")
            ppt_tieles = ""
            for files in list_files:
                ppt_tieles = ppt_tieles + str(files[0]) + "\r"
            document.add_paragraph(ppt_tieles)
            for files in list_files:
                try:
                    document.add_paragraph(files[0])
                    document.add_picture(files[1], width=Inches(3))
                except Exception as e:
                    pass
            document.add_paragraph("最后送大家一首诗:")
            paragraph = document.add_paragraph()  # 单独控制
            paragraph.add_run("山高路远坑深,\r")
            paragraph.add_run("大军纵横驰奔,\r")
            paragraph.add_run("谁敢横刀立马？\r")
            paragraph.add_run("惟有点赞加关注大军。\r")
            paragraph.bold = True  # 字体加粗
            file_full_path=save_path+os.sep+word_title+".docx"
            document.save(file_full_path)
        except Exception as e:
            print("Create Word Fail reason:" + str(e))
    def gen_passandtxt(self,file_count,word_title):
        print("Create PassWord and Pass.txt")
        message=SpriderTools.gen_password()
        password = "".join(message)
        content=""
        content = content + "\n分享"+str(file_count)+"个"+word_title+"，总有一款适合您\r\n"
        content = content + "\n\r"
        content=content+"\n都到这里了您就支持一下呗！谢谢老铁~~"
        content=content+"\n\r"
        content=content+"\n文件我就不一一列举了,送老铁一首打油诗"
        content=content+"\n学习知识费力气，"
        content=content+"\n收集整理更不易。"
        content=content+"\n知识付费甚欢喜，"
        content=content+"\n为咱码农谋福利。"
        content=content+"\n\r"
        content=content+"\n\r"
        content=content+"\n感谢您的支持"
        content=content+"\n\r"
        content=content+"\n-------------------------------------------华丽分割线-------------------------------------------------------"
        content=content+"\n友情提醒解压密码："+password+""
        full_path=self.save_path+os.sep+""+str(file_count)+"sell_pass.txt"
        with open(full_path, 'a', encoding='utf-8') as f:
            f.write(content)
    def copy_file(self,target_path):
        print("copy files")
        import os
        import shutil
        # src_apk_file_path="薅羊毛专业版.apk"
        # dst_apk_file_path=target_path+os.sep+"薅羊毛专业版.apk"
        # shutil.copyfile(src_apk_file_path, dst_apk_file_path)  # 移动文件
        #
        # src_pdf_file_path = "薅羊毛专业版.pdf"
        # dst_pdf_file_path = target_path + os.sep + "薅羊毛专业版.pdf"
        # shutil.copyfile(src_pdf_file_path, dst_pdf_file_path)  # 移动文件
        src_doc_file_path = "readme.docx"
        dst_doc_file_path = target_path + os.sep + "readme.docx"
        shutil.copyfile(src_doc_file_path, dst_doc_file_path)  # 移动文件
        pass