分享79个ASP影音娱乐源码，总有一款适合您

import os
import shutil
import requests
from bs4 import BeautifulSoup
from framework.base.BaseFrame import BaseFrame
from sprider.access.SpriderAccess import SpriderAccess
from sprider.plugin.SpriderTools import SpriderTools
from sprider.plugin.UserAgent import UserAgent
class CNSourceCoder:
    base_url = "http://www.downcode.com"
    max_pager = 10  # 每页的数量 影响再次采集的起始位置.
    word_image_count = 5  # word插入图片数量 同时也是明细采集图片和描述的数量
    page_end_number = 0
    word_content_list = []
    haved_sprider_count = 0  # 已经采集的数量
    sprider_detail_index = 0  # 明细方法采集的数量 累加

def sprider_detail(self, driver, element_list, max_page):
        """
        采集明细页面
        :param driver:
        :param element_list:
        :param max_page:
        :param down_path:
        :return:
        """
        index = 0
        element_array = []
        element_length = len(element_list)
        print("当前页" + str(self.page_count) + "每页共" + str(element_length))
        for element in element_list:
            #div_element = element.find_element(By.CLASS_NAME, 'bot-div')
            a_element = element.find_element(By.TAG_NAME, 'a')
            next_url = a_element.get_attribute("href")
            coder_title = element.find_element(By.TAG_NAME, 'h2').text
            e = coder_title + "$" + next_url
            element_array.append(e)
            pass
        if int(self.page_count) == int(self.merchant):
            self.sprider_start_index = int(self.sprider_start_count) % int(self.max_pager)
            index = self.sprider_start_index
        while index < element_length:
            if os.path.exists(self.down_path) is False:
                os.makedirs(self.down_path)
            if self.haved_sprider_count == self.sprider_count:
                BaseFrame().debug("采集到达数量采集停止...")
                break
            # element = element_list[index]
            element = element_array[index]
            time.sleep(1)
            index = index + 1
            sprider_info = "正在采集第" + str(self.page_count) + "页的第" + str(index) + "个资源，共" + str(max_page) + "页资源"
            BaseFrame().debug(sprider_info)
            next_url = element.split("$")[1]
            coder_title = element.split("$")[0]
            coder_title = str(coder_title).replace(":", "").replace("/", "").strip()  # 去掉windows不识别的字符
            try:
                # sproier_entity= SpriderTools.builder_sprider_entity(self.base_url,next_url,coder_title,self.page_count,self.second_folder_name)
                # if SpriderAccess().query_sprider_entity_by_urlandindex(coder_title) is None:
                #     SpriderAccess().save_sprider(sproier_entity)
                # else:
                #     BaseFrame().debug(coder_title + next_url + "数据采集过因此跳过")
                #     continue
                if next_url=="http://www.downcode.com/downcode/j_20065.shtml":
                    sleep(1)  # 延迟会
                # else:
                #     continue
                driver.get(next_url)  # 请求明细页面1
                BaseFrame().debug("采集明细内容网址：" + next_url)
                sleep(0.5)  # 延迟会
                if SeleniumTools.judeg_element_isexist(driver, "CLASS_NAME", "one_4") == 3:
                    BaseFrame().debug(coder_title + "下载按钮不存在，因此跳过哦....")
                    continue
                BaseFrame().debug("准备点击下载按钮...")
                elements = driver.find_elements(By.CLASS_NAME, "one_4")
                if len(elements)==2:
                    a_element = elements[1].find_element(By.TAG_NAME, "a")
                    a_element.click()
                    sleep(0.5)
                else:
                    BaseFrame().error(str(len(elements))+"查找下载页面失败进入下一个。。。")
                    continue
                result, message = SpriderTools.judge_file_exist(True, 240, 1,
                                                                self.down_path,"zip|rar|gz|tgz",
                                                                self.tempFileList)  # 判断源码
                if result is True:
                    title_content = ""
                    title_image = ""
                    if self.sprider_detail_index <= self.word_image_count:
                        pass
                        # try:
                        #     # 下载图片
                        #     div_element_image = driver.find_element(By.CLASS_NAME, "left-show-img-box")
                        #
                        #     img_element_image = div_element_image.find_element(By.TAG_NAME, "img")  # 其实是2个查询第一个
                        #     image_href = img_element_image.get_attribute("src")
                        #     DownLoad(self.save_path).down_cover_image__(image_href, coder_title)  # 资源的 封面
                        #     title_image = self.save_path + os.sep + "image" + os.sep + coder_title + ".jpg"
                        # except Exception as e:
                        #     BaseFrame().error("下载图片:"+str(e))
                        # 采集描述
                        # try:
                        #     if SeleniumTools.judeg_element_isexist(driver, "CLASS_NAME", "one_4") == 3:
                        #         pass
                        #     else:
                        #         element_content = driver.find_elements(By.CLASS_NAME, "one_4")
                        #         title_content = element_content[0].text
                        #         BaseFrame().right("采集模板简介成功")
                        # except Exception as e:
                        #     BaseFrame().error("采集模板简介失败" + str(e))
                        #     pass
                    driver.back()
                    coder_title = str(coder_title).replace(":", "").replace("/", "").strip()  # 去掉windows不识别的字符
                    files = os.listdir(self.down_path)
                    file_name = files[0]  # 获取默认值
                    srcFile = self.down_path + os.sep + file_name
                    file_ext = os.path.splitext(srcFile)[-1]
                    #print("源文件"+srcFile)
                    dstFile = self.save_path + os.sep + coder_title + file_ext
                    #print("目标文件" + dstFile)
                    shutil.move(srcFile, dstFile)  # 移动文件
                    #print("移动成功")
                    sprider_content = [coder_title, title_image, title_content]  # 采集成功的记录
                    self.word_content_list.append(sprider_content)  # 增加到最终的数组
                    self.haved_sprider_count = self.haved_sprider_count + 1
                    BaseFrame().right(
                        "已经采集完成第" + str(self.haved_sprider_count) + "个，共需要采集" + str(
                            self.sprider_count) + "个")
                    self.sprider_detail_index = self.sprider_detail_index + 1
                else:
                    print("result=" + str(result) + "" + message)
                    coder_title = str(coder_title).replace("/", "")  # 去掉windows不识别的字符
                    try:
                        if str(message) == "0个文件认定是False":
                            BaseFrame().error(coder_title + "文件不存在...")
                            pass
                        else:
                            # BaseFrame().error("检测下载文件出错可能原因是等待时间不够已经超时，再等待60秒...")
                            # time.sleep(60)
                            # shutil.rmtree(self.down_path)  # 如果没下载完是无法删除的
                            self.add_temp_files()
                    except Exception as e:
                        BaseFrame().error(e)
                        # 使用数组append记录文件名字 移动的时候过滤
                        self.add_temp_files()
                    pass
            except Exception as e:
                BaseFrame().error("sprider_detail()执行过程出现错误:" + str(e))
                self.add_temp_files()
        if int(self.page_count) == int(max_page):
            SpriderTools.builder_word(self.second_folder_name, self.word_content_list,self.file_path,self.word_image_count,self.first_folder_name )
            SpriderTools.copy_file( self.word_content_list,self.save_path,  self.second_folder_name,self.file_path)
            SpriderTools.gen_passandtxt(self.second_folder_name,  self.word_content_list,self.file_path)
            BaseFrame().debug("文件编写完毕，请到对应的磁盘查看word文件和下载文件！")
    def add_temp_files(self):
        files = os.listdir(self.down_path)  # 读取目录下所有文件
        if len(self.tempFileList)==0:
            self.tempFileList.append(files[0])
        for file in files:
            if file.__contains__(".crdownload") is True:
                for tempFile in self.tempFileList:
                    if tempFile == file:
                        break
                    self.tempFileList.append(file)
        self.tempFileList=list(set(self.tempFileList))
        pass

def sprider(self,sprider_name,sprider_count,start_number,
                pager_number,sprider_type,is_show_browse,root_path,
                first_column_name,second_column_name,is_record_db):
        """
        http://www.downcode.com/sort/j_1_2_1.shtml
        :param sprider_name:
        :return:
        """
        self.first_folder_name=sprider_type.upper()
        self.base_path=root_path
        self.first_column_name = first_column_name
        self.second_column_name = second_column_name
        self.sprider_start_count=start_number
        self.is_record_db=is_record_db
        BaseFrame().debug("开始采集[源码下载站]" + self.first_folder_name + "源码...")
        BaseFrame().right("本次采集参数：sprider_count(采集数量):" + str(sprider_count) + "")
        BaseFrame().right("本次采集参数：sprider_name(采集名称):" + sprider_name + "")
        sprider_url = self.base_url + "/{0}/{1}_1.shtml".format(self.first_column_name, self.second_column_name)# 根据栏目构建URL
        BaseFrame().debug("本次采集参数：sprider_url:" + sprider_url)
        self.second_folder_name = str(sprider_count) + "个" + sprider_name  # 二级目录也是wordTitle
        self.merchant = int(self.sprider_start_count) // int(self.max_pager) + 1  # 起始页码用于效率采集
        self.file_path = self.base_path + os.sep + self.first_folder_name + os.sep + self.second_folder_name
        BaseFrame().right("本次采集参数：file_path=" + self.file_path + "")
        # 浏览器的下载路径
        self.down_path = self.base_path + os.sep + self.first_folder_name+ os.sep + self.second_folder_name+ "\\Temp\\"
        BaseFrame().right("本次采集参数：down_path=" + self.down_path + "")
        # First/PPT/88个动态PPT模板/动态PPT模板
        self.save_path = self.base_path + os.sep + self.first_folder_name + os.sep + self.second_folder_name + os.sep + sprider_name
        BaseFrame().right("本次采集参数：save_path=" + self.save_path + "")
        if os.path.exists(self.down_path) is True:
            shutil.rmtree(self.down_path)
        if os.path.exists(self.down_path) is False:
            os.makedirs(self.down_path)
        if os.path.exists(self.save_path) is True:
            shutil.rmtree(self.save_path)
        if os.path.exists(self.save_path) is False:
            os.makedirs(self.save_path)
        response = requests.get(sprider_url, timeout=10, headers=UserAgent().get_random_header(self.base_url))
        response.encoding = 'gb2312'
        soup = BeautifulSoup(response.text, "html5lib")
        #print(soup)
        element_list = soup.find_all('div', attrs={"class": 'j_text_sort_a'})
        page_end_number = 1
        page_end_url = ""
        page_end_title = soup.find("a", attrs={"title": '最后页'})
        page_end_url = page_end_title.get("href")
        if page_end_url is None or page_end_url == "":
            page_end_number = 1
        else:
            page_end_number = (page_end_url.split(".shtml")[0].split("_")[3])
        self.page_count = self.merchant
        while self.page_count <= int(page_end_number):  # 翻完停止
            try:
                if self.page_count == 1:
                    self.sprider_detail(element_list, page_end_number,sprider_count)
                    pass
                else:
                    if int(self.haved_sprider_count) == int(sprider_count):
                        BaseFrame().debug("sprider采集到达数量采集停止...")
                        BaseFrame().debug("开始写文章...")
                        SpriderTools.builder_word(self.second_folder_name,
                                                  self.word_content_list,
                                                  self.file_path,
                                                  self.word_image_count,
                                                  self.first_folder_name)
                        SpriderTools.copy_file(self.word_content_list,
                                               self.save_path,
                                               self.second_folder_name,
                                               self.file_path)
                        SpriderTools.gen_passandtxt(self.second_folder_name, self.word_content_list, self.file_path)
                        BaseFrame().debug("文件编写完毕，请到对应的磁盘查看word文件和下载文件！")
                        break
                    next_url = self.base_url + "/{0}/{1}_{2}.shtml".format(self.first_column_name,
                                                                           self.second_column_name,
                                                                           self.page_count)
                    response = requests.get(next_url, timeout=10, headers=UserAgent().get_random_header(self.base_url))
                    response.encoding = 'gb2312'
                    soup = BeautifulSoup(response.text, "html5lib")
                    element_list = soup.find_all('div', attrs={"class": 'j_text_sort_a'})
                    self.sprider_detail(element_list, page_end_number,sprider_count)
                    pass
                self.page_count = self.page_count + 1  # 页码增加1
            except Exception as e:
                BaseFrame().error("sprider()执行过程出现错误:" + str(e))