分享72个Html杂七杂八模板，总有一款适合您

import os
import shutil
import time
from time import sleep
from docx import Document
from docx.shared import Inches
from selenium.webdriver.edge.options import Options
from framework.base.BaseFrame import BaseFrame
from sprider.business.SeleniumTools import SeleniumTools
from sprider.business.SpiderTools import SpiderTools
from selenium import webdriver
from selenium.webdriver.common.by import By
from sprider.model.SpriderEntity import SpriderEntity
from sprider.access.SpriderAccess import SpriderAccess
title_name = "Html模板"
class MoBanWangHtmlSelenium:
    #region 经常修改的变量
    base_url = "http://www.mobanwang.com"  # html模板王
    save_path = "D:\\Spider\\Html"
    spider_count =72  #采集数量
    spider_start_count =1812 #正在采集第55页的第12个资源，共204页资源
    #endregion
    word_content_list = []
    folder_name = "Html模板"
    page_end_number = 50 # 最后一页
    max_pager = 30  # 每页的数量 影响再次采集的起始位置
    haved_spider_count = 0  # 已经采集的数量
    page_count = 1  # 每个栏目开始业务content="text/html; charset=gb2312"
    filter_down_file = []
    word_image_count = 1  # word插入图片数量 同时也是明细采集图片和描述的数量
    sprider_detail_index = 0  # 明细方法采集的数量 累加
    tempFileList = []
    def spider(self, title_name):
        """
        :return:
        """
        if title_name == "Html册模板":
            self.first_column_name="mb"
            self.second_column_name = "album"
        elif title_name == "Html模板":
            self.first_column_name = "mb"
            self.second_column_name = "other"
        BaseFrame().debug("开始采集" + self.folder_name + "...")
        self.merchant = int(self.spider_start_count) // int(self.max_pager) + 1  # 起始页码用于效率采集
        BaseFrame().right("本次采集参数：日志路径=" + str(BaseFrame().LOG_PATH) + "")
        BaseFrame().right("本次采集参数：spider_count=" + str(self.spider_count) + "")
        BaseFrame().right("本次采集参数：title_name=" + title_name + "")
        self.sprider_category = title_name  # 一级目录self.folder_name
        #多少个 某某某
        second_folder_name = str(self.spider_count) + "个" + title_name  # 二级目录
        self.sprider_type = second_folder_name #多少个某某某东东
        #浏览器的下载路径
        #self.down_path = self.save_path + os.sep +"\\" + self.folder_name + "\\" + second_folder_name + "\\Temp\\"
        self.down_path = self.save_path + os.sep + self.folder_name + os.sep + second_folder_name + "\\Temp"
        BaseFrame().right("本次采集参数：down_path=" +  self.down_path + "")
        # 原始路径+一级目录+二级目录
        # First/PPT/88个动态PPT模板
        self.file_path = self.save_path + os.sep + self.folder_name + os.sep + second_folder_name
        BaseFrame().right("本次采集参数：file_path=" + self.file_path + "")
        # First/PPT/88个动态PPT模板/动态PPT模板
        self.save_path = self.save_path + os.sep + self.folder_name + os.sep + second_folder_name + os.sep + title_name
        BaseFrame().right("本次采集参数：save_path=" + self.save_path + "")
        #根据栏目构建URL
        spider_url = self.base_url + "/{0}/{1}".format(self.first_column_name,self.second_column_name)
        if os.path.exists(self.down_path) is True:
            shutil.rmtree(self.down_path)
        if os.path.exists(self.down_path) is False:
            os.makedirs(self.down_path)
        if os.path.exists(self.save_path) is True:
            shutil.rmtree(self.save_path)
        if os.path.exists(self.save_path) is False:
            os.makedirs(self.save_path)
        edge_options = Options()
        diy_prefs = {'profile.default_content_settings.popups': 0,
                     'download.default_directory': '{0}'.format(self.down_path)}
        # 添加路径到selenium配置中
        edge_options.add_experimental_option('prefs', diy_prefs)
        edge_options.add_argument('--headless')  # 隐藏浏览器
        # 实例化chrome浏览器时，关联忽略证书错误
        browser = webdriver.Edge(options=edge_options)
        # 窗口最大化
        #  browser.maximize_window()
        browser.set_window_size(1300, 1000)  # 分辨率 1280*800
        browser.get(spider_url)
        # content = driver.page_source
        #masonry-brick
        element_list = browser.find_elements(By.CLASS_NAME, "mb-mblist")
        if SeleniumTools.judeg_element_isexist(browser, "LINK_TEXT", "下一页") == 1:
            element = browser.find_element(By.LINK_TEXT, "下一页")
            next_url = element.get_attribute("href")
            page_url = next_url.split("_")[1] # 47.html
            self.page_end_number=int(page_url.split(".")[0])
            #class ="page-link" href="http://www.mobanwang.com/mb/ind/List_47.html" > 下一页 < / a >
        real_page_number=int(self.page_end_number)+1
        self.page_count = self.merchant
        while True:
            try:
                if self.page_count == real_page_number-int(self.page_end_number):
                    self.spider_detail(browser, element_list, self.page_count, self.page_end_number, self.down_path)
                    pass
                else:
                    if self.haved_spider_count == self.spider_count:
                        BaseFrame().debug("spider采集到达数量采集停止...")
                        BaseFrame().debug("开始写文章...")
                        self.builder_word(self.folder_name, self.word_content_list)
                        BaseFrame().debug("文件编写完毕，请到对应的磁盘查看word文件和下载文件！")
                        break
                    next_url = self.base_url + "/{0}/{1}/List_{2}.html".format(self.first_column_name,
                                                                       self.second_column_name,
                                                                       real_page_number -self.page_count)
                    BaseFrame().debug("开始采集:" + next_url)
                    browser.get(next_url)
                    element_list = browser.find_elements(By.CLASS_NAME, "mb-mblist")
                    self.spider_detail(browser, element_list, self.page_count, self.page_end_number, self.down_path)
                if int(self.page_end_number)==self.page_count:
                    break
                self.page_count = self.page_count + 1  # 页码增加1
            except Exception as e:
                print("spider()执行过程出现错误:" + str(e))
                break
    def spider_detail(self, driver, element_list, page_count, max_page, down_path):
        """
        采集明细页面
        :param driver:
        :param element_list:
        :param page_count:
        :param max_page:
        :param down_path:
        :return:
        """
        index = 0
        element_array = []
        element_length = len(element_list)
        print("当前页" + str(page_count) + "每页共" + str(element_length))
        for element in element_list:
            element_a = element.find_element(By.TAG_NAME, "a")
            # print(html_content)
            next_url = element_a.get_attribute("href")
            coder_title = element_a.get_attribute("title")
            e = coder_title + "$" + next_url
            element_array.append(e)
            pass
        #再次开始采集不从第一页开始采集 提升效率
        if int(self.page_count) == int(self.merchant):
            self.sprider_start_index = int(self.spider_start_count) % int(self.max_pager)
            index = self.sprider_start_index
        while index < element_length:
            if os.path.exists(down_path) is False:
                os.makedirs(down_path)
            if self.haved_spider_count == self.spider_count:
                BaseFrame().debug("spider_detail采集到达数量采集停止...")
                break
            # element = element_list[index]
            element = element_array[index]
            time.sleep(1)
            index = index + 1
            spider_info = "正在采集第" + str(page_count) + "页的第" + str(index) + "个资源，共" + str(max_page) + "页资源"
            BaseFrame().debug(spider_info)
            next_url = element.split("$")[1]
            coder_title = element.split("$")[0]
            try:
                codeEntity = SpriderEntity()  # 下载过的资源不再下载
                codeEntity.sprider_base_url = self.base_url
                codeEntity.create_datetime = SpiderTools.get_current_datetime()
                codeEntity.sprider_url = next_url
                codeEntity.sprider_pic_title = coder_title
                codeEntity.sprider_pic_index = str(index)
                codeEntity.sprider_pager_index = page_count
                codeEntity.sprider_type = self.sprider_type
                if SpriderAccess().query_sprider_entity_by_urlandindex(coder_title) is None:
                    SpriderAccess().save_sprider(codeEntity)
                else:
                    BaseFrame().debug(coder_title + next_url + "数据采集过因此跳过")
                    continue
                driver.get(next_url)  # 请求明细页面1
                BaseFrame().debug("采集明细内容网址：" + next_url)
                #mb-color-666
                span_element = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div[1]/span[2]")
                if "psd" in span_element.text : #html│psd│flash
                    BaseFrame().right("PSD格式的文件因此去跳过")
                    continue
                if SeleniumTools.judeg_element_isexist(driver, "LINK_TEXT", "免费下载") == 1:
                    print("准备点击【免费下载】按钮...")
                    div_element = driver.find_element(By.LINK_TEXT, "免费下载")  # 下载源码.click()
                    div_element.click()
                else:
                    if SeleniumTools.judeg_element_isexist(driver, "LINK_TEXT", "点击下载") == 1:
                        print("准备【点击下载】按钮...")
                        a_element = driver.find_element(By.LINK_TEXT, "点击下载")  # 下载源码.click()
                        a_element.click()
                    else:
                        BaseFrame().debug(coder_title + "下载按钮不存在，因此跳过哦....")
                        continue
                result, message = SpiderTools.judge_file_exist(True, 600, 1, down_path, "zip|rar|pptx|ppt", self.tempFileList)  # 判断源码
                if result is True:
                    title_content = ""
                    title_image = ""
                    spider_content = [coder_title, title_image, title_content]  # 图片和描述采集成功的记录
                    self.word_content_list.append(spider_content)  # 增加到最终的数组
                    self.haved_spider_count = self.haved_spider_count + 1
                    BaseFrame().right(
                        "已经采集完成第" + str(self.haved_spider_count) + "个，共需要采集" + str(self.spider_count) + "个")
                    time.sleep(1)
                    driver.back()
                    #移动文件
                    coder_title = str(coder_title).replace("::", "").replace("/", "").strip()  # 去掉windows不识别的字符
                    files = os.listdir(down_path)
                    file_name = files[0]  # 获取默认值
                    srcFile = down_path + os.sep + file_name
                    file_ext = os.path.splitext(srcFile)[-1]
                    dstFile = down_path + os.sep + coder_title + file_ext
                    os.rename(srcFile, dstFile)
                    srcFile = dstFile
                    dstFile = self.save_path + os.sep + coder_title + file_ext
                    shutil.move(srcFile, dstFile)  # 移动文件
                    self.sprider_detail_index = self.sprider_detail_index + 1
                else:
                    print("result=" + str(result) + "" + message)
                    files = os.listdir(down_path)  # 读取目录下所有文件
                    coder_title = str(coder_title).replace("/", "")  # 去掉windows不识别的字符
                    try:
                        if str(message) == "0个文件认定是False":
                            BaseFrame().error(coder_title + "文件不存在...")
                            pass
                        else:
                            BaseFrame().error("检测下载文件出错可能原因是等待时间不够已经超时，再等待60秒...")
                            time.sleep(60)
                            shutil.rmtree(down_path)  # 如果没下载完是无法删除的
                            # 清空数组
                            self.filter_down_file.clear()
                    except Exception as e:
                        BaseFrame().error(e)
                        # 使用数组append记录文件名字 移动的时候过滤
                        shutil.rmtree(down_path)  # 如果没下载完是无法删除的
                    pass
            except Exception as e:
                BaseFrame().error("sprider_detail()执行过程出现错误:" + str(e))
                #shutil.rmtree(down_path)  # 如果没下载完是无法删除的
        if int(page_count) == int(max_page):
            self.builder_word(self.folder_name, self.word_content_list)
            BaseFrame().debug("文件编写完毕，请到对应的磁盘查看word文件和下载文件！")
            return