分享68个Python爬虫源码总有一个是你想要的

# region 采集
    def spider_git(self):
        """
        采集git网
        :return:
        """
        spider_url = self.txt_spider_url.GetValue()
        # if os.path.exists(self.first_path):
        #     os.makedirs(self.first_path)
        # else:
        #     os.makedirs(self.first_path)
        edge_options = Options()
        diy_prefs = {'profile.default_content_settings.popups': 0,
                     'download.default_directory': '{0}'.format("D:\\Temp")}
        # 添加路径到selenium配置中
        edge_options.add_experimental_option('prefs', diy_prefs)
        edge_options.add_argument('--headless')  # 隐藏浏览器
        # 实例化chrome浏览器时，关联忽略证书错误
        browser = webdriver.Edge(options=edge_options)
        browser.set_window_size(1300, 1000)  # 分辨率 1280*800
        browser.get(spider_url)
        self.browser = browser  # 将浏览器的独行值赋予给全局
        title_element = browser.find_element(By.CLASS_NAME, "my-3")
        coder_title=str(title_element.text)
        # response = requests.get(spider_url,timeout=10, headers=UserAgent().get_random_header("https://github.com/"))
        # response.encoding = 'UTF-8'
        # soup = BeautifulSoup(response.text, "html5lib")
        # p_element = soup.find('p', attrs={"class": 'my-3'})
        # print(p_element.text.strip())
        try:
            folder_name = str(coder_title).strip()
            folder_name = folder_name.split("。")[0]
            folder_name = folder_name.strip().replace("⭐", "") \
                .replace("🍌", "") \
                .replace("/", "&") \
                .replace("⏰", "") \
                .replace("🌈", "") \
                .replace("🎉", "") \
                .replace("网易", "163") \
                .replace("京东", "某东") \
                .replace("淘宝", "某宝") \
                .replace("QQ", "企鹅") \
                .replace("腾讯", "鹅厂") \
                .replace("知乎", "zhihu") \
                .replace("斗鱼", "DY") \
                .replace("百度", "某度") \
                .replace("抖音", "电音") \
                .replace("天猫", "TCat") \
                .replace("美团", "MEITUAN") \
                .replace("今日头条", "今日头疼") \
                .replace(": ", " ").replace(".", " ").replace(":", " ").replace("开源", "")
            if len(folder_name) > 150:
                folder_name = folder_name[0:150]
                pass
            file_name=""
            files = os.listdir(self.base_path) #理论就一个文件
            for file in files:
                file_ext = os.path.splitext(file)[-1]
                if file_ext is None or file_ext == "":
                    continue
                if "zip" == str(file_ext.split(".")[1]):
                    file_name=file
            if file_name=="":
                self.lable_down_number.config(text="下载文件夹不存在ZIP文件因此程序停止...")
                self.is_spider = False
                return
            srcFile = self.base_path + os.sep + file_name  # 原始zip文件
            ZipTools.extract_zip(srcFile, self.three_path, "123123")  # 解压文件
            extract_folder_name = self.three_path + os.sep + file_name.replace(".zip", "")
            target_folder_name= self.three_path + os.sep + folder_name
            os.rename(extract_folder_name, target_folder_name)
            os.remove(srcFile)  # 移除原始文件
            self.down_number = int(self.down_number) + 1
            self.lable_down_number.config(text="下载文件数量:" + str(self.down_number))
            self.lable_message.config(text="文件解压及保存成功！")
            self.is_spider = False
        except Exception as e:
            print(e)
            self.lable_message.config(text=str(e))
            self.is_spider = False
            self.txt_spider_url.SetValue("")