Python的Requests来爬取今日头条的图片和文章

MONGO_HOST = 'localhost'
MONGO_PORT = 27017
MONGO_DB = 'toutiao'
MONGO_TABLE = 'toutiao'
GROUP_START = 1
GROUP_END = 20
KEYWORD = '原油'

import json
import os
from urllib.parse import urlencode
import pymongo
import requests
from multiprocessing import Pool
from requests.exceptions import ConnectionError
from hashlib import md5
from config import *
client = pymongo.MongoClient(MONGO_HOST, MONGO_PORT)
db = client[MONGO_DB]
def get_page_index(offset, keyword):
    data = {
        'autoload': 'true',
        'count': 20,
        'cur_tab': 1,
        'app_name': 'web_search',
        'format': 'search_tab',
        'keyword': keyword,
        'offset': offset,
    }
    params = urlencode(data)
    base = 'https://www.toutiao.com/api/search/content'
    url = base + '?' + params
    try:
        response = requests.get(url)
        if response.status_code == 200:
            data = json.loads(response.text)
            if data  and 'data' in data.keys():
                if data.get('data') is not None:
                    for item in data.get('data'):
                        if item is not None:
                           yield [item.get('article_url'), item.get('abstract'), item.get('large_image_url')]
    except ConnectionError:
        print('Error occurred')
        return None
def download_image(url):
    print('Downloading', url)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            save_image(response.content)
        return None
    except Exception:
        return None
def save_image(content):
    file_path = '{0}/picture/{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')
    print(file_path)
    if not os.path.exists(file_path):
        with open(file_path, 'wb') as f:
            f.write(content)
            f.close()
def save_to_mongo(result):
    if db[MONGO_TABLE].insert_one(result):
        print('Successfully Saved to Mongo', result)
        return True
    return False
def main(offset):
    items = get_page_index(offset, KEYWORD)
    for item in items:
            if (item[2] is not None) and len(item[2])!=0:
                download_image(item[2])
            if (item[0] is not None and len(item[0]) != 0)\
                    or  (item[1] is not None and len(item[1]) != 0)\
                    or  (item[2] is not None and len(item[2]) != 0):
                json = {
                    'article_url': item[0],
                    'abstract': item[1],
                    'large_image_url': item[2]
                }
                save_to_mongo(json)
if __name__ == '__main__':
    pool = Pool()
    groups = ([x * 20 for x in range(GROUP_START, GROUP_END + 1)])
    pool.map(main, groups)
    pool.close()
    pool.join()

pip install virtualenv

~$virtualenv TestEnv
New python executable in ~/TestEnv/bin/python
Installing setuptools, pip, wheel...done.

语法：virtualenv --system-site-packages TestEnv

语法：virtualenv --no-site-packages ubuntu_env