街拍图片网站抓取代码记录

# -*- coding:utf-8 -*-


import requests
from urllib  import urlencode
import urllib
from requests.exceptions import RequestException
import re
import os




def get_info(offset,keyword):
    data =  {'offset':offset,
             'format':'json',
             'keyword':keyword,
             'autoload':'true',
             'count':'20',
             'cur_tab':'1'}
    url = 'http://www.toutiao.com/search_content/?'+urlencode(data)
    try:
        response=requests.get(url)
        if response.status_code == 200:
           json=response.json()
           for item in json.get('data'):
                articl_url=item.get('article_url')
                html=requests.get(articl_url).text
                reg=r'img src="(.*?)"'
                imag_url=re.findall(reg,html)
                if len(imag_url) != 0:
                    for url in imag_url:
                        global num
                        urllib.urlretrieve(url, 'jiepai/%d.jpg' %num)
                        print "已下载第%d 美照" %num
                        num += 1
                else:
                    continue
    except RequestException:
        print 'error'
num=1
if not  os.path.exists('jiepai'):
    os.mkdir('jiepai')
for i in range(0,1000,20):
    get_info(i,'街拍')