python爬虫爬取图片-阿里云开发者社区

开发者社区> 大数据> 正文
登录阅读全文

python爬虫爬取图片

简介: 爬取 http://www.xiaohuar.com/ 美女校花 图片的爬虫 # -*- coding:utf-8 -*- import os import requests # from PIL import Image from lxml import etree class Spider(object): """ crawl image """ def

爬取 http://www.xiaohuar.com/ 美女校花 图片的爬虫


# -*- coding:utf-8 -*-

import os
import requests
# from PIL import Image
from lxml import etree


class Spider(object):
    """ crawl image """
    def __init__(self):
        self.index = 0
        self.url = "http://www.xiaohuar.com"
        self.proxies = {"http": "http://172.17.18.80:8080", "https": "https://172.17.18.80:8080"}
        pass

    def download_image(self, image_url):
        real_url = self.url + image_url
        print "downloading the {0} image".format(self.index)
        with open("{0}.jpg".format(self.index), 'wb') as f:
            self.index += 1
            f.write(requests.get(real_url, proxies=self.proxies).content)
            pass
        pass

    def start_crawl(self):
        start_url = "http://www.xiaohuar.com/hua/"
        r = requests.get(start_url, proxies=self.proxies)
        if r.status_code == 200:
            temp = r.content.decode("gbk")
            html = etree.HTML(temp)
            links = html.xpath('//div[@class="item_t"]//img/@src')
            map(self.download_image, links)
            # next_page_url = html.xpath('//div[@class="page_num"]//a/text()')
            # print next_page_url[-1]
            # print next_page_url[-2]
            # print next_page_url[-3]
            next_page_url = html.xpath(u'//div[@class="page_num"]//a[contains(text(),"下一页")]/@href')
            page_num = 2
            while next_page_url:
                print "download {0} page images".format(page_num)
                r_next = requests.get(next_page_url[0], proxies=self.proxies)
                if r_next.status_code == 200:
                    html = etree.HTML(r_next.content.decode("gbk"))
                    links = html.xpath('//div[@class="item_t"]//img/@src')
                    map(self.download_image, links)
                    try:
                        next_page_url = html.xpath(u'//div[@class="page_num"]//a[contains(text(),"下一页")]/@href')
                    except BaseException as e:
                        next_page_url = None
                        # print e
                    page_num += 1
                    pass
                else:
                    print "response status code : {0}".format(r_next.status_code)
                pass
        else:
            print "response status code : {0}".format(r.status_code)
        pass


if __name__ == "__main__":
    t = Spider()
    t.start_crawl()
    pause = raw_input("press any key to continue")
    pass






版权声明:本文内容由阿里云实名注册用户自发贡献,版权归原作者所有,阿里云开发者社区不拥有其著作权,亦不承担相应法律责任。具体规则请查看《阿里云开发者社区用户服务协议》和《阿里云开发者社区知识产权保护指引》。如果您发现本社区中有涉嫌抄袭的内容,填写侵权投诉表单进行举报,一经查实,本社区将立刻删除涉嫌侵权内容。

分享:
大数据
使用钉钉扫一扫加入圈子
+ 订阅

大数据计算实践乐园,近距离学习前沿技术

其他文章