php使用redis异步队列爬取网站图片的教程(1)

<?php

/**

 * Created by PhpStorm.

 * User: tioncico

 * Date: 2018/2/26 0026

 * Time: 21:34

 */

namespace Queue;

class MyCurl

{

    private static $url    = ''; // 访问的url

    private static $oriUrl = ''; // referer url

    private static $data   = array(); // 可能发出的数据 post,put

    private static $method; // 访问方式，默认是GET请求

    public static function send($url, $data = array(), $method = 'get')

    {

        if (!$url) exit('url can not be null');

        self::$url    = $url;

        self::$method = $method;

        $urlArr       = parse_url($url);

        self::$oriUrl = $urlArr['scheme'] . '://' . $urlArr['host'];

        self::$data   = $data;

        if (!in_array(

            self::$method,

            array(

                'get',

                'post',

                'put',

                'delete'

            )

        )

        ) {

            exit('error request method type!');

        }

        $func = self::$method . 'Request';

        return self::$func(self::$url);

    }

    /**

     * 基础发起curl请求函数

     * @param int $is_post 是否是post请求

     */

    private static function doRequest($is_post = 0)

    {

        $ch = curl_init();//初始化curl

        curl_setopt($ch, CURLOPT_URL, self::$url);//抓取指定网页

        curl_setopt($ch, CURLOPT_AUTOREFERER, true);

        // 来源一定要设置成来自本站

        curl_setopt($ch, CURLOPT_REFERER, self::$oriUrl);

        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//要求结果为字符串且输出到屏幕上

        if ($is_post == 1) curl_setopt($ch, CURLOPT_POST, $is_post);//post提交方式

        if (!empty(self::$data)) {

            self::$data = self::dealPostData(self::$data);

            curl_setopt($ch, CURLOPT_POSTFIELDS, self::$data);

        }

        $data = curl_exec($ch);//运行curl

        curl_close($ch);

        return $data;

    }

    /**

     * 发起get请求

     */

    public static function getRequest()

    {

        return self::doRequest(0);

    }

    /**

     * 发起post请求

     */

    public static function postRequest()

    {

        return self::doRequest(1);

    }

    /**

     * 处理发起非get请求的传输数据

     *

     * @param array $postData

     */

    public static function dealPostData($postData)

    {

        if (!is_array($postData)) exit('post data should be array');

        foreach ($postData as $k => $v) {

            $o .= "$k=" . urlencode($v) . "&";

        }

        $postData = substr($o, 0, -1);

        return $postData;

    }

    /**

     * 发起put请求

     */

    public static function putRequest($param)

    {

        return self::doRequest(2);

    }

    /**

     * 发起delete请求

     */

    public static function deleteRequest($param)

    {

        return self::doRequest(3);

    }

}

<?php

/**

 * Created by PhpStorm.

 * User: tioncico

 * Date: 2018/2/28 0028

 * Time: 19:54

 */

namespace Queue;

use QL\QueryList;

class Down

{

    private static $instance;

    public function __construct() {

    }

    static function getInstance(){

        if(is_object(self::$instance)){

            return self::$instance;

        }else{

            self::$instance = new Down();

            return self::$instance;

        }

    }

    public static function add_img($str){//这里进来的是详情,每个详情有n页,每页一张高清大图

        $id = substr($str,strripos($str,'/')+1);//截取出id

        $res   = \Queue\MyCurl::send($str, array('ip'=>'127.0.0.1'), 'get');

//        var_dump($res);

        $rules = array(

            //采集id为one这个元素里面的纯文本内容

            'page' => array(

                '.pagenavi span',

                'html'

            ),

        );

        $hj    = QueryList::Query($res, $rules);

        $data = $hj->getData(function ($x) {//总页数

            return $x;

        });

        //获取到了页面元素

        $count = $data[count($data)-2]['page'];//这个是获取倒数第2个,倒数第2个是总页数

        echo '共有'.$count."张图\n";

        for($i=1;$i<=$count;$i++){

            $url = $str.'/'.$i;

            self::add_img_file($id,$url);//开始保存图片

            echo "{$i}/$count\n";

        }

    }

    public static function add_img_file($id,$url){

        $res   = \Queue\MyCurl::send($url, array('ip'=>'127.0.0.1','Referer'=>'http://www.mzitu.com/116663/2'), 'get');

        $rules = array(

            //采集id为one这个元素里面的纯文本内容

            'img_url' => array(

                '.main-image img',

                'src'

            ),

        );//获取到了高清图片链接

        $hj    = QueryList::Query($res, $rules);

        $data = $hj->getData(function ($x) {//总页数

            return $x;

        });

        $img_url = $data[0]['img_url'];

        $path = BASE_DIR.'/img/'.$id.'/';

        @mkdir($path,0777);//新建文件夹存取图片TODO缺少存进数据库

        chmod($path,0777);

        $file_path = $path.substr($img_url,strripos($img_url,'/')+1);

//        var_dump($file_path);

        $data = MyCurl::send($img_url);

        $write = @fopen($file_path, "w+");

        fwrite($write, $data);

        fclose($write);//存取图片

    }

}