OkHttp的特性优点及爬虫示例

// 导入所需的库importcom.squareup.okhttp.*;
importorg.jsoup.Jsoup;
importorg.jsoup.nodes.Document;
importorg.jsoup.nodes.Element;
importorg.jsoup.select.Elements;
importjava.io.IOException;
importjava.util.ArrayList;
importjava.util.List;
importjava.util.concurrent.CountDownLatch;
// 定义一个景点类，用于存储景点的信息classScenicSpot {
privateStringname; // 景点名称privateStringurl; // 景点链接privateStringintro; // 景点简介privateList<String>comments; // 景点评论publicScenicSpot(Stringname, Stringurl) {
this.name=name;
this.url=url;
this.intro="";
this.comments=newArrayList<>();
    }
publicStringgetName() {
returnname;
    }
publicStringgetUrl() {
returnurl;
    }
publicStringgetIntro() {
returnintro;
    }
publicvoidsetIntro(Stringintro) {
this.intro=intro;
    }
publicList<String>getComments() {
returncomments;
    }
publicvoidaddComment(Stringcomment) {
this.comments.add(comment);
    }
@OverridepublicStringtoString() {
return"ScenicSpot{"+"name='"+name+'\''+", url='"+url+'\''+", intro='"+intro+'\''+", comments="+comments+'}';
    }
}
// 定义一个回调接口，用于异步处理响应interfaceCallbackHandler {
voidhandle(Responseresponse) throwsIOException;
}
// 定义一个爬虫类，用于采集大众点评的景点信息classDianpingCrawler {
// 大众点评的网址privatestaticfinalStringBASE_URL="http://www.dianping.com";     
//亿牛云动态转发代理IP 爬虫加强版 代理服务地址privatestaticfinalStringPROXY_HOST="www.16yun.cn"; 
//亿牛云动态转发代理IP 爬虫加强版 代理端口号privatestaticfinalintPROXY_PORT=8080; 
//亿牛云动态转发代理IP 爬虫加强版 代理用户名privatestaticfinalStringPROXY_USERNAME="16YUN"; 
//亿牛云动态转发代理IP 爬虫加强版 代理密码privatestaticfinalStringPROXY_PASSWORD="16IP";
privateOkHttpClientclient; // OkHttp客户端对象publicDianpingCrawler() {
// 创建一个OkHttpClient对象，并配置代理和认证信息client=newOkHttpClient();
Proxyproxy=newProxy(Proxy.Type.HTTP, newInetSocketAddress(PROXY_HOST, PROXY_PORT));
Authenticatorauthenticator=newAuthenticator() {
@OverridepublicRequestauthenticate(Proxyproxy, Responseresponse) throwsIOException {
Stringcredential=Credentials.basic(PROXY_USERNAME, PROXY_PASSWORD);
returnresponse.request().newBuilder()
                        .header("Proxy-Authorization", credential)
                        .build();
            }
@OverridepublicRequestauthenticateProxy(Proxyproxy, Responseresponse) throwsIOException {
returnnull;
            }
        };
client.setProxy(proxy);
client.setAuthenticator(authenticator);
    }
// 发送一个GET请求，并使用回调接口处理响应publicvoidget(Stringurl, CallbackHandlercallbackHandler) throwsIOException {
Requestrequest=newRequest.Builder()
                .url(url)
                .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
                .build();
client.newCall(request).enqueue(newCallback() {
@OverridepublicvoidonFailure(Requestrequest, IOExceptione) {
e.printStackTrace();
            }
@OverridepublicvoidonResponse(Responseresponse) throwsIOException {
callbackHandler.handle(response);
            }
        });
    }
// 根据城市和关键词获取景点列表的网址publicStringgetScenicListUrl(Stringcity, Stringkeyword) {
returnBASE_URL+"/search/keyword/"+city+"/0_"+keyword;
    }
// 根据景点列表的网址获取景点对象的列表publicList<ScenicSpot>getScenicSpots(StringscenicListUrl) throwsIOException {
List<ScenicSpot>scenicSpots=newArrayList<>();
// 发送一个GET请求，获取景点列表的网页内容get(scenicListUrl, response-> {
// 解析网页内容，提取景点的名称和链接Documentdocument=Jsoup.parse(response.body().string());
Elementselements=document.select(".shop-list .shop-name");
for (Elementelement : elements) {
Stringname=element.text();
Stringurl=BASE_URL+element.attr("href");
// 创建一个景点对象，并添加到列表中ScenicSpotscenicSpot=newScenicSpot(name, url);
scenicSpots.add(scenicSpot);
            }
        });
returnscenicSpots;
    }
// 根据景点对象获取景点的简介和评论publicvoidgetScenicInfo(ScenicSpotscenicSpot) throwsIOException {
// 发送一个GET请求，获取景点的网页内容get(scenicSpot.getUrl(), response-> {
// 解析网页内容，提取景点的简介和评论Documentdocument=Jsoup.parse(response.body().string());
Stringintro=document.select(".shop-intro").text();
Elementselements=document.select(".comment-list .content");
List<String>comments=newArrayList<>();
for (Elementelement : elements) {
Stringcomment=element.text();
comments.add(comment);
            }
// 设置景点对象的简介和评论scenicSpot.setIntro(intro);
scenicSpot.getComments().addAll(comments);
        });
    }
// 根据城市和关键词获取附近TOP20的景点介绍以及每个景点TOP的评价publicList<ScenicSpot>getTopScenicSpots(Stringcity, Stringkeyword) throwsIOException, InterruptedException {
// 获取景点列表的网址StringscenicListUrl=getScenicListUrl(city, keyword);
// 获取景点对象的列表List<ScenicSpot>scenicSpots=getScenicSpots(scenicListUrl);
// 创建一个倒计时锁，用于等待所有异步请求完成CountDownLatchcountDownLatch=newCountDownLatch(scenicSpots.size());
// 遍历每个景点对象，获取其简介和评论，并在完成后减少倒计时锁的计数for (ScenicSpotscenicSpot : scenicSpots) {
getScenicInfo(scenicSpot);
countDownLatch.countDown();
        }
// 等待所有异步请求完成countDownLatch.await();
// 返回前20个景点对象的列表returnscenicSpots.subList(0, 20);
    }
// 测试方法，打印结果publicstaticvoidmain(String[] args) throwsIOException, InterruptedException {
DianpingCrawlercrawler=newDianpingCrawler();
List<ScenicSpot>scenicSpots=crawler.getTopScenicSpots("1", "景点");
for (ScenicSpotscenicSpot : scenicSpots) {
System.out.println(scenicSpot);
        }
    }
}