背景信息
主要功能
Proxyee 是一个 JAVA 编写的 HTTP 代理服务器类库，支持 HTTP、HTTPS、Websocket 协议，并且支持 MITM(中间人攻击)，可以对 HTTP、HTTPS 协议的报文进行捕获和篡改。
获取地址
https://github.com/monkeyWie/proxyee
使用方式
在pom中加入如下依赖
<dependency>
<groupId>com.github.monkeywie</groupId>
<artifactId>proxyee</artifactId>
<version>1.5.4</version>
</dependency>源码阅读
代码结构
java模块
一个很清晰简单的单模块maven项目
test模块
这里面包含了十余种示例来示范如何使用proxyee
pom依赖
<dependencies>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler-proxy</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>1.58</version>
</dependency>
</dependencies>依赖内容相当简洁，只有netty和bouncycastle
引入netty模块主要是用于代理服务器的实现
bouncycastle是一种用于 Java 平台的开放源码的轻量级密码术包；它支持大量的密码术算法，并提供JCE 1.2.1的实现。
启动
public static void main(String[] args) throws Exception {
HttpProxyServerConfig config = new HttpProxyServerConfig();
config.setHandleSsl(true);
new HttpProxyServer()
.serverConfig(config)
.proxyInterceptInitializer(new HttpProxyInterceptInitializer() {
@Override
public void init(HttpProxyInterceptPipeline pipeline) {
pipeline.addLast(new CertDownIntercept()); //处理证书下载
pipeline.addLast(new HttpProxyIntercept() {
@Override
public void beforeRequest(Channel clientChannel, HttpRequest httpRequest,
HttpProxyInterceptPipeline pipeline) throws Exception {
//替换UA，伪装成手机浏览器
/*httpRequest.headers().set(HttpHeaderNames.USER_AGENT,
"Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1");*/
//转到下一个拦截器处理
pipeline.beforeRequest(clientChannel, httpRequest);
}
@Override
public void afterResponse(Channel clientChannel, Channel proxyChannel,
HttpResponse httpResponse, HttpProxyInterceptPipeline pipeline) throws Exception {
//拦截响应，添加一个响应头
httpResponse.headers().add("intercept", "test");
//转到下一个拦截器处理
pipeline.afterResponse(clientChannel, proxyChannel, httpResponse);
}
});
}
})
.httpProxyExceptionHandle(new HttpProxyExceptionHandle() {
@Override
public void beforeCatch(Channel clientChannel, Throwable cause) throws Exception {
cause.printStackTrace();
}
@Override
public void afterCatch(Channel clientChannel, Channel proxyChannel, Throwable cause)
throws Exception {
cause.printStackTrace();
}
})
.start(9999);
}这是代码中Test模块的InterceptHttpProxyServer类，这个类提供了调用的示例，我们以此来研究服务启动的过程。
首先我们来看start方法
public void start(int port) {
start(null, port);
}
public void start(String ip, int port) {
try {
ChannelFuture channelFuture = doBind(ip, port);
CountDownLatch latch = new CountDownLatch(1);
channelFuture.addListener(future -> {
if (future.cause() != null) {
httpProxyExceptionHandle.startCatch(future.cause());
}
latch.countDown();
});
latch.await();
channelFuture.channel().closeFuture().sync();
} catch (Exception e) {
httpProxyExceptionHandle.startCatch(e);
} finally {
close();
}
}我们来看doBind方法中做了些什么
private ChannelFuture doBind(String ip, int port) {
init();
bossGroup = new NioEventLoopGroup(serverConfig.getBossGroupThreads());
workerGroup = new NioEventLoopGroup(serverConfig.getWorkerGroupThreads());
ServerBootstrap bootstrap = new ServerBootstrap();
bootstrap.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
// .option(ChannelOption.SO_BACKLOG, 100)
.handler(new LoggingHandler(LogLevel.DEBUG))
.childHandler(new ChannelInitializer<Channel>() {
@Override
protected void initChannel(Channel ch) throws Exception {
ch.pipeline().addLast("httpCodec", new HttpServerCodec());
ch.pipeline().addLast("serverHandle",
new HttpProxyServerHandler(serverConfig, proxyInterceptInitializer, proxyConfig,
httpProxyExceptionHandle));
}
});
return ip == null ? bootstrap.bind(port) : bootstrap.bind(ip, port);
}首先调用了init()方法，
private void init() {
if (serverConfig == null) {
serverConfig = new HttpProxyServerConfig();
}
serverConfig.setProxyLoopGroup(new NioEventLoopGroup(serverConfig.getProxyGroupThreads()));
if (serverConfig.isHandleSsl()) {
try {
serverConfig.setClientSslCtx(
SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE)
.build());
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
X509Certificate caCert;
PrivateKey caPriKey;
if (caCertFactory == null) {
caCert = CertUtil.loadCert(classLoader.getResourceAsStream("ca.crt"));
caPriKey = CertUtil.loadPriKey(classLoader.getResourceAsStream("ca_private.der"));
} else {
caCert = caCertFactory.getCACert();
caPriKey = caCertFactory.getCAPriKey();
}
//读取CA证书使用者信息
serverConfig.setIssuer(CertUtil.getSubject(caCert));
//读取CA证书有效时段(server证书有效期超出CA证书的，在手机上会提示证书不安全)
serverConfig.setCaNotBefore(caCert.getNotBefore());
serverConfig.setCaNotAfter(caCert.getNotAfter());
//CA私钥用于给动态生成的网站SSL证书签证
serverConfig.setCaPriKey(caPriKey);
//生产一对随机公私钥用于网站SSL证书动态创建
KeyPair keyPair = CertUtil.genKeyPair();
serverConfig.setServerPriKey(keyPair.getPrivate());
serverConfig.setServerPubKey(keyPair.getPublic());
} catch (Exception e) {
serverConfig.setHandleSsl(false);
log.warn("SSL init fail,cause:" + e.getMessage());
}
}
if (proxyInterceptInitializer == null) {
proxyInterceptInitializer = new HttpProxyInterceptInitializer();
}
if (httpProxyExceptionHandle == null) {
httpProxyExceptionHandle = new HttpProxyExceptionHandle();
}
}在这里主要是进行了SSL证书相关的操作和初始化，对proxyInterceptInitializer和httpProxyExceptionHandle进行了判空处理，这里在此示例的main方法中已经进行了初始化。
public HttpProxyServer proxyInterceptInitializer(
HttpProxyInterceptInitializer proxyInterceptInitializer) {
this.proxyInterceptInitializer = proxyInterceptInitializer;
return this;
}
public HttpProxyServer httpProxyExceptionHandle(
HttpProxyExceptionHandle httpProxyExceptionHandle) {
this.httpProxyExceptionHandle = httpProxyExceptionHandle;
return this;
}接着我们来看一下对于netty的操作
bossGroup = new NioEventLoopGroup(serverConfig.getBossGroupThreads());
workerGroup = new NioEventLoopGroup(serverConfig.getWorkerGroupThreads());那么NioEventLoopGroup是何方神圣呢？
从这个类的名字来看，NioEventLoopGroup，它是一个无限循环（Loop），在循环中不断处理接收到的事件（Event），从设计上来看，EventLoop采用了一种协同设计，它建立在两个基本的API之上：Concurrent和Channel，也就是并发和网络。并发是因为它采用了线程池来管理大量的任务，并且这些任务可以并发的执行。其继承了EventExecutor接口，而EventExecutor就是一个事件的执行器。另外为了与Channel的事件进行交互，EventLoop继承了EventLoopGroup接口。
一个Netty服务端启动时，通常会有两个NioEventLoopGroup：一个是监听线程组，主要是监听客户端请求，另一个是工作线程组，主要是处理与客户端的数据
在这里，bossGroup专门用于监听客户端的连接事件，连接成功以后，workerGroup则会负责连接中发生的IO事件
最后用netty中的bootStrap启动了netty的服务端，并且建立了一个channel。
@Override
protected void initChannel(Channel ch) throws Exception {
ch.pipeline().addLast("httpCodec", new HttpServerCodec());
ch.pipeline().addLast("serverHandle",
new HttpProxyServerHandler(serverConfig, proxyInterceptInitializer, proxyConfig,
httpProxyExceptionHandle));
}在这里addLast中添加的是作者自定义的Netty IO handler。
第一步，添加了一个httpCodec的handler。这是Netty自带的提供http服务的handler。它是一个“全双工”handler，也就是既是一个InHandler，也是一个outHandler，负责把接收到的二进制请求反序列化成Http Request对象，并序列化Http Response。
然后，添加了一个“serverHandler”，类型是HttpProxyServerHandler，它也继承了Netty的ChannelInboundHandlerAdapter，所以它是一个处理IO请求的“拦截器”。
代理
我们主要来看HttpProxyServerHandler 这个类
他继承了ChannelInboundHandlerAdapter
我们来看作者override的方法
@Override
public void channelRead(final ChannelHandlerContext ctx, final Object msg) throws Exception {
if (msg instanceof HttpRequest) {
HttpRequest request = (HttpRequest) msg;
// 第一次建立连接取host和端口号和处理代理握手
if (status == 0) {
requestProto = ProtoUtil.getRequestProto(request);
if (requestProto == null) { // bad request
ctx.channel().close();
return;
}
// 首次连接处理
if (serverConfig.getHttpProxyAcceptHandler() != null
&& !serverConfig.getHttpProxyAcceptHandler().onAccept(request, ctx.channel())) {
status = 2;
ctx.channel().close();
return;
}
// 代理身份验证
if (!authenticate(ctx, request)) {
status = 2;
ctx.channel().close();
return;
}
status = 1;
if ("CONNECT".equalsIgnoreCase(request.method().name())) {// 建立代理握手
status = 2;
HttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpProxyServer.SUCCESS);
ctx.writeAndFlush(response);
ctx.channel().pipeline().remove("httpCodec");
// fix issue #42
ReferenceCountUtil.release(msg);
return;
}
}
interceptPipeline = buildPipeline();
interceptPipeline.setRequestProto(requestProto.copy());
// fix issue #27
if (request.uri().indexOf("/") != 0) {
URL url = new URL(request.uri());
request.setUri(url.getFile());
}
interceptPipeline.beforeRequest(ctx.channel(), request);
} else if (msg instanceof HttpContent) {
if (status != 2) {
interceptPipeline.beforeRequest(ctx.channel(), (HttpContent) msg);
} else {
ReferenceCountUtil.release(msg);
status = 1;
}
} else { // ssl和websocket的握手处理
if (serverConfig.isHandleSsl()) {
ByteBuf byteBuf = (ByteBuf) msg;
if (byteBuf.getByte(0) == 22) {// ssl握手
requestProto.setSsl(true);
int port = ((InetSocketAddress) ctx.channel().localAddress()).getPort();
SslContext sslCtx = SslContextBuilder
.forServer(serverConfig.getServerPriKey(), CertPool.getCert(port, requestProto.getHost(), serverConfig)).build();
ctx.pipeline().addFirst("httpCodec", new HttpServerCodec());
ctx.pipeline().addFirst("sslHandle", sslCtx.newHandler(ctx.alloc()));
// 重新过一遍pipeline，拿到解密后的的http报文
ctx.pipeline().fireChannelRead(msg);
return;
}
}
handleProxyData(ctx.channel(), msg, false);
}
}在这里进行了一个判断msg是HttpRequest还是HttpContent或者都不是的情况。
HttpRequest时：
首先进行了是否是首次链接的判断，并进行链接和握手的处理包括身份的认证。然后初始化proxyee自定义的pipeline，最后调用beforeRequest（HttpRequest msg）方法
我们来看看这个方法
public void beforeRequest(Channel clientChannel, HttpRequest httpRequest) throws Exception {
this.httpRequest = httpRequest;
if (this.posBeforeHead < intercepts.size()) {
HttpProxyIntercept intercept = intercepts.get(this.posBeforeHead++);
intercept.beforeRequest(clientChannel, this.httpRequest, this);
}
this.posBeforeHead = 0;
}这里采用递归的方法去把intercepts中存储的所有intercept拿出来并执行
HttpContent时：
调用了beforeRequest（HttpContent msg）方法
public void beforeRequest(Channel clientChannel, HttpContent httpContent) throws Exception {
if (this.posBeforeContent < intercepts.size()) {
HttpProxyIntercept intercept = intercepts.get(this.posBeforeContent++);
intercept.beforeRequest(clientChannel, httpContent, this);
}
this.posBeforeContent = 0;
}这里与上面beforeRequest(HttpRequest msg)没有太多区别
两者都不是时：
首先进行了ssl和websocket的握手处理，然后进行了proxyData的处理
我们再来看下HttpProxyInterceptPipeline这个作者自定义的pipeline，实现了Iterable接口，用于遍历HttpProxyIntercept。
@Override
public Iterator<HttpProxyIntercept> iterator() {
return intercepts.iterator();
}我们再来看一下如何建立pipeline的
private HttpProxyInterceptPipeline buildPipeline() {
HttpProxyInterceptPipeline interceptPipeline = new HttpProxyInterceptPipeline(new HttpProxyIntercept() {
@Override
public void beforeRequest(Channel clientChannel, HttpRequest httpRequest, HttpProxyInterceptPipeline pipeline)
throws Exception {
handleProxyData(clientChannel, httpRequest, true);
}
@Override
public void beforeRequest(Channel clientChannel, HttpContent httpContent, HttpProxyInterceptPipeline pipeline)
throws Exception {
handleProxyData(clientChannel, httpContent, true);
}
@Override
public void afterResponse(Channel clientChannel, Channel proxyChannel, HttpResponse httpResponse,
HttpProxyInterceptPipeline pipeline) throws Exception {
clientChannel.writeAndFlush(httpResponse);
if (HttpHeaderValues.WEBSOCKET.toString().equals(httpResponse.headers().get(HttpHeaderNames.UPGRADE))) {
// websocket转发原始报文
proxyChannel.pipeline().remove("httpCodec");
clientChannel.pipeline().remove("httpCodec");
}
}
@Override
public void afterResponse(Channel clientChannel, Channel proxyChannel, HttpContent httpContent,
HttpProxyInterceptPipeline pipeline) throws Exception {
clientChannel.writeAndFlush(httpContent);
}
});
interceptInitializer.init(interceptPipeline);
return interceptPipeline;
}这里加入了一个默认的HttpProxyIntercept，置于pipeline的队尾
用户在main方法中自定义的intercept实际上都在这个pipeline中，从第一个开始执行到默认的这个Intercept结束
handleProxyData中又干了写什么呢
private void handleProxyData(Channel channel, Object msg, boolean isHttp) throws Exception {
if (cf == null) {
// connection异常 还有HttpContent进来，不转发
if (isHttp && !(msg instanceof HttpRequest)) {
return;
}
if (interceptPipeline == null) {
interceptPipeline = buildOnlyConnectPipeline();
interceptPipeline.setRequestProto(requestProto.copy());
}
interceptPipeline.beforeConnect(channel);
// by default we use the proxy config set in the pipeline
ProxyHandler proxyHandler = ProxyHandleFactory.build(interceptPipeline.getProxyConfig() == null ?
proxyConfig : interceptPipeline.getProxyConfig());
RequestProto requestProto = interceptPipeline.getRequestProto();
if (isHttp) {
HttpRequest httpRequest = (HttpRequest) msg;
// 检查requestProto是否有修改
RequestProto newRP = ProtoUtil.getRequestProto(httpRequest);
if (!newRP.equals(requestProto)) {
// 更新Host请求头
if ((requestProto.getSsl() && requestProto.getPort() == 443)
|| (!requestProto.getSsl() && requestProto.getPort() == 80)) {
httpRequest.headers().set(HttpHeaderNames.HOST, requestProto.getHost());
} else {
httpRequest.headers().set(HttpHeaderNames.HOST, requestProto.getHost() + ":" + requestProto.getPort());
}
}
}
/*
* 添加SSL client hello的Server Name Indication extension(SNI扩展) 有些服务器对于client
* hello不带SNI扩展时会直接返回Received fatal alert: handshake_failure(握手错误)
* 例如：https://cdn.mdn.mozilla.net/static/img/favicon32.7f3da72dcea1.png
*/
ChannelInitializer channelInitializer = isHttp ? new HttpProxyInitializer(channel, requestProto, proxyHandler)
: new TunnelProxyInitializer(channel, proxyHandler);
Bootstrap bootstrap = new Bootstrap();
bootstrap.group(serverConfig.getProxyLoopGroup()) // 注册线程池
.channel(NioSocketChannel.class) // 使用NioSocketChannel来作为连接用的channel类
.handler(channelInitializer);
if (proxyHandler != null) {
// 代理服务器解析DNS和连接
bootstrap.resolver(NoopAddressResolverGroup.INSTANCE);
} else {
bootstrap.resolver(serverConfig.resolver());
}
requestList = new LinkedList();
cf = bootstrap.connect(requestProto.getHost(), requestProto.getPort());
cf.addListener((ChannelFutureListener) future -> {
if (future.isSuccess()) {
future.channel().writeAndFlush(msg);
synchronized (requestList) {
requestList.forEach(obj -> future.channel().writeAndFlush(obj));
requestList.clear();
isConnect = true;
}
} else {
requestList.forEach(obj -> ReferenceCountUtil.release(obj));
requestList.clear();
getExceptionHandle().beforeCatch(channel, future.cause());
future.channel().close();
channel.close();
}
});
} else {
synchronized (requestList) {
if (isConnect) {
cf.channel().writeAndFlush(msg);
} else {
requestList.add(msg);
}
}
}
}实际上是对数据的转发
再来看一下ProxyHandleFactory中的build方法
public static ProxyHandler build(ProxyConfig config) {
ProxyHandler proxyHandler = null;
if (config != null) {
boolean isAuth = config.getUser() != null && config.getPwd() != null;
InetSocketAddress inetSocketAddress = new InetSocketAddress(config.getHost(),
config.getPort());
switch (config.getProxyType()) {
case HTTP:
if (isAuth) {
proxyHandler = new HttpProxyHandler(inetSocketAddress,
config.getUser(), config.getPwd());
} else {
proxyHandler = new HttpProxyHandler(inetSocketAddress);
}
break;
case SOCKS4:
proxyHandler = new Socks4ProxyHandler(inetSocketAddress);
break;
case SOCKS5:
if (isAuth) {
proxyHandler = new Socks5ProxyHandler(inetSocketAddress,
config.getUser(), config.getPwd());
} else {
proxyHandler = new Socks5ProxyHandler(inetSocketAddress);
}
break;
}
}
return proxyHandler;
}在这里会根据类型的不同会调用不同的初始化方法来得到proxyHandler
调用自定义intercept并且转发数据的操作流程就是这样，更深层次的关于netty的内容，nio的相关操作再次就不做深述。
总结
总的来说，proxyee是以netty为核心实现的一个较为轻量化的代理服务器，整体代码量和依赖都很少，在源码阅读的过程中能够不断加深对于netty的理解，从这种角度来说是一个很好的学习netty的项目。同时因为他可以对 HTTP、HTTPS 协议的报文进行捕获和篡改，我们可以以此来封装一层实现接口调用时入参、返回的统一记录，白名单的校验。
