浅谈Dubbo框架踩坑记之优雅重启问题

cause: org.apache.dubbo.remoting.TimeoutException: Waiting server-side response timeout by scan timer. start time: 2021-09-09 11:59:56.822, end time: 2021-09-09 11:59:58.828, client elapsed: 0 ms, server elapsed: 2006 ms, timeout: 2000 ms, request: Request [id=307463, version=2.0.2, twoway=true, event=false, broken=false, data=null], channel: /XXXXXX:52149 -> /XXXXXX:20880] with root cause]

  @Override
    public void close(final int timeout) {
        startClose();
        if (timeout > 0) {
            final long max = (long) timeout;
            final long start = System.currentTimeMillis();
            if (getUrl().getParameter(Constants.CHANNEL_SEND_READONLYEVENT_KEY, true)) {
                //发送 readonly 事件报文通知 consumer 服务不可用
                sendChannelReadOnlyEvent();
            }
            while (HeaderExchangeServer.this.isRunning()
                    && System.currentTimeMillis() - start < max) {
                try {
                    Thread.sleep(10);
                } catch (InterruptedException e) {
                    logger.warn(e.getMessage(), e);
                }
            }
        }
        doClose();
        server.close(timeout);
    }

dubbo:
  application:
        shutwait: 10000 # 优雅退出等待时间，单位毫秒 默认等待 10s

 /**
     * Get the weight of the invoker's invocation which takes warmup time into account
     * if the uptime is within the warmup time, the weight will be reduce proportionally
     *
     * @param invoker    the invoker
     * @param invocation the invocation of this invoker
     * @return weight
     */
    int getWeight(Invoker<?> invoker, Invocation invocation) {
        int weight;
        URL url = invoker.getUrl();
        // Multiple registry scenario, load balance among multiple registries.
        if (REGISTRY_SERVICE_REFERENCE_PATH.equals(url.getServiceInterface())) {
            weight = url.getParameter(REGISTRY_KEY + "." + WEIGHT_KEY, DEFAULT_WEIGHT);
        } else {
            weight = url.getMethodParameter(invocation.getMethodName(), WEIGHT_KEY, DEFAULT_WEIGHT);
            if (weight > 0) {
                //获取服务启动时间 timestamp
                long timestamp = invoker.getUrl().getParameter(TIMESTAMP_KEY, 0L);
                if (timestamp > 0L) {
                    //使用当前时间减去服务提供者启动时间，计算服务提供者已运行时间 `uptime`
                    long uptime = System.currentTimeMillis() - timestamp;
                    if (uptime < 0) {
                        return 1;
                    }
                    //获取服务预热时间基数，默认是10分钟
                    int warmup = invoker.getUrl().getParameter(WARMUP_KEY, DEFAULT_WARMUP);
                    //如果服务启动时间 小于 warmup 则重新计算权重
                    if (uptime > 0 && uptime < warmup) {
                        //根据已运行时间动态计算服务预热过程的权重
                        weight = calculateWarmupWeight((int)uptime, warmup, weight);
                    }
                }
            }
        }
        return Math.max(weight, 0);
    }

 /**
     * Calculate the weight according to the uptime proportion of warmup time
     * the new weight will be within 1(inclusive) to weight(inclusive)
     *
     * @param uptime the uptime in milliseconds
     * @param warmup the warmup time in milliseconds
     * @param weight the weight of an invoker
     * @return weight which takes warmup into account
     */
    static int calculateWarmupWeight(int uptime, int warmup, int weight) {
        int ww = (int) ( uptime / ((float) warmup / weight));
        return ww < 1 ? 1 : (Math.min(ww, weight));
    }

dubbo:
    provider:
         warmup: 600000 # 单位毫秒 默认10分钟

 public synchronized void export() {
        //是否已经暴露
        if (!shouldExport()) {
            return;
        }
        if (bootstrap == null) {
            bootstrap = DubboBootstrap.getInstance();
            bootstrap.init();
        }
        checkAndUpdateSubConfigs();
        //init serviceMetadata
        serviceMetadata.setVersion(version);
        serviceMetadata.setGroup(group);
        serviceMetadata.setDefaultGroup(group);
        serviceMetadata.setServiceType(getInterfaceClass());
        serviceMetadata.setServiceInterfaceName(getInterface());
        serviceMetadata.setTarget(getRef());
        if (shouldDelay()) { //是否需要延迟暴露
            DELAY_EXPORT_EXECUTOR.schedule(this::doExport, getDelay(), TimeUnit.MILLISECONDS);
        } else {
            //真正执行服务暴露的方法
            doExport();
        }
        exported();
    }

dubbo:
    provider:
         delay: 5000 # 默认null不延迟, 单位毫秒

/XXX:57330 -> /XXXX:20880 is established., dubbo version: 2.7.7, current host: XXXX
2021-09-07 15:01:07.748 [NettyClientWorker-1-16] INFO  o.a.d.r.t.netty4.NettyClientHandler   -  [DUBBO] The connection of /XXXX:57332 -> /XXXX:20880 is established., dubbo version: 2.7.7, current host: XXXX
# 简单统计一下发现 客户端启动时建立了3600个长连接
$ less /u01/logs/order-service-api_XXX/dubbo.log  | grep NettyClientWorker- |grep  '2021-09-07 15' | wc -l
3600

private ExchangeClient[] getClients(URL url) {
        boolean useShareConnect = false;
        //获取配置连接数， 如果没有配置默认0
        int connections = url.getParameter(CONNECTIONS_KEY, 0);
        List<ReferenceCountExchangeClient> shareClients = null;
        // if not configured, connection is shared, otherwise, one connection for one service
        if (connections == 0) {
            //注意： 如果Provider 配置了connections， 就不会使用共享连接，Consumer就算配置了shareConnections也不会生效
            useShareConnect = true;
            /*
             * The xml configuration should have a higher priority than properties.
             */
            String shareConnectionsStr = url.getParameter(SHARE_CONNECTIONS_KEY, (String) null);
            connections = Integer.parseInt(StringUtils.isBlank(shareConnectionsStr) ? ConfigUtils.getProperty(SHARE_CONNECTIONS_KEY,
                    DEFAULT_SHARE_CONNECTIONS) : shareConnectionsStr);
            shareClients = getSharedClient(url, connections);
        }
        ExchangeClient[] clients = new ExchangeClient[connections];
        for (int i = 0; i < clients.length; i++) {
            if (useShareConnect) {
                clients[i] = shareClients.get(i);
            } else {
                //初始化创连接
                clients[i] = initClient(url);
            }
        }
        return clients;
    }

dubbo:
  provider:
    connections: 200

initClient(URL url) {
        // client type setting.
        String str = url.getParameter(CLIENT_KEY, url.getParameter(SERVER_KEY, DEFAULT_REMOTING_CLIENT));
        url = url.addParameter(CODEC_KEY, DubboCodec.NAME);
        // enable heartbeat by default
        url = url.addParameterIfAbsent(HEARTBEAT_KEY, String.valueOf(DEFAULT_HEARTBEAT));
        // BIO is not allowed since it has severe performance issue.
        if (str != null && str.length() > 0 && !ExtensionLoader.getExtensionLoader(Transporter.class).hasExtension(str)) {
            throw new RpcException("Unsupported client type: " + str + "," +
                    " supported client type is " + StringUtils.join(ExtensionLoader.getExtensionLoader(Transporter.class).getSupportedExtensions(), " "));
        }
        ExchangeClient client;
        try {
            // 是否配置了懒加载
            if (url.getParameter(LAZY_CONNECT_KEY, false)) {
                client = new LazyConnectExchangeClient(url, requestHandler);
            } else {
                //没有配置懒加载会初始化长连接
                client = Exchangers.connect(url, requestHandler);
            }
        } catch (RemotingException e) {
            throw new RpcException("Fail to create remoting client for service(" + url + "): " + e.getMessage(), e);
        }
        return client;
    }

dubbo:
  provider:
    connections: 2