Hbase java client 压测报错-问答-阿里云开发者社区-阿里云

开发者社区> hbase小能手> 正文

Hbase java client 压测报错

2018-11-06 17:39:41 3544 2

使用apache 压测工具,2000并发时出现如下错误:
2018-10-12 16:09:35.527 WARN org.apache.hadoop.hbase.client.ScannerCallable Line:367 - Ignore, probably already closed
org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: Call queue is full on /10.x'x.xxxx.xx:16020, too many items queued ?

    at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1234)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:223)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:328)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:32831)
    at org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:362)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:197)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:145)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:210)
    at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:327)
    at org.apache.hadoop.hbase.client.ClientScanner.close(ClientScanner.java:748)
    at org.apache.hadoop.hbase.MetaTableAccessor.fullScan(MetaTableAccessor.java:610)
    at org.apache.hadoop.hbase.MetaTableAccessor.tableExists(MetaTableAccessor.java:366)
    at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:403)
    at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:413)
    at com.asiainfo.tag.utils.HBaseUtils.tableExists(HBaseUtils.java:65)
    at com.asiainfo.tag.service.impl.CustGroupServiceImpl.checkTagInGroup(CustGroupServiceImpl.java:46)
    at com.asiainfo.tag.service.impl.ServiceCustGroupProxy.checkTagInGroup(ServiceCustGroupProxy.java:30)
    at com.asiainfo.service.CustGroupCheckImpl.checkExistsNo(CustGroupCheckImpl.java:30)
    at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:497)
    at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179)
    at org.apache.cxf.jaxws.JAXWSMethodInvoker.performInvocation(JAXWSMethodInvoker.java:66)
    at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96)
    at org.apache.cxf.jaxws.AbstractJAXWSMethodInvoker.invoke(AbstractJAXWSMethodInvoker.java:232)
    at org.apache.cxf.jaxws.JAXWSMethodInvoker.invoke(JAXWSMethodInvoker.java:85)
    at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:74)
    at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at org.apache.cxf.interceptor.ServiceInvokerInterceptor$2.run(ServiceInvokerInterceptor.java:126)
    at org.apache.cxf.workqueue.SynchronousExecutor.execute(SynchronousExecutor.java:37)
    at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:131)
    at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308)
    at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
    at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:267)
    at org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:234)
    at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:208)
    at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:160)
    at org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:216)
    at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:301)
    at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:220)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:661)
    at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:276)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:108)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:81)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199)
    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:478)
    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81)
    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342)
    at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:803)
    at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
    at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1459)
    at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
    at java.lang.Thread.run(Thread.java:745)

===2018-10-12 16:09:35.528 WARN com.asiainfo.tag.service.impl.ServiceCustGroupProxy Line:32 - ================checkTagInGroup 自动集群切换

取消 提交回答
全部回答(2)
  • hiekay
    2019-07-17 23:12:31

    队列 满了, 你可以设置 大一些

    0 0
  • 我是管理员
    2019-07-17 23:12:31

    根据异常日志推测是消费队列(默认长度为10倍的Handler数)打满导致,查看源码ServerRpcConnection类处理请求processRequest()方法:
    /**

    • @param buf
    • Has the request header and the request param and optionally
    • encoded data buffer all in this one array.
    • @throws IOException
    • @throws InterruptedException
      */

    protected void processRequest(ByteBuff buf) throws IOException,

      InterruptedException {
    ...
    
    if (!this.rpcServer.scheduler.dispatch(new CallRunner(this.rpcServer, call))) {
      this.rpcServer.callQueueSizeInBytes.add(-1 * call.getSize());
      this.rpcServer.metrics.exception(RpcServer.CALL_QUEUE_TOO_BIG_EXCEPTION);
      call.setResponse(null, null, RpcServer.CALL_QUEUE_TOO_BIG_EXCEPTION,
        "Call queue is full on " + this.rpcServer.server.getServerName() +
            ", too many items queued ?");
      call.sendResponseIfReady();
    }

    }
    得知Server端通过executeRpcCall()方法执行RPC远程调用CallRunner消费次数超过"hbase.ipc.server.max.callqueue.length"配置值就会引起"too many items queued"异常:
    protected boolean executeRpcCall(final ThreadPoolExecutor executor, final AtomicInteger queueSize,

      final CallRunner task) {
    // Executors provide no offer, so make our own.
    int queued = queueSize.getAndIncrement();
    if (maxQueueLength > 0 && queued >= maxQueueLength) {
      queueSize.decrementAndGet();
      return false;
    }
    
    executor.execute(new FifoCallRunner(task){
      @Override
      public void run() {
        task.setStatus(RpcServer.getStatus());
        task.run();
        queueSize.decrementAndGet();
      }
    });
    
    return true;

    }
    public FifoRpcScheduler(Configuration conf, int handlerCount) {

    this.handlerCount = handlerCount;
    this.maxQueueLength = conf.getInt(RpcScheduler.IPC_SERVER_MAX_CALLQUEUE_LENGTH,
        handlerCount * RpcServer.DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER);

    }

    public static final String IPC_SERVER_MAX_CALLQUEUE_LENGTH =

      "hbase.ipc.server.max.callqueue.length";

    9

    0 0
添加回答
相关问答

40

回答

[@徐雷frank][¥20]什么是JAVA的平台无关性

大河人家 2018-10-29 23:55:20 144240浏览量 回答数 40

17

回答

【大咖问答】对话PostgreSQL 中国社区发起人之一,阿里云数据库高级专家 德哥

阿里ACE 彭飞 2019-07-10 09:36:10 961991浏览量 回答数 17

162

回答

惊喜翻倍:免费ECS+免费环境配置~!(ECS免费体验6个月活动3月31日结束)

豆妹 2014-10-29 17:52:21 222930浏览量 回答数 162

145

回答

【新手入门】云服务器linux使用手册

fanyue88888 2012-11-26 17:14:18 157035浏览量 回答数 145

8

回答

OceanBase 使用动画(持续更新)

mq4096 2019-02-20 17:16:36 333778浏览量 回答数 8

110

回答

OSS存储服务-客户端工具

newegg11 2012-05-17 15:37:18 292218浏览量 回答数 110

22

回答

爬虫数据管理【问答合集】

我是管理员 2018-08-10 16:37:41 146738浏览量 回答数 22

18

回答

阿里云开放端口权限

xcxx 2016-07-20 15:03:33 643003浏览量 回答数 18

31

回答

[@倚贤][¥20]刚学完html/css/js的新手学习servlet、jsp需要注意哪些问题?

弗洛伊德6 2018-10-27 21:52:43 145578浏览量 回答数 31

249

回答

阿里云LNAMP(Linux + Nginx + Apache + MySQL + PHP)环境一键安装脚本

云代维 2014-02-14 15:26:06 302553浏览量 回答数 249
+关注
hbase小能手
HBase是一个分布式的、面向列的开源数据库,一个结构化数据的分布式存储系统。HBase不同于一般的关系数据库,它是一个适合于非结构化数据存储的数据库。阿里云HBase技术团队共同探讨HBase及其生态的问题。
110
文章
338
问答
推荐问答
相关电子书
更多
文娱运维技术
立即下载
《SaaS模式云原生数据仓库应用场景实践》
立即下载
《看见新力量:二》电子书
立即下载