15版本会有问题 ,cannal (停止、启动)一直这一行操作,试多几次 ,会出现页面出现无响应卡死的情况 ,跟@InspireYi 上楼的问题一样 。 dump了node的stack信息出来,看到上一次stop的时候占用了锁 ,导致下一次启动的时候一直在等待 ,再次查看了manager日志 ,manager给node的rpc调用超时。 下面是stack信息:
"DubboServerHandler-127.0.0.1:2088-thread-50" #97 daemon prio=5 os_prio=0 tid=0x00007fc740096000 nid=0x9338 in Object.wait() [0x00007fc6ef0d1000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1252) - locked <0x000000070c149278> (a java.lang.Thread) at java.lang.Thread.join(Thread.java:1326) at com.alibaba.otter.canal.parse.inbound.AbstractEventParser.stop(AbstractEventParser.java:297) at com.alibaba.otter.canal.parse.inbound.mysql.AbstractMysqlEventParser.stop(AbstractMysqlEventParser.java:95) at com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser.stop(MysqlEventParser.java:173) at com.alibaba.otter.canal.instance.core.AbstractCanalInstance.stop(AbstractCanalInstance.java:102) at com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded.stop(CanalServerWithEmbedded.java:116) at com.alibaba.otter.node.etl.select.selector.canal.CanalEmbedSelector.stop(CanalEmbedSelector.java:225) at com.alibaba.otter.node.etl.select.SelectTask.shutdown(SelectTask.java:524) at com.alibaba.otter.node.etl.OtterController.stopPipeline(OtterController.java:216) at com.alibaba.otter.node.etl.OtterController.process(OtterController.java:146) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.notifyListener(NodeTaskServiceImpl.java:429) - locked <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.onNotifyChannel(NodeTaskServiceImpl.java:421) - locked <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.alibaba.otter.shared.communication.core.impl.AbstractCommunicationEndpoint.acceptEvent(AbstractCommunicationEndpoint.java:72) at com.alibaba.dubbo.common.bytecode.Wrapper0.invokeMethod(Wrapper0.java) at com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:46) at com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:72) at com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:108) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:84) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:170) at com.alibaba.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52) at com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
"DubboServerHandler-127.0.0.1:2088-thread-49" #96 daemon prio=5 os_prio=0 tid=0x00007fc740093800 nid=0x9337 waiting for monitor entry [0x00007fc6ef113000] java.lang.Thread.State: BLOCKED (on object monitor) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.onNotifyChannel(NodeTaskServiceImpl.java:419) - waiting to lock <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.alibaba.otter.shared.communication.core.impl.AbstractCommunicationEndpoint.acceptEvent(AbstractCommunicationEndpoint.java:72) at com.alibaba.dubbo.common.bytecode.Wrapper0.invokeMethod(Wrapper0.java) at com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:46) at com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:72) at com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:108) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:84) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:170) at com.alibaba.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52) at com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
"DubboServerHandler-127.0.0.1:2088-thread-48" #95 daemon prio=5 os_prio=0 tid=0x00007fc740092000 nid=0x9336 waiting for monitor entry [0x00007fc6ef154000] java.lang.Thread.State: BLOCKED (on object monitor) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.onNotifyChannel(NodeTaskServiceImpl.java:419) - waiting to lock <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.alibaba.otter.shared.communication.core.impl.AbstractCommunicationEndpoint.acceptEvent(AbstractCommunicationEndpoint.java:72) at com.alibaba.dubbo.common.bytecode.Wrapper0.invokeMethod(Wrapper0.java) at com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:46) at com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:72) at com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:108) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:84) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:170) at com.alibaba.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52) at com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
"DubboServerHandler-127.0.0.1:2088-thread-47" #94 daemon prio=5 os_prio=0 tid=0x00007fc740090000 nid=0x9335 waiting for monitor entry [0x00007fc6ef258000] java.lang.Thread.State: BLOCKED (on object monitor) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.onNotifyChannel(NodeTaskServiceImpl.java:419) - waiting to lock <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.alibaba.otter.shared.communication.core.impl.AbstractCommunicationEndpoint.acceptEvent(AbstractCommunicationEndpoint.java:72) at com.alibaba.dubbo.common.bytecode.Wrapper0.invokeMethod(Wrapper0.java) at com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:46) at com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:72) at com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:108) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:84) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:170) at com.alibaba.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52) at com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
"DubboServerHandler-127.0.0.1:2088-thread-46" #93 daemon prio=5 os_prio=0 tid=0x00007fc74008e800 nid=0x9334 waiting for monitor entry [0x00007fc6ef299000] java.lang.Thread.State: BLOCKED (on object monitor) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.onNotifyChannel(NodeTaskServiceImpl.java:419) - waiting to lock <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.alibaba.otter.shared.communication.core.impl.AbstractCommunicationEndpoint.acceptEvent(AbstractCommunicationEndpoint.java:72) at com.alibaba.dubbo.common.bytecode.Wrapper0.invokeMethod(Wrapper0.java) at com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:46) at com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:72) at com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:108) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:84) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:170) at com.alibaba.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52) at com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
"DubboServerHandler-127.0.0.1:2088-thread-45" #92 daemon prio=5 os_prio=0 tid=0x00007fc74008c800 nid=0x9333 waiting for monitor entry [0x00007fc6ef2d9000] java.lang.Thread.State: BLOCKED (on object monitor) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.onNotifyChannel(NodeTaskServiceImpl.java:419) - waiting to lock <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.alibaba.otter.shared.communication.core.impl.AbstractCommunicationEndpoint.acceptEvent(AbstractCommunicationEndpoint.java:72) at com.alibaba.dubbo.common.bytecode.Wrapper0.invokeMethod(Wrapper0.java) at com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:46) at com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:72) at com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:108) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:84) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:170) at com.alibaba.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52) at com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
"DubboServerHandler-127.0.0.1:2088-thread-44" #91 daemon prio=5 os_prio=0 tid=0x00007fc74008b000 nid=0x9332 waiting for monitor entry [0x00007fc6ef31a000] java.lang.Thread.State: BLOCKED (on object monitor) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.onNotifyChannel(NodeTaskServiceImpl.java:419) - waiting to lock <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.alibaba.otter.shared.communication.core.impl.AbstractCommunicationEndpoint.acceptEvent(AbstractCommunicationEndpoint.java:72) at com.alibaba.dubbo.common.bytecode.Wrapper0.invokeMethod(Wrapper0.java) at com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:46) at com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:72) at com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:108) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:84) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:170) at com.alibaba.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52) at com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
"DubboServerHandler-127.0.0.1:2088-thread-43" #90 daemon prio=5 os_prio=0 tid=0x00007fc740089000 nid=0x9331 waiting for monitor entry [0x00007fc6ef35b000] java.lang.Thread.State: BLOCKED (on object monitor) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.onNotifyChannel(NodeTaskServiceImpl.java:419) - waiting to lock <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.alibaba.otter.shared.communication.core.impl.AbstractCommunicationEndpoint.acceptEvent(AbstractCommunicationEndpoint.java:72) at com.alibaba.dubbo.common.bytecode.Wrapper0.invokeMethod(Wrapper0.java) at com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:46) at com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:72) at com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:108) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:84) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:170) at com.alibaba.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52) at com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
"DubboServerHandler-127.0.0.1:2088-thread-42" #89 daemon prio=5 os_prio=0 tid=0x00007fc740087000 nid=0x9330 waiting for monitor entry [0x00007fc6ef39d000] java.lang.Thread.State: BLOCKED (on object monitor) at com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl.onNotifyChannel(NodeTaskServiceImpl.java:419) - waiting to lock <0x0000000720742ad0> (a com.alibaba.otter.node.common.config.impl.NodeTaskServiceImpl) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.alibaba.otter.shared.communication.core.impl.AbstractCommunicationEndpoint.acceptEvent(AbstractCommunicationEndpoint.java:72) at com.alibaba.dubbo.common.bytecode.Wrapper0.invokeMethod(Wrapper0.java) at com.alibaba.dubbo.rpc.proxy.javassist.JavassistProxyFactory$1.doInvoke(JavassistProxyFactory.java:46) at com.alibaba.dubbo.rpc.proxy.AbstractProxyInvoker.invoke(AbstractProxyInvoker.java:72) at com.alibaba.dubbo.rpc.protocol.dubbo.DubboProtocol$1.reply(DubboProtocol.java:108) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.handleRequest(HeaderExchangeHandler.java:84) at com.alibaba.dubbo.remoting.exchange.support.header.HeaderExchangeHandler.received(HeaderExchangeHandler.java:170) at com.alibaba.dubbo.remoting.transport.DecodeHandler.received(DecodeHandler.java:52) at com.alibaba.dubbo.remoting.transport.dispatcher.ChannelEventRunnable.run(ChannelEventRunnable.java:82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) ........................... 下图是manager请求node启动channal 日志:
问题又来了 ,最新版canal v1.0.26 alpha 1 ,点击多次 停止、启动 ,同一个node几点会有多个mysql binlog dump 链接
jstack 信息:
原提问者GitHub用户 InspireYi
版权声明:本文内容由阿里云实名注册用户自发贡献,版权归原作者所有,阿里云开发者社区不拥有其著作权,亦不承担相应法律责任。具体规则请查看《阿里云开发者社区用户服务协议》和《阿里云开发者社区知识产权保护指引》。如果您发现本社区中有涉嫌抄袭的内容,填写侵权投诉表单进行举报,一经查实,本社区将立刻删除涉嫌侵权内容。