背景介绍
在日常排查过程中，有遇到线程死锁的问题，系统自带的jstack可以用来定位发生死锁的线程，而arthas作为日常分析排查问题不可缺少的利器，如果能够将线程死锁检测集成进arthas，线程死锁分析排查会更加方便和高效。
实现思路
jstack实现方式
以下程序主要由两个部分组成：
- 由synchronized和java.util.concurrent.Lock引起的线程死锁示例
- 线程死锁检测实现逻辑
产生线程死锁逻辑
import java.util.concurrent.locks.ReentrantLock; public class DeadlockBuilder { private static final ReentrantLock REENTRANT_LOCK_A = new ReentrantLock(); private static final ReentrantLock REENTRANT_LOCK_B = new ReentrantLock(); public static void deadlockByMonitor() { Object a = new Object(); Object b = new Object(); Thread threadA = new Thread(() -> { synchronized (a) { sleep(); synchronized (b) { } } }); threadA.setName("Thread_Monitor_A"); threadA.start(); Thread threadB = new Thread(() -> { synchronized (b) { sleep(); synchronized (a) { } } }); threadB.setName("Thread_Monitor_B"); threadB.start(); } public static void deadlockBySynchronizer() { Thread threadA = new Thread(() -> { REENTRANT_LOCK_A.lock(); try { sleep(); REENTRANT_LOCK_B.lock(); try { } finally { REENTRANT_LOCK_B.unlock(); } } finally { REENTRANT_LOCK_A.unlock(); } }); threadA.setName("Thread_Synchronizer_A"); threadA.start(); Thread threadB = new Thread(() -> { REENTRANT_LOCK_B.lock(); try { sleep(); REENTRANT_LOCK_A.lock(); try { } finally { REENTRANT_LOCK_A.unlock(); } } finally { REENTRANT_LOCK_B.unlock(); } }); threadB.setName("Thread_Synchronizer_B"); threadB.start(); } private static void sleep() { try { Thread.sleep(100L); } catch (Exception e) { e.printStackTrace(); } } }
死锁检测实现逻辑
import java.lang.management.ManagementFactory; import java.lang.management.MonitorInfo; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.lang.management.LockInfo; import java.util.*; public class DeadlockDetector { private static final ThreadMXBean THREADMX_BEAN = ManagementFactory.getThreadMXBean(); public static void main(String[] args) throws Exception { System.out.println(ManagementFactory.getRuntimeMXBean().getName()); DeadlockBuilder.deadlockByMonitor(); DeadlockBuilder.deadlockBySynchronizer(); Thread.sleep(1000); DeadlockInfo deadlock = detect(); render(deadlock); } private static DeadlockInfo detect() { int globalDfn = 0, thisDfn; ThreadInfo currentThread; ThreadInfo previousThread; LockInfo waitingToLockInfo; ThreadInfo[] infos = THREADMX_BEAN.dumpAllThreads(THREADMX_BEAN.isObjectMonitorUsageSupported(), THREADMX_BEAN.isSynchronizerUsageSupported()); Map<ThreadInfo, Integer> threadTable = new HashMap<>(); Map<Integer, ThreadInfo> ownerThreadPerLock = new HashMap<>(); for (ThreadInfo threadInfo : infos) { threadTable.put(threadInfo, -1); for (MonitorInfo monitorInfo : threadInfo.getLockedMonitors()) { ownerThreadPerLock.putIfAbsent(monitorInfo.getIdentityHashCode(), threadInfo); } for (LockInfo lockedSync : threadInfo.getLockedSynchronizers()) { ownerThreadPerLock.putIfAbsent(lockedSync.getIdentityHashCode(), threadInfo); } } DeadlockInfo deadlock = new DeadlockInfo(); deadlock.setOwnerThreadPerLock(ownerThreadPerLock); for (Map.Entry<ThreadInfo, Integer> e : threadTable.entrySet()) { Integer value = e.getValue(); if (value >= 0) { // this thread was already visited continue; } thisDfn = globalDfn; ThreadInfo thread = e.getKey(); previousThread = thread; waitingToLockInfo = thread.getLockInfo(); while (waitingToLockInfo != null) { currentThread = ownerThreadPerLock.get(waitingToLockInfo.getIdentityHashCode()); if (currentThread == null) { // No dependency on another thread break; } Integer val = threadTable.get(thread); int dfn = (val != null) ? val : -1; if (dfn < 0) { // First visit to this thread threadTable.put(currentThread, globalDfn++); } else if (dfn < thisDfn) { // Thread already visited, and not on a (new) cycle break; } else if (currentThread == previousThread) { // Self-loop, ignore break; } else { // We have a (new) cycle deadlock.getThreads().add(currentThread); break; } previousThread = currentThread; waitingToLockInfo = currentThread.getLockInfo(); } } return deadlock; } public static void render(DeadlockInfo deadlock){ Map<Integer, ThreadInfo> ownerThreadPerLock = deadlock.getOwnerThreadPerLock(); for(ThreadInfo thread : deadlock.getThreads()){ LockInfo waitingToLockInfo; ThreadInfo currentThread = thread; System.out.println("Found one Java-level deadlock:"); System.out.println("============================="); do{ System.out.println(currentThread.getThreadName() + "(" + currentThread.getThreadId() + "):"); waitingToLockInfo = currentThread.getLockInfo(); if(waitingToLockInfo != null){ System.out.println(" waiting to lock info @" + waitingToLockInfo + ","); System.out.print(" which is held by "); currentThread = ownerThreadPerLock.get(waitingToLockInfo.getIdentityHashCode()); System.out.println(currentThread.getThreadName()); } }while (!currentThread.equals(thread)); System.out.println(); } int numberOfDeadlocks = deadlock.getThreads().size(); switch (numberOfDeadlocks) { case 0: System.out.println("No deadlocks found."); break; case 1: System.out.println("Found a total of 1 deadlock."); break; default: System.out.println("Found a total of " + numberOfDeadlocks + " deadlocks."); break; } } } class DeadlockInfo{ private List<ThreadInfo> threads = new LinkedList<>(); private Map<Integer, ThreadInfo> ownerThreadPerLock = new HashMap<>(); public Map<Integer, ThreadInfo> getOwnerThreadPerLock() { return ownerThreadPerLock; } public void setOwnerThreadPerLock(Map<Integer, ThreadInfo> ownerThreadPerLock) { this.ownerThreadPerLock = ownerThreadPerLock; } public List<ThreadInfo> getThreads() { return threads; } public void setThreads(List<ThreadInfo> threads) { this.threads = threads; } }
直接运行上面的程序就可以看到效果了。
JMX实现方式
import java.lang.management.*; public class ThreadMXDeadlockDetector { private static final ThreadMXBean THREADMX_BEAN = ManagementFactory.getThreadMXBean(); public static void main(String[] args) throws Exception { System.out.println(ManagementFactory.getRuntimeMXBean().getName()); DeadlockBuilder.deadlockByMonitor(); DeadlockBuilder.deadlockBySynchronizer(); Thread.sleep(1000); DeadlockInfo deadlock = detect(); DeadlockDetector.render(deadlock); } private static DeadlockInfo detect(){ DeadlockInfo deadlockInfo = new DeadlockInfo(); long[] ids = THREADMX_BEAN.findDeadlockedThreads(); if(ids == null){ return deadlockInfo; } ThreadInfo[] threads = THREADMX_BEAN.getThreadInfo(ids, THREADMX_BEAN.isObjectMonitorUsageSupported(),THREADMX_BEAN.isSynchronizerUsageSupported()); for(ThreadInfo threadInfo : threads){ LockInfo lockInfo = threadInfo.getLockInfo(); if(!deadlockInfo.getOwnerThreadPerLock().containsKey(lockInfo.getIdentityHashCode())){ deadlockInfo.getThreads().add(threadInfo); } for (MonitorInfo monitorInfo : threadInfo.getLockedMonitors()) { deadlockInfo.getOwnerThreadPerLock().putIfAbsent(monitorInfo.getIdentityHashCode(), threadInfo); } for (LockInfo lockedSync : threadInfo.getLockedSynchronizers()) { deadlockInfo.getOwnerThreadPerLock().putIfAbsent(lockedSync.getIdentityHashCode(), threadInfo); } } return deadlockInfo; } }
集成进arthas以提高排查效率
arthas jvm命令显示了死锁线程个数，线程名称、线程ID并没有展示出来，如果想知道死锁线程具体信息需要再使用jstack来分析；
如果死锁线程信息可以在arthas中展示出来可以提高问题排查效率。
集成进arthas主要涉及以下几个类的修改：
- ThreadCommand中增加线程死锁检测指令@Option(shortName = “d”, longName = “deadlock”, flag = true)
- ThreadUtil中增加线程死锁检测的实现方法
- ThreadView中增加线程死锁检测的结果展示逻辑
然后编译打包，运行上面死锁生成的代码，使用arthas thread -d命令效果如下：
总结
- 构造由synchronized和java.util.concurrent.Lock引起线程死锁的示例；
- jstack及JMX实现线程死锁检测的逻辑；
- 将线程死锁检测集成进arthas。
