线上排查堆栈
查看占用cpu最大的线程
[root@iZbp1c400avts4bhhmibppZ ~]# ps H -eo pid,tid,pcpu | sort -n -k 3 | tail -10 1015 1022 0.2 816 924 0.2 1029 1291 0.3 1343 1407 0.3 1628 2030 0.3 1870 1975 0.3 1870 2001 0.3 22443 22443 1.0 1029 1978 1.4 1870 1973 2.6
或者
# 查看进程 top [root@iZbp1c400avts4bhhmibppZ ~]# top top - 09:22:33 up 18:27, 1 user, load average: 0.01, 0.05, 0.05 Tasks: 113 total, 2 running, 111 sleeping, 0 stopped, 0 zombie %Cpu(s): 2.0 us, 1.0 sy, 0.0 ni, 96.9 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 3688284 total, 482424 free, 2254508 used, 951352 buff/cache KiB Swap: 0 total, 0 free, 0 used. 1201720 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 1870 root 20 0 4408136 330340 14956 S 4.5 9.0 41:03.94 java 1029 root 10 -10 136340 19368 11024 S 2.5 0.5 28:01.65 AliYunDunM+ 1015 root 10 -10 100492 7756 6532 S 0.5 0.2 6:29.37 AliYunDun 1343 mysql 20 0 1828648 398428 14948 S 0.5 10.8 6:03.44 mysqld 1628 root 20 0 2526596 387452 4416 S 0.5 10.5 7:41.22 java 3250 root 20 0 2568792 540432 9196 S 0.5 14.7 2:36.94 java 1 root 20 0 43564 3996 2624 S 0.0 0.1 0:03.23 systemd 2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd 4 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:+ 6 root 20 0 0 0 0 S 0.0 0.0 0:02.54 ksoftirqd/0 7 root rt 0 0 0 0 S 0.0 0.0 0:00.22 migration/0 8 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_bh 9 root 20 0 0 0 0 S 0.0 0.0 1:10.24 rcu_sched 10 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 lru-add-dr+ 11 root rt 0 0 0 0 S 0.0 0.0 0:00.28 watchdog/0 12 root rt 0 0 0 0 S 0.0 0.0 0:00.25 watchdog/1 13 root rt 0 0 0 0 S 0.0 0.0 0:00.16 migration/1 14 root 20 0 0 0 0 S 0.0 0.0 0:00.82 ksoftirqd/1 16 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/1:+ 18 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kdevtmpfs 19 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 netns 20 root 20 0 0 0 0 S 0.0 0.0 0:00.04 khungtaskd 21 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 writeback 22 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kintegrityd 23 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset 24 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset 25 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 bioset 26 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kblockd 27 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 md 28 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 edac-poller 29 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 watchdogd 36 root 20 0 0 0 0 S 0.0 0.0 0:00.42 kswapd0 37 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd 38 root 39 19 0 0 0 S 0.0 0.0 0:00.60 khugepaged # 选择占用最高的进程PID查看线程 # top -Hp PID [root@iZbp1c400avts4bhhmibppZ ~]# top -Hp 1870 top - 09:23:04 up 18:27, 1 user, load average: 0.00, 0.04, 0.05 Threads: 119 total, 1 running, 118 sleeping, 0 stopped, 0 zombie %Cpu(s): 1.7 us, 1.5 sy, 0.0 ni, 96.8 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 3688284 total, 482156 free, 2254560 used, 951568 buff/cache KiB Swap: 0 total, 0 free, 0 used. 1201588 avail Mem PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 1973 root 20 0 4408136 330340 14956 S 2.7 9.0 29:07.90 java 2001 root 20 0 4408136 330340 14956 S 0.7 9.0 4:05.41 java 1975 root 20 0 4408136 330340 14956 S 0.3 9.0 3:38.35 java 1870 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1871 root 20 0 4408136 330340 14956 S 0.0 9.0 0:02.08 java 1872 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.20 java 1873 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.20 java 1874 root 20 0 4408136 330340 14956 S 0.0 9.0 0:09.28 java 1875 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1876 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.01 java 1877 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1878 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.01 java 1886 root 20 0 4408136 330340 14956 S 0.0 9.0 0:02.37 java 1887 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.01 java 1888 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.02 java 1894 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1895 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1896 root 20 0 4408136 330340 14956 S 0.0 9.0 0:10.76 java 1897 root 20 0 4408136 330340 14956 S 0.0 9.0 0:02.99 java 1899 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1900 root 20 0 4408136 330340 14956 R 0.0 9.0 0:44.60 java 1914 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.58 java 1917 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1918 root 20 0 4408136 330340 14956 S 0.0 9.0 0:11.82 java 1919 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.37 java 1920 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.73 java 1921 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1922 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.25 java 1923 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1924 root 20 0 4408136 330340 14956 S 0.0 9.0 0:00.00 java 1926 root 20 0 4408136 330340 14956 S 0.0 9.0 0:02.45 java 1932 root 20 0 4408136 330340 14956 S 0.0 9.0 0:02.55 java 1934 root 20 0 4408136 330340 14956 S 0.0 9.0 0:01.54 java 1935 root 20 0 4408136 330340 14956 S 0.0 9.0 1:14.72 java
查看进程下的线程
[root@iZbp1c400avts4bhhmibppZ ~]# pstree -p 1870 java(1870)─┬─{java}(1871) ├─{java}(1872) ├─{java}(1873) ├─{java}(1874) ├─{java}(1875) ├─{java}(1876) ├─{java}(1877) ├─{java}(1878) ├─{java}(1886) ├─{java}(1887) ├─{java}(1888) ├─{java}(1894) ├─{java}(1895) ├─{java}(1896) ├─{java}(1897) ├─{java}(1899) ├─{java}(1900) ├─{java}(1914) ├─{java}(1917) ├─{java}(1918) ├─{java}(1919) ├─{java}(1920) ├─{java}(1921) ├─{java}(1922) ├─{java}(1923) ├─{java}(1924) ├─{java}(1926) ├─{java}(1932) ├─{java}(1934) ├─{java}(1935) ├─{java}(1936) ├─{java}(1938) ├─{java}(1966) ├─{java}(1968) ├─{java}(1970) ├─{java}(1971) ├─{java}(1972) ├─{java}(1973) ├─{java}(1974) ├─{java}(1975) ├─{java}(1976) ├─{java}(1977) ├─{java}(1979) ├─{java}(1980) ├─{java}(1981) ├─{java}(1982) ├─{java}(1983) ├─{java}(1984) ├─{java}(1985) ├─{java}(1986) ├─{java}(1988) ├─{java}(1989) ├─{java}(1996) ├─{java}(2001) ├─{java}(2002) ├─{java}(2028) ├─{java}(2056) ├─{java}(2059) ├─{java}(2060) ├─{java}(2062) ├─{java}(2075) ├─{java}(2088) ├─{java}(2089) ├─{java}(2090) ├─{java}(2091) ├─{java}(2092) ├─{java}(2093) ├─{java}(2094) ├─{java}(2095) ├─{java}(2096) ├─{java}(2097) ├─{java}(2098) ├─{java}(2099) ├─{java}(2181) ├─{java}(2203) ├─{java}(2204) ├─{java}(2205) ├─{java}(2206) ├─{java}(2331) ├─{java}(2332) ├─{java}(2333) ├─{java}(2334) ├─{java}(2335) ├─{java}(2336) ├─{java}(2337) ├─{java}(2508) ├─{java}(2538) ├─{java}(2539) ├─{java}(2697) ├─{java}(2698) ├─{java}(2784) ├─{java}(2785) ├─{java}(3405) ├─{java}(3510) ├─{java}(3586) ├─{java}(3711) ├─{java}(3910) ├─{java}(4108) ├─{java}(4311) ├─{java}(4508) ├─{java}(4690) ├─{java}(4926) ├─{java}(5004) ├─{java}(5081) ├─{java}(5154) ├─{java}(5228) ├─{java}(5275) ├─{java}(5302) ├─{java}(5375) ├─{java}(5422) ├─{java}(5449) ├─{java}(5523) ├─{java}(5597) ├─{java}(5669) ├─{java}(5747) ├─{java}(5820) ├─{java}(5893) └─{java}(5966)
查看线程详细信息
[root@iZbp1c400avts4bhhmibppZ ~]# cat /proc/1870/task/1871/status Name: java Umask: 0022 State: S (sleeping) Tgid: 1870 Ngid: 0 Pid: 1871 PPid: 1867 TracerPid: 0 Uid: 0 0 0 0 Gid: 0 0 0 0 FDSize: 256 Groups: 0 VmPeak: 4408136 kB VmSize: 4408136 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 330340 kB VmRSS: 330340 kB RssAnon: 315384 kB RssFile: 14956 kB RssShmem: 0 kB VmData: 2732128 kB VmStk: 132 kB VmExe: 4 kB VmLib: 19964 kB VmPTE: 1204 kB VmSwap: 0 kB Threads: 119 SigQ: 0/14325 SigPnd: 0000000000000000 ShdPnd: 0000000000000000 SigBlk: 0000000000000004 SigIgn: 0000000000000000 SigCgt: 2000000181005ccf CapInh: 0000000000000000 CapPrm: 00000000a80425fb CapEff: 00000000a80425fb CapBnd: 00000000a80425fb CapAmb: 0000000000000000 NoNewPrivs: 0 Seccomp: 2 Speculation_Store_Bypass: vulnerable Cpus_allowed: 3 Cpus_allowed_list: 0-1 Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 Mems_allowed_list: 0 voluntary_ctxt_switches: 323 nonvoluntary_ctxt_switches: 811
