最近在已部署MySQL Enterprise Monitor的服务器上新增了MySQL实例，导致MySQL Enterprise Monitor异常宕机了，无法重新启动成功。收到了Bind on TCP/IP port: Address already in use的错误提示。下面是这个问题的解决办法，供大家参考。
1、故障现象
[root@SZAPP03 init.d]# ./mysql-monitor-server start
Starting mysql service [ OK ]
150127 09:57:34 mysqld_safe Logging to '/opt/mysql/enterprise/monitor/mysql/runtime/mysqld.log'.
150127 09:57:35 mysqld_safe Starting mysqld daemon with databases from /opt/mysql/enterprise/monitor/mysql/data/
./mysql-monitor-server : tomcat (pid 28303) already running
150127 09:57:42 mysqld_safe mysqld from pid file /opt/mysql/enterprise/monitor/mysql/runtime/mysqld.pid ended
2、故障分析
#查看日志，提示为Bind on TCP/IP port: Address already in use，地址在使用，如下
[root@SZAPP03 init.d]# tail -100 /opt/mysql/enterprise/monitor/mysql/runtime/mysqld.log |grep "ERROR" -A5
2015-01-27 09:57:36 30753 [ERROR] Can't start server: Bind on TCP/IP port: Address already in use
2015-01-27 09:57:36 30753 [ERROR] Do you already have another mysqld server running on port: 13306 ?
2015-01-27 09:57:36 30753 [ERROR] Aborting
2015-01-27 09:57:36 30753 [Note] Binlog end
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'partition'
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'ARCHIVE'
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'BLACKHOLE'
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'ARCHIVE'
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'BLACKHOLE'
#根据上述错误提示，核查缺省的mysql enterprise monitor 端口号13306，如下，并没有被占用
[root@SZAPP03 init.d]# netstat -nltp|grep mysql
tcp 0 0 :::3306 :::* LISTEN 8734/mysqld
tcp 0 0 :::3307 :::* LISTEN 9489/mysqld
#这个引起了我的注意，于是尝试先kill到tomcat相关进程
[root@SZAPP03 init.d]# ps -ef|grep 28303|grep -v grep
mysqlmem 28303 28302 4 Jan20 ? 07:35:03 jsvc.exec -java-home /opt/mysql/enterprise/monitor/java
-user mysqlmem -pidfile /opt/mysql/enterprise/monitor/apache-tomcat/temp/catalina.pid -wait 10 -outfile
/opt/mysql/enterprise/monitor/apache-tomcat/logs/catalina-daemon.out -errfile &1 -classpath
/opt/mysql/enterprise/monitor/apache-tomcat/bin/bootstrap.jar:
/opt/mysql/enterprise/monitor/apache-tomcat/bin/commons-daemon.jar:
/opt/mysql/enterprise/monitor/apache-tomcat/bin/tomcat-juli.jar -Dnop -Xmx768M -Xms768M
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/mysql/enterprise/monitor/apache-tomcat/temp
-XX:+UseParallelOldGC -XX:MaxPermSize=512M -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager
-Djava.endorsed.dirs= -Dcatalina.base=/opt/mysql/enterprise/monitor/apache-tomcat
-Dcatalina.home=/opt/mysql/enterprise/monitor/apache-tomcat
-Djava.io.tmpdir=/opt/mysql/enterprise/monitor/apache-tomcat/temp org.apache.catalina.startup.Bootstrap
[root@SZAPP03 init.d]# kill -9 28303
#再次检查是否有tomcat相关进程存在，逐一kill tomcat相关进程
[root@SZAPP03 init.d]# ps -ef|grep tomcat
[root@SZAPP03 init.d]# kill -9 28302
[root@SZAPP03 init.d]# kill -9 30867
# Author : Leshami
# Blog : http://blog.csdn.net/leshami
#再次重新启动mysql em，此时启动正常
[root@SZAPP03 init.d]# ./mysql-monitor-server start
Starting mysql service [ OK ]
150127 10:16:08 mysqld_safe Logging to '/opt/mysql/enterprise/monitor/mysql/runtime/mysqld.log'.
150127 10:16:08 mysqld_safe Starting mysqld daemon with databases from /opt/mysql/enterprise/monitor/mysql/data/
Starting tomcat service [ OK ]
#检查mysql em的状态
[root@SZAPP03 init.d]# ./mysql-monitor-server status
MySQL Enterprise MySQL is running
MySQL Enterprise Tomcat is running
#检查mysql 的端口号
[root@SZAPP03 init.d]# netstat -nltp|grep mysql
tcp 0 0 :::3306 :::* LISTEN 8734/mysqld
tcp 0 0 :::3307 :::* LISTEN 9489/mysqld
tcp 0 0 :::13306 :::* LISTEN 31584/mysqld
1、故障现象
[root@SZAPP03 init.d]# ./mysql-monitor-server start
Starting mysql service [ OK ]
150127 09:57:34 mysqld_safe Logging to '/opt/mysql/enterprise/monitor/mysql/runtime/mysqld.log'.
150127 09:57:35 mysqld_safe Starting mysqld daemon with databases from /opt/mysql/enterprise/monitor/mysql/data/
./mysql-monitor-server : tomcat (pid 28303) already running
150127 09:57:42 mysqld_safe mysqld from pid file /opt/mysql/enterprise/monitor/mysql/runtime/mysqld.pid ended
2、故障分析
#查看日志，提示为Bind on TCP/IP port: Address already in use，地址在使用，如下
[root@SZAPP03 init.d]# tail -100 /opt/mysql/enterprise/monitor/mysql/runtime/mysqld.log |grep "ERROR" -A5
2015-01-27 09:57:36 30753 [ERROR] Can't start server: Bind on TCP/IP port: Address already in use
2015-01-27 09:57:36 30753 [ERROR] Do you already have another mysqld server running on port: 13306 ?
2015-01-27 09:57:36 30753 [ERROR] Aborting
2015-01-27 09:57:36 30753 [Note] Binlog end
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'partition'
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'ARCHIVE'
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'BLACKHOLE'
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'ARCHIVE'
2015-01-27 09:57:36 30753 [Note] Shutting down plugin 'BLACKHOLE'
#根据上述错误提示，核查缺省的mysql enterprise monitor 端口号13306，如下，并没有被占用
[root@SZAPP03 init.d]# netstat -nltp|grep mysql
tcp 0 0 :::3306 :::* LISTEN 8734/mysqld
tcp 0 0 :::3307 :::* LISTEN 9489/mysqld
3、故障解决
#故障现象里有一个提示为tomcat (pid 28303) already running#这个引起了我的注意，于是尝试先kill到tomcat相关进程
[root@SZAPP03 init.d]# ps -ef|grep 28303|grep -v grep
mysqlmem 28303 28302 4 Jan20 ? 07:35:03 jsvc.exec -java-home /opt/mysql/enterprise/monitor/java
-user mysqlmem -pidfile /opt/mysql/enterprise/monitor/apache-tomcat/temp/catalina.pid -wait 10 -outfile
/opt/mysql/enterprise/monitor/apache-tomcat/logs/catalina-daemon.out -errfile &1 -classpath
/opt/mysql/enterprise/monitor/apache-tomcat/bin/bootstrap.jar:
/opt/mysql/enterprise/monitor/apache-tomcat/bin/commons-daemon.jar:
/opt/mysql/enterprise/monitor/apache-tomcat/bin/tomcat-juli.jar -Dnop -Xmx768M -Xms768M
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/mysql/enterprise/monitor/apache-tomcat/temp
-XX:+UseParallelOldGC -XX:MaxPermSize=512M -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager
-Djava.endorsed.dirs= -Dcatalina.base=/opt/mysql/enterprise/monitor/apache-tomcat
-Dcatalina.home=/opt/mysql/enterprise/monitor/apache-tomcat
-Djava.io.tmpdir=/opt/mysql/enterprise/monitor/apache-tomcat/temp org.apache.catalina.startup.Bootstrap
[root@SZAPP03 init.d]# kill -9 28303
#再次检查是否有tomcat相关进程存在，逐一kill tomcat相关进程
[root@SZAPP03 init.d]# ps -ef|grep tomcat
[root@SZAPP03 init.d]# kill -9 28302
[root@SZAPP03 init.d]# kill -9 30867
# Author : Leshami
# Blog : http://blog.csdn.net/leshami
#再次重新启动mysql em，此时启动正常
[root@SZAPP03 init.d]# ./mysql-monitor-server start
Starting mysql service [ OK ]
150127 10:16:08 mysqld_safe Logging to '/opt/mysql/enterprise/monitor/mysql/runtime/mysqld.log'.
150127 10:16:08 mysqld_safe Starting mysqld daemon with databases from /opt/mysql/enterprise/monitor/mysql/data/
Starting tomcat service [ OK ]
#检查mysql em的状态
[root@SZAPP03 init.d]# ./mysql-monitor-server status
MySQL Enterprise MySQL is running
MySQL Enterprise Tomcat is running
#检查mysql 的端口号
[root@SZAPP03 init.d]# netstat -nltp|grep mysql
tcp 0 0 :::3306 :::* LISTEN 8734/mysqld
tcp 0 0 :::3307 :::* LISTEN 9489/mysqld
tcp 0 0 :::13306 :::* LISTEN 31584/mysqld
