现在开始介绍php+mysql+shell监控系统
1
、目的
此监控系统主要是通过
php+mysql+shell的方式，通过shell脚本对各个机器的其各个服务进行监控，达到及时的了解其各个应用服务的状态（如果宕掉与启动），在检测应用服务宕掉时，记录在日志里与mysql数据库服务器里，同时进行服务宕掉的邮件提示，并自动启动宕掉的应用服务；在检测应该服务启动时，同时进行服务启动的邮件提示，所有的监控内容都能在php制作的web里进行浏览，同时能根据mysql里的数据，把资源监控数据视图化，在浏览数据的时候，更方便。
2
、实现的机制
监控方式主要是通过
shell脚本的实现，针对应用服务运行的端口进行监控，如果其端口开启，则证明此服务运行，反之则此服务没有运行。
3
、监控的应用服务
监控的应用服务有：
A、应用服务器的服务监控
web：
http、yu_tomcat、tomcat 共3个
交换：
pas、ppas、mas、mmas、cas共5个
引擎：
memcache、datastorageservice、http、dbstatserver共4个
B:应用服务器的资源监控
硬盘使用率、
cpu使用率、硬件启动信息错误、i/o使用率、15分钟内的负载、内存使用率（包括内存与swap）、日志错误信息、当前用户登录数
4
、已经监控的省份
目前天津、广西、贵州、海南、河北、河南、湖北、湖南、宁夏、陕西、青海、西藏、新疆、包头、甘肃、江西都已经部署完成，经过
3个月的测试，没有发现问题。
5
、shell监控脚本内容与解释
（
1）web服务器（以甘肃web为例）
- #!/bin/bash
- #ip
- web_ip=$(/sbin/ifconfig eth0|grep "inet addr"|cut -d : -f 2|awk '{print $1}')
- ##eth0网卡的ip
- #name
- web_name=$(hostname)
- ##web主机名
- monitor_name=web
- ##监控的服务器是什么类型
- here='gansu'
- ##监控的省份
- #mysql info
- mysql_ip='1.1.1.1'
- ##数据库的ip
- mysql_username='root'
- ##数据库的用户
- mysql_passwd='****'
- #数据库的密码
- mysql_database='monitor'
- ##选择的数据库
- memory_table=''$here'_memory'
- load_table=''$here'_load'
- io_table=''$here'_io'
- hardware_table=''$here'_hardware'
- message_table=''$here'_message'
- user_table=''$here'_user'
- disk_table=''$here'_disk'
- cpu_table=''$here'_cpu'
- service_table=''$here'_service'
- ##想数据库里写入的表
- #date and log
- day="$(date +%Y%m%d)"
- worklog='/usr/local/monitor/logs/all_work_log'
- downlog='/usr/local/monitor/logs/all_down_log'
- ##日志与实际
- now="$(date +%Y-%m-%d-%T)"
- #web service
- tomcat="$(netstat -antl|grep 8080|wc -l)"
- web_http="$(ps -ef|grep httpd|grep -v grep|wc -l)"
- yz_tomcat="$(netstat -antl|grep 8081|wc -l)"
- ##通过端口与服务在后头的运行情况监控其运行状态
- #cpu service
- alert_cpu='80'
- ##cpu的报警阀值
- web_cpu_idle="$(top -b -n 1 | grep Cpu | awk '{print $5}' | cut -f 1 -d .)"
- web_cpu="$(echo 100 - $web_cpu_idle|/usr/bin/bc)"
- ##监控cpu的使用率
- #memory service
- alert_mem='100'
- ##内存的报警阀值
- web_mem="$(/usr/bin/free -m|grep Mem|awk '{print $4}')"
- web_swap="$(/usr/bin/free -m|grep Swap|awk '{print $3}')"
- ##memory与swap的使用率
- alert_swap='0'
- ##swap的报警阀值
- #memory log
- memory_worklog='/usr/local/monitor/logs/mem_work_log'
- memory_downlog='/usr/local/monitor/logs/mem_down_log'
- #load service
- cpu_count="$(grep -c 'model name' /proc/cpuinfo)"
- alert_load="$(echo $cpu_count/2|/usr/bin/bc)"
- web_load="$(uptime|awk '{print $NF}'|cut -f 1 -d .)"
- web_load_15="$(uptime|awk '{print $NF}')"
- #监控负载的使用率
- #load log
- load_worklog='/usr/local/monitor/logs/load_work_log'
- load_downlog='/usr/local/monitor/logs/load_down_log'
- #io service
- alert_io='80'
- web_io_idle_back="$(/usr/bin/iostat|awk 'NR==4{print $NF}'|cut -f 1 -d .)"
- web_io_idle="$(echo 100 - $web_io_idle_back|/usr/bin/bc)"
- ##io的使用值
- #io log
- io_worklog='/usr/local/monitor/logs/io_work_log'
- io_downlog='/usr/local/monitor/logs/io_down_log'
- #hardware service
- web_hardware_error="$(dmesg|grep -i error|wc -l)"
- web_info_error="$(dmesg|grep -i error)"
- ##开机启动的错误信息
- #hardware error log
- hard_worklog='/usr/local/monitor/logs/hard_work_log'
- hard_downlog='/usr/local/monitor/logs/hard_down_log'
- #message service
- web_message_error_count="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error|wc -l)"
- web_message_error="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error)"
- ##日志的错误信息
- #message error log
- message_worklog='/usr/local/monitor/logs/message_work_log'
- message_downlog='/usr/local/monitor/logs/message_down_log'
- #user service
- web_user_count="$(/usr/bin/who|wc -l)"
- web_user_info="$(/usr/bin/who)"
- ##当前登录的用户值
- #user log
- user_worklog='/usr/local/monitor/logs/user_work_log'
- user_downlog='/usr/local/monitor/logs/user_down_log'
- #disk service
- alert_disk='80'
- web_disk="$(/bin/df -H|sort -k5nr|grep -vE 'Filesystem|tmpfs|cdrom'|awk '{print $5 " " $1}')"
- web_disk_use=$(echo $web_disk|awk '{print $1}'|cut -d '%' -f1)
- web_disk_partition=$(echo $web_disk|awk '{print $2}')
- #硬盘使用率
- #disk log
- disk_worklog='/usr/local/monitor/logs/disk_work_log'
- disk_downlog='/usr/local/monitor/logs/disk_down_log'
- #cpu log
- cpu_worklog='/usr/local/monitor/logs/cpu_work_log'
- cpu_downlog='/usr/local/monitor/logs/cpu_down_log'
- #notification mail
- email='denglei@ctfo.com'
- ##报警的接收人
- #monitor
- if [ ! -d "$loghere" ];then
- mkdir $loghere
- fi
- #check web service
- if [ $web_http -ge 1 ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: http Monitor_Server: $monitor_name is working" >> $worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$web_name','$web_ip','$monitor_name','web_http','working',now())";
- else
- /etc/init.d/httpd start
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: http Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: http was a problem" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: http Monitor_Server: $montior_name is down" >> $downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$web_name','$web_ip','$monitor_name','web_http','downing',now())";
- fi
- if [ $tomcat -ge 1 ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: tomcat Monitor_Server: $monitor_name is working" >> $worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$web_name','$web_ip','$monitor_name','tomcat','working',now())";
- else
- /usr/local/monitor/shell/web_tomcat.sh
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: tomcat Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: tomcat was a problem" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: tomcat Monitor_Server: $montior_name is down" >> $downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$web_name','$web_ip','$monitor_name','tomcat','downing',now())";
- fi
- if [ $yz_tomcat -ge 1 ];then
- echo "$now ShengFen: $here Server: $web_name Service: Ip: $web_ip yz_tomcat Monitor_Server: $monitor_name is working" >> $worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$web_name','$web_ip','$monitor_name','yz_tomcat','working',now())";
- else
- /usr/local/monitor/shell/web_yz_tomcat.sh
- echo "$now ShengFen: $here Server: $web_name Service: Ip: $web_ip yz_tomcat Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: yz_tomcat was a problem" $email
- echo "$now ShengFen: $here Server: $web_name Service: Ip: $web_ip yz_tomcat Monitor_Server: $montior_name is down" >> $downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$web_name','$web_ip','$monitor_name','yz_tomcat','downing',now())";
- fi
- #check cpu_idle
- if [ $web_cpu -ge $alert_cpu ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $web_cpu"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: cpu_use was Exceed Threshold value: 80%" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $web_cpu" >> $cpu_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$web_name','$web_ip','$monitor_name','cpu_use','$alert_cpu','$web_cpu','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $web_cpu" >> $cpu_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$web_name','$web_ip','$monitor_name','cpu_use','$alert_cpu','$web_cpu','normal',now())";
- fi
- #check memory
- if [ $web_mem -le $alert_mem ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: memory Monitor_Server: $monitor_name Free_mem: $web_mem"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: memory was Exceed Threshold value" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: memory Monitor_Server: $monitor_name Free_mem: $web_mem" >> $memory_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$web_name','$web_ip','$monitor_name','memory','$alert_mem','$web_mem','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: memory Monitor_Server: $monitor_name Free_mem: $web_mem" >> $memory_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$web_name','$web_ip','$monitor_name','memory','$alert_mem','$web_mem','normal',now())";
- fi
- #check swap
- if [ $web_swap -gt $alert_swap ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: swap Monitor_Server: $monitor_name Swap_web: $web_swap"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: swap Ip: $web_ip was Exceed Threshold value" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: swap Monitor_Server: $monitor_name Swap_web: $web_swap" >> $memory_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$web_name','$web_ip','$monitor_name','swap','$alert_swap','$web_swap','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: swap Monitor_Server: $monitor_name Swap_web: $web_swap" >> $memory_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$web_name','$web_ip','$monitor_name','swap','$alert_swap','$web_swap','normal',now())";
- fi
- #check load_15
- if [ $web_load -ge $alert_load ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $web_load_15"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: load_15 Ip: $web_ip was Exceed Threshold value" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $web_load_15" >> $load_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$web_name','$web_ip','$monitor_name','load_15','$alert_load','$web_load_15','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $server Ip: $ip Service: load Monitor_Server: $monitor_name Load_use: $web_load_15" >> $load_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$web_name','$web_ip','$monitor_name','load_15','$alert_load','$web_load_15','normal',now())";
- fi
- #check io_idle
- if [ $web_io_idle -ge $alert_io ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $web_io_idle"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: io_use was Exceed Threshold value: 80%" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $web_io_idle" >> $io_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$web_name','$web_ip','$monitor_name','io_use','$alert_io','$web_io_idle','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $web_io_idle" >> $io_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$web_name','$web_ip','$monitor_name','io_use','$alert_io','$web_io_idle','normal',now())";
- fi
- #check hareware error info
- if [ $web_hardware_error -gt 0 ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: hardware_error Monitor_Server: $monitor_name Error: $web_info_error"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: hardware_error were some hardware imformation error" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: hardware_error Monitor_Server: $monitor_name Error: $web_info_error" >> $hard_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$web_name','$web_ip','$monitor_name','hardware_error','0','$web_hardware_error','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: hardware_error Monitor_Server: $monitor_name Error: Nothing" >> $hard_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$web_name','$web_ip','$monitor_name','hardware_error','0','$web_hardware_error','normal',now())";
- fi
- #check message error
- if [ $web_message_error_count -ge 1 ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: message_error Monitor_Server: $monitor_name Message_error: $web_message_error"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: message_error were some message imformation error" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: message_error Monitor_Server: $monitor_name Message_error: $web_message_error" >> $message_downlog-$day
- /usr/bin/mysql-h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$web_name','$web_ip','$monitor_name','message_error','1','$web_message_error_count','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: message_error Monitor_Server: $monitor_name Message_error: Nothing" >> $message_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$web_name','$web_ip','$monitor_name','message_error','1','$web_message_error_count','normal',now())";
- fi
- #check user
- if [ $web_user_count -ge 3 ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: user Monitor_Server: $monitor_name User: $web_user_info"|/bin/mail -s "Notification ShengFen: $here Server: $web_name Service: user was Exceed Threshold value: 3" $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: user Monitor_Server: $monitor_name User: $web_user_info" >> $user_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$web_name','$web_ip','$monitor_name','user','3','$web_user_count','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: user Monitor_Server: $monitor_name User: normal" >> $user_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$web_name','$web_ip','$monitor_name','user','3','$web_user_count','normal',now())";
- fi
- #check disk
- if [ $web_disk_use -ge $alert_disk ];then
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $web_disk_partition ($web_disk_use%)"|/bin/mail -s "Warning!!! $here Server: $web_name Service: disk_use was Exceed Threshold value : $alert_disk% " $email
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $web_disk_partion ($web_disk_use%)" >> $disk_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $disk_table values ('','$here','$web_name','$web_ip','$monitor_name','disk_use','$alert_disk','$web_disk_partition','$web_disk_use','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $web_name Ip: $web_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $web_disk_partition ($web_disk_use%)" >> $disk_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $disk_table values ('','$here','$web_name','$web_ip','$monitor_name','disk_use','$alert_disk','$web_disk_partition','$web_disk_use','normal',now())";
- fi
（
2）交换服务器（以甘肃交换为例,解释内容参照甘肃web）
- #!/bin/bash
- #ip
- jh_ip=$(/sbin/ifconfig eth0|grep "inet addr"|cut -d : -f 2|awk '{print $1}')
- #name
- jh_name=$(hostname)
- monitor_name=jiaohuan
- here='gansu'
- #mysql info
- mysql_ip='1.1.1.1'
- mysql_username='root'
- mysql_passwd='****'
- mysql_database='monitor'
- memory_table=''$here'_memory'
- load_table=''$here'_load'
- io_table=''$here'_io'
- hardware_table=''$here'_hardware'
- message_table=''$here'_message'
- user_table=''$here'_user'
- disk_table=''$here'_disk'
- cpu_table=''$here'_cpu'
- service_table=''$here'_service'
- #date and log
- day="$(date +%Y%m%d)"
- worklog='/usr/local/monitor/logs/all_work_log'
- downlog='/usr/local/monitor/logs/all_down_log'
- now="$(date +%Y-%m-%d-%T)"
- loghere='/usr/local/monitor/logs'
- #jiaohuan service
- pas="$(ps -ef|grep pas|grep -v grep|grep -v ppas|wc -l)"
- ppas="$(ps -ef|grep ppas|grep -v grep|wc -l)"
- cas="$(ps -ef|grep cas|grep -v grep|wc -l)"
- mas="$(ps -ef|grep mas|grep -v grep|grep -v mmas|wc -l)"
- mmas="$(ps -ef|grep mmas|grep -v grep|wc -l)"
- #cpu service
- alert_cpu='80'
- jh_cpu_idle="$(top -b -n 1 | grep Cpu | awk '{print $5}' | cut -f 1 -d .)"
- jh_cpu="$(echo 100 - $jh_cpu_idle|/usr/bin/bc)"
- #memory service
- alert_mem='100'
- jh_mem="$(/usr/bin/free -m|grep Mem|awk '{print $4}')"
- jh_swap="$(/usr/bin/free -m|grep Swap|awk '{print $3}')"
- alert_swap='0'
- #memory log
- memory_worklog='/usr/local/monitor/logs/mem_work_log'
- memory_downlog='/usr/local/monitor/logs/mem_down_log'
- #load service
- cpu_count="$(grep -c 'model name' /proc/cpuinfo)"
- alert_load="$(echo $cpu_count/2|/usr/bin/bc)"
- jh_load="$(uptime|awk '{print $NF}'|cut -f 1 -d .)"
- jh_load_15="$(uptime|awk '{print $NF}')"
- #load log
- load_worklog='/usr/local/monitor/logs/load_work_log'
- load_downlog='/usr/local/monitor/logs/load_down_log'
- #io service
- alert_io='80'
- jh_io_idle_back="$(/usr/bin/iostat|awk 'NR==4{print $NF}'|cut -f 1 -d .)"
- jh_io_idle="$(echo 100 - $jh_io_idle_back|/usr/bin/bc)"
- #io log
- io_worklog='/usr/local/monitor/logs/io_work_log'
- io_downlog='/usr/local/monitor/logs/io_down_log'
- #hardware service
- jh_hardware_error="$(dmesg|grep -i error|wc -l)"
- jh_info_error="$(dmesg|grep -i error)"
- #hardware error log
- hard_worklog='/usr/local/monitor/logs/hard_work_log'
- hard_downlog='/usr/local/monitor/logs/hard_down_log'
- #message service
- jh_message_error_count="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error|wc -l)"
- jh_message_error="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error)"
- #message error log
- message_worklog='/usr/local/monitor/logs/message_work_log'
- message_downlog='/usr/local/monitor/logs/message_down_log'
- #user service
- jh_user_count="$(/usr/bin/who|wc -l)"
- jh_user_info="$(/usr/bin/who)"
- #user log
- user_worklog='/usr/local/monitor/logs/user_work_log'
- user_downlog='/usr/local/monitor/logs/user_down_log'
- #disk service
- alert_disk='80'
- jh_disk="$(/bin/df -H|sort -k5nr|grep -vE 'Filesystem|tmpfs|cdrom'|awk '{print $5 " " $1}')"
- jh_disk_use=$(echo $jh_disk|awk '{print $1}'|cut -d '%' -f1)
- jh_disk_partition=$(echo $jh_disk|awk '{print $2}')
- #disk log
- disk_worklog='/usr/local/monitor/logs/disk_work_log'
- disk_downlog='/usr/local/monitor/logs/disk_down_log'
- #cpu log
- cpu_worklog='/usr/local/monitor/logs/cpu_work_log'
- cpu_downlog='/usr/local/monitor/logs/cpu_down_log'
- #notification mail
- email='denglei@ctfo.com'
- #monitor
- if [ ! -d "$loghere" ];then
- mkdir $loghere
- fi
- #monitor
- #jiaohuan check
- if [ $pas -ge 1 ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: pas Monitor_Server: $monitor_name is working" >> $worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','pas','working',now())";
- else
- /usr/local/lbs/bin4.0.7.7/pas -daemon
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: pas Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: pas was a problem" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: pas Monitor_Server: $montior_name is down" >> $downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','pas','downing',now())";
- fi
- if [ $ppas -ge 1 ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: ppas Monitor_Server: $monitor_name is working" >> $worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','ppas','working',now())";
- else
- /usr/local/lbs/bin4.0.7.7/ppas -daemon
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: ppas Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: ppas was a problem" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: ppas Monitor_Server: $montior_name is down" >> $downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','ppas','downing',now())";
- fi
- if [ $mas -ge 1 ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: mas Monitor_Server: $monitor_name is working" >> $worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','mas','working',now())";
- else
- /usr/local/lbs/bin4.0.7.7/mas -daemon
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: mas Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: mas was a problem" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: mas Monitor_Server: $montior_name is down" >> $downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','mas','downing',now())";
- fi
- if [ $mmas -ge 1 ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: mmas Monitor_Server: $monitor_name is working" >> $worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','mmas','working',now())";
- else
- /usr/local/lbs/bin4.0.7.7/mmas -daemon
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: mmas Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: mmas was a problem" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: mmas Monitor_Server: $montior_name is down" >> $downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','mmas','downing',now())";
- fi
- if [ $cas -ge 1 ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: cas Monitor_Server: $monitor_name is working" >> $worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','cas','working',now())";
- else
- /usr/local/lbs/bin4.0.7.7/cas -daemon
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: cas Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: cas was a problem" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: cas Monitor_Server: $montior_name is down" >> $downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$jh_name','$jh_ip','$monitor_name','cas','downing',now())";
- fi
- #check cpu_idle
- if [ $jh_cpu -ge $alert_cpu ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $jh_cpu"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: cpu_use was Exceed Threshold value: 80%" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $jh_cpu" >> $cpu_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$jh_name','$jh_ip','$monitor_name','cpu_use','$alert_cpu','$jh_cpu','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $jh_cpu" >> $cpu_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$jh_name','$jh_ip','$monitor_name','cpu_use','$alert_cpu','$jh_cpu','normal',now())";
- fi
- #check memory
- if [ $jh_mem -le $alert_mem ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: memory Monitor_Server: $monitor_name Free_mem: $jh_mem"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: memory was Exceed Threshold value" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: memory Monitor_Server: $monitor_name Free_mem: $jh_mem" >> $memory_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$jh_name','$jh_ip','$monitor_name','memory','$alert_mem','$jh_mem','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: memory Monitor_Server: $monitor_name Free_mem: $jh_mem" >> $memory_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$jh_name','$jh_ip','$monitor_name','memory','$alert_mem','$jh_mem','normal',now())";
- fi
- #check swap
- if [ $jh_swap -gt $alert_swap ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: swap Monitor_Server: $monitor_name Swap_jh: $jh_swap"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: swap Ip: $jh_ip was Exceed Threshold value" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: swap Monitor_Server: $monitor_name Swap_jh: $jh_swap" >> $memory_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$jh_name','$jh_ip','$monitor_name','swap','$alert_swap','$jh_swap','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: swap Monitor_Server: $monitor_name Swap_jh: $jh_swap" >> $memory_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$jh_name','$jh_ip','$monitor_name','swap','$alert_swap','$jh_swap','normal',now())";
- fi
- #check load_15
- if [ $jh_load -ge $alert_load ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $jh_load_15"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: load_15 Ip: $jh_ip was Exceed Threshold value" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $jh_load_15" >> $load_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$jh_name','$jh_ip','$monitor_name','load_15','$alert_load','$jh_load_15','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $server Ip: $ip Service: load Monitor_Server: $monitor_name Load_use: $jh_load_15" >> $load_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$jh_name','$jh_ip','$monitor_name','load_15','$alert_load','$jh_load_15','normal',now())";
- fi
- #check io_idle
- if [ $jh_io_idle -ge $alert_io ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $jh_io_idle"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: io_use was Exceed Threshold value: 80%" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $jh_io_idle" >> $io_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$jh_name','$jh_ip','$monitor_name','io_use','$alert_io','$jh_io_idle','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $jh_io_idle" >> $io_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$jh_name','$jh_ip','$monitor_name','io_use','$alert_io','$jh_io_idle','normal',now())";
- fi
- #check hareware error info
- if [ $jh_hardware_error -gt 0 ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: hardware_error Monitor_Server: $monitor_name Error: $jh_info_error"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: hardware_error were some hardware imformation error" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: hardware_error Monitor_Server: $monitor_name Error: $jh_info_error" >> $hard_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$jh_name','$jh_ip','$monitor_name','hardware_error','0','$jh_hardware_error','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: hardware_error Monitor_Server: $monitor_name Error: Nothing" >> $hard_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$jh_name','$jh_ip','$monitor_name','hardware_error','0','$jh_hardware_error','normal',now())";
- fi
- #check message error
- if [ $jh_message_error_count -ge 1 ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: message_error Monitor_Server: $monitor_name Message_error: $jh_message_error"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: message_error were some message imformation error" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: message_error Monitor_Server: $monitor_name Message_error: $jh_message_error" >> $message_downlog-$day
- /usr/bin/mysql-h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$jh_name','$jh_ip','$monitor_name','message_error','1','$jh_message_error_count','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: message_error Monitor_Server: $monitor_name Message_error: Nothing" >> $message_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$jh_name','$jh_ip','$monitor_name','message_error','1','$jh_message_error_count','normal',now())";
- fi
- #check user
- if [ $jh_user_count -ge 3 ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: user Monitor_Server: $monitor_name User: $jh_user_info"|/bin/mail -s "Notification ShengFen: $here Server: $jh_name Service: user was Exceed Threshold value: 3" $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: user Monitor_Server: $monitor_name User: $jh_user_info" >> $user_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$jh_name','$jh_ip','$monitor_name','user','3','$jh_user_count','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: user Monitor_Server: $monitor_name User: normal" >> $user_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$jh_name','$jh_ip','$monitor_name','user','3','$jh_user_count','normal',now())";
- fi
- #check disk
- if [ $jh_disk_use -ge $alert_disk ];then
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $jh_disk_partition ($jh_disk_use%)"|/bin/mail -s "Warning!!! $here Server: $jh_name Service: disk_use was Exceed Threshold value : $alert_disk% " $email
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $jh_disk_partion ($jh_disk_use%)" >> $disk_downlog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $disk_table values ('','$here','$jh_name','$jh_ip','$monitor_name','disk_use','$alert_disk','$jh_disk_partition','$jh_disk_use','abnormal',now())";
- else
- echo "$now ShengFen: $here Server: $jh_name Ip: $jh_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $jh_disk_partition ($jh_disk_use%)" >> $disk_worklog-$day
- /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $disk_table values ('','$here','$jh_name','$jh_ip','$monitor_name','disk_use','$alert_disk','$jh_disk_partition','$jh_disk_use','normal',now())";
- fi
本文转自 reinxu 51CTO博客，原文链接：http://blog.51cto.com/dl528888/1035131，如需转载请自行联系原作者
