prometheus 报警规则样例
groups:
- name: "带宽检测"
rules:
- alert: "带宽告警"
expr: (irate(node_network_transmit_bytes_total{device!~"lo"}[1m]) / 1000) > 2000000
for: 30s
annotations:
summary: "{{ $labels.job}} - {{ $labels.instance }} 平均带宽大于20M每秒"
description: "Prometheus 报警: 带宽大于20M每秒 \n 主机名: {{ $labels.hostname }}\n ip: {{ $labels.ip }}\n 当前值: {{ $value }} \n 应用: {{ $labels.app }} \n 可用区: {{$labels.region}} \n 产品线: {{ $labels.product }} \n"
groups:
- name: "cpu检测"
rules:
- alert: "cpu负载告警"
expr: (100-(avg(irate(node_cpu_seconds_total{mode="idle",job="node_exporter_alert"}[15m]))by (instance,hostname,region,app,product)) * 100) > 95
for: 5m
annotations:
value: " {{ $value }} "
summary: "{{ $labels.job}} - {{ $labels.instance }} CPU使用率高于95%"
description: "Prometheus 报警: cpu负载使用率超过98%\n 主机名: {{ $labels.hostname }}\n ip: {{ $labels.instance }}\n 当前值: {{ $value }} \n 应用: {{ $labels.app }} \n 可用区: {{$labels.region}} \n 产品线: {{ $labels.product }} \n"
groups:
- name: 磁盘报警
rules:
- alert: NodeDiskUsage
expr: (node_filesystem_size_bytes{job="node_exporter_alert"} - node_filesystem_avail_bytes{job="node_exporter_alert"}) / node_filesystem_size_bytes{job="node_exporter_alert"} * 100 > 85
for: 1m
labels:
severity: high
annotations:
value: " {{ $value }} "
description: "Prometheus 报警: 磁盘报警\n 主机名: {{ $labels.hostname }}\n ip: {{ $labels.ip }}\n 当前值: {{ $value }} \n 应用: {{ $labels.app }} \n 可用区: {{$labels.region}} \n 产品线: {{ $labels.product }} \n"
groups:
- name: "dnsmasq_check"
rules:
- alert: "dnsmasq check"
expr: namedprocess_namegroup_num_procs == 0
for: 10s
annotations:
summary: "dnsmasq down"
description: " dnsmasq , job: "
groups:
- name: dns-resolv-alarm
rules:
- alert: dns_resolv_error
expr: dns{job="pushgateway"} == 0
for: 1m
labels:
team: op
annotations:
summary: "[DNS解析报警] [{{$labels.exported_instance}}] 域名解析失败"
description: "[DNS解析报警] [{{$labels.exported_instance}}] 域名解析报警"
groups:
- name: "http检测规则"
rules:
- alert: "http服务检测"
expr: probe_success{job="blackbox-http"} == 0
for: 2m
labels:
severity: warning
annotations:
description: "Prometheus 报警: http检测\n 实例: {{ $labels.instance }}\n 当前值: {{ $value }} \n 应用: {{ $labels.app }} \n 产品线: {{ $labels.product }} \n"
groups:
- name: "java检测规则"
rules:
- alert: "java服务检测"
expr: count without (name,version)(irate(jmx_exporter_build_info[1m])) == 0
for: 1m
labels:
env: dev
annotations:
description: "java应用 job: {{ $labels.job }} \n instance: {{ $labels.instance }} "
summary: "java检测"
groups:
- name: "load 报警"
rules:
- alert: "load负载告警"
expr: node_load15{job="node_exporter_alert"} > 50
for: 1m
annotations:
description: "Prometheus 报警: load高于50\n 主机名: {{ $labels.hostname }}\n ip: {{ $labels.ip }}\n 当前值: {{ $value }} \n 应用: {{ $labels.app }} \n 可用区: {{$labels.region}} \n 产品线: {{ $labels.product }} \n"
groups:
- name: "logstash检测规则"
rules:
- alert: "logstash服务检测"
expr: logstash_info_node == 0
for: 1m
annotations:
description: "日志收集服务 , ip: {{ $labels.instance }} "
summary: "logstash检测"
groups:
- name: "内存检测"
rules:
- alert: "内存检测大于4G"
expr: 100 - ( node_memory_Cached_bytes{job="node_exporter_alert"} + node_memory_Buffers_bytes{job="node_exporter_alert"} + node_memory_MemFree_bytes{job="node_exporter_alert"} ) / (node_memory_MemTotal_bytes{job="node_exporter_alert"} > 5000000000 ) * 100 > 98
for: 30s
labels:
severity: critical
annotations:
description: "Prometheus 报警: 内存使用率超过98%\n 主机名: {{ $labels.hostname }}\n ip: {{ $labels.ip }}\n 当前值: {{ $value }} \n 应用: {{ $labels.app }} \n 可用区: {{$labels.region}} \n 产品线: {{ $labels.product }} \n"
- alert: "内存检测小于4G"
expr: 100 - ( node_memory_Cached_bytes{job="node_exporter_alert"} + node_memory_Buffers_bytes{job="node_exporter_alert"} + node_memory_MemFree_bytes{job="node_exporter_alert"} ) / (node_memory_MemTotal_bytes{job="node_exporter_alert"} <= 5000000000 ) * 100 > 98
for: 30s
labels:
severity: critical
annotations:
description: "Prometheus 报警: 内存使用率超过98%\n 主机名: {{ $labels.hostname }}\n ip: {{ $labels.ip }}\n 当前值: {{ $value }} \n 应用: {{ $labels.app }} \n 可用区: {{$labels.region}} \n 产品线: {{ $labels.product }}\n"
groups:
- name: "RedisClient"
rules:
- alert: "Clients20k"
expr: redis_connected_clients > 20000
for: 5m
labels:
severity: warning
annotations:
summary: "Redis 实例 {{ $labels.addr }} client 超 20k "
description: "Redis {{ $labels.instance }} client 超过 20k \n VALUE = {{ $value }}\n LABELS: {{ $labels }}"
groups:
- name: "RedisMem"
rules:
- alert: "OutOfMemory60"
expr: redis_memory_used_bytes / node_memory_MemTotal_bytes > 60
for: 5m
labels:
severity: warning
annotations:
summary: "Redis 实例 {{ $labels.addr }} 内存超过 60% "
description: "Redis {{ $labels.instance }} 内存超过 60% \n VALUE = {{ $value }}\n LABELS: {{ $labels }}"
- alert: "OutOfMemory70"
expr: redis_memory_used_bytes / node_memory_MemTotal_bytes > 70
for: 5m
labels:
severity: warning
annotations:
summary: "Redis 实例 {{ $labels.addr }} 内存超过 70% "
description: "Redis {{ $labels.instance }} 内存超过 70% \n VALUE = {{ $value }}\n LABELS: {{ $labels }}"
- alert: "OutOfMemory80"
expr: redis_memory_used_bytes / node_memory_MemTotal_bytes > 80
for: 5m
labels:
severity: warning
annotations:
summary: "Redis 实例 {{ $labels.addr }} 内存超过 80% "
description: "Redis {{ $labels.instance }} 内存超过 80% \n VALUE = {{ $value }}\n LABELS: {{ $labels }}"
- alert: "OutOfMemory90"
expr: redis_memory_used_bytes / node_memory_MemTotal_bytes > 90
for: 5m
labels:
severity: critical
annotations:
summary: "Redis 实例 {{ $labels.addr }} 内存超过 90% "
description: "Redis {{ $labels.instance }} 内存超过 90% \n VALUE = {{ $value }}\n LABELS: {{ $labels }}"
groups:
- name: "RedisUP"
rules:
- alert: "RedisDown"
expr: redis_up == 0
for: 1m
labels:
severity: warning
annotations:
summary: "Redis down"
description: " Redis {{ $labels.instance }} is down \n VALUE = {{ $value }}\n"
groups:
- name: "tcp服务检测"
rules:
- alert: "tcp服务检测"
expr: probe_success{job="blackbox-tcp"} == 0 or probe_success{job="redis-sync"} == 0
for: 1m
labels:
severity: warning
annotations:
value: " {{ $value }} "
description: "Prometheus 报警: tcp检测\n 实例: {{ $labels.instance }}\n 当前值: {{ $value }} \n 应用: {{ $labels.app }} \n 产品线: {{ $labels.product }} \n"
groups:
- name: zookeeperStatsAlert
rules:
- alert: 堆积请求数过大
expr: avg(zk_outstanding_requests) by (instance) > 10
for: 1m
labels:
severity: critical
annotations:
summary: "Instance {{ $labels.instance }} "
description: "积请求数过大"
- alert: 阻塞中的 sync 过多
expr: avg(zk_pending_syncs) by (instance) > 10
for: 1m
labels:
severity: critical
annotations:
summary: "Instance {{ $labels.instance }} "
description: "塞中的 sync 过多"
- alert: 平均响应延迟过高
expr: avg(zk_avg_latency) by (instance) > 10
for: 1m
labels:
severity: critical
annotations:
summary: "Instance {{ $labels.instance }} "
description: '平均响应延迟过高'
- alert: 打开文件描述符数大于系统设定的大小
expr: zk_open_file_descriptor_count > zk_max_file_descriptor_count * 0.85
for: 1m
labels:
severity: critical
annotations:
summary: "Instance {{ $labels.instance }} "
description: '打开文件描述符数大于系统设定的大小'
- alert: zookeeper服务器宕机
expr: zk_up == 0
for: 1m
labels:
severity: critical
annotations:
summary: "Instance {{ $labels.instance }} "
description: 'zookeeper服务器宕机'
- alert: zk主节点丢失
expr: absent(zk_server_state{state="leader"}) != 1
for: 1m
labels:
severity: critical
annotations:
summary: "Instance {{ $labels.instance }} "
description: 'zk主节点丢失'
