彻底搞懂监控系统，使用Prometheus和Grafana 如何实现运维告警？

[smtp]
enabled = true
host = smtp.163.com:25
user = xxx@163.com  # 邮件地址
# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
password = xxx    # 授权码
cert_file =
key_file =
skip_verify = true
from_address = xxx@163.com  # 发件人地址
from_name = Grafana
ehlo_identity = dashboard.example.com
[emails]
welcome_email_on_sign_up = false
templates_pattern = emails/*.html
content_types = text/html

# 解压到/usr/local/prometheus目录下：
tar -zxvf alertmanager-0.24.0.linux-amd64.tar.gz -C /usr/local/prometheus
# 修改目录名：
cd /usr/local/prometheus
mv alertmanager-0.24.0.linux-amd64 alertmanager-0.24.0

global:
  resolve_timeout: 5m  # alertmanager在持续多久没有收到新告警后标记为resolved
  smtp_from: 'xxx@163.com' # 发件人邮箱地址
  smtp_smarthost: 'smtp.163.com:25' # 邮箱smtp地址
  smtp_auth_username: 'xxx@163.com' # 发件人的登陆用户名，默认和发件人地址一致
  smtp_auth_password: 'xxx'  # 发件人的登陆密码，有时候是授权码
  smtp_hello: '163.com'
  smtp_require_tls:  # 是否需要tls协议。默认是true
route:
  group_by: [alertname] # 通过alertname的值对告警进行分类
  group_wait: 10s # 一组告警第一次发送之前等待的时延，即产生告警10s将组内新产生的消息合并发送，通常是0s~几分钟（默认是30s）
  group_interval: 2m # 一组已发送过初始告警通知的告警，接收到新告警后，下次发送通知前等待时延，通常是5m或更久（默认是5m）
  repeat_interval: 5m # 一组已经发送过通知的告警，重复发送告警的间隔，通常设置为3h或者更久（默认是4h）
  receiver: 'default-receiver' # 设置告警接收人
receivers:
- name: 'default-receiver'
  email_configs:
  - to: 'xxx@163.com'
    send_resolved: true # 发送恢复告警通知
inhibit_rules: # 抑制规则
  - source_match:  # 源匹配级别，当匹配成功发出通知，其他级别产生告警将被抑制
      severity: 'critical' # 告警时间级别（告警级别根据规则自定义）
    target_match: 
      severity: 'warning' # 匹配目标成功后，新产生的目标告警为'warning'将被抑制
    equal: ['alertname','dev','instance'] # 基于这些标签抑制匹配告警的级别

#alertmanager
cd /usr/local/prometheus/alertmanager-0.24.0
#执行启动命令，指定数据访问的url
alertmanager --config.file=/usr/local/prometheus/alertmanager-0.24.0/alertmanager.yml

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets: [10.2.1.231:9093]
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
# 加载指定的规则文件
rule_files:
  - "first.rules"
  - "rules/*.yml"

groups:
  - name: alertmanager_pod.rules
    rules:
    - alert: Pod_all_cpu_usage
      expr: (sum by(name)(rate(container_cpu_usage_seconds_total{image!=""}[5m]))*100) > 1
      for: 2m
      labels:
        serverity: critical
        service: pods
      annotations:
        description: 容器 {{ $labels.name }} CPU 资源利用率大于 10% , (current value is {{ $value }})
        summary: Dev CPU 负载告警
    - alert: Pod_all_memory_usage
      expr: sort_desc(avg by(name)(irate(node_memory_MemFree_bytes {name!=""}[5m]))) > 2147483648 # 内存大于2G
      for: 2m
      labels:
        severity: critical
      annotations:
        description: 容器 {{ $labels.name }} Memory 资源利用大于 2G , (current value is {{ $value }})
        summary: Dev Memory 负载告警
    - alert: Pod_all_network_receive_usage
      expr: sum by(name)(irate(container_network_reveive_bytes_total{container_name="POD"}[1m])) > 52428800 # 大于50M
      for: 2m
      labels:
        severity: critical
      annotations:
        description: 容器 {{ $labels.name }} network_receive 资源利用大于 50M , (current value is {{ $value }})
    - alert: node内存可用大小
      expr: node_memory_MemFree_bytes < 5368709120  # 内存小于5G
      for: 2m
      labels:
        severity: critical
      annotations:
        description: node可用内存小于5G