Docker的监控-Prometheus(普罗米修斯)

docker run -it --rm --name sysdig --privileged=true \
--volume=/var/run/docker.sock:/host/var/run/docker.sock \
--volume=/dev:/host/dev --volume=/proc:/host/proc:ro \
--volume=/boot:/host/boot:ro --volume=/lib/modules:/host/lib/modules:ro \
--volume=/usr:/host/usr:ro sysdig/sysdig
//如果下载插件失败，可以运行下边的命令，重新下载
sysdig-probe-loader
//下载成功之后，可以运行sysdig命令，查看监控项
csysdig

sudo curl -L https://github.com/weaveworks/scope/releases/download/latest_release/scope -o /usr/local/bin/scope
sudo chmod a+x /usr/local/bin/scope
scope launch -app.basicAuth -app.basicAuth.password 123456 -app.basicAuth.username user -probe.basicAuth -probe.basicAuth.password 123456 -probe.basicAuth.username user

[root@docker01 ~]# scope launch 192.168.8.10 192.168.8.20
1c4f97ee6dc2543e32961bd7be155fda256797532a6787807c28c666f9612ab7
Scope probe started
Weave Scope is listening at the following URL(s):
  * http://172.19.0.1:4040/
  * http://172.20.0.1:4040/
  * http://192.168.10.100:4040/
  * http://172.22.16.1:4040/
  * http://192.168.122.1:4040/
root@docker02 ~]# scope launch 192.168.8.20 192.168.8.10
3d1f206dda3f5a0aa476b646eccb86814bb48611b11aac771ba60b7e5476875c
Scope probe started
Weave Scope is listening at the following URL(s):
  * http://192.168.10.101:4040/
  * http://192.168.122.1:4040/

//部署node-EXporter,收集硬件和系统信息。
docker run -d -p 9100:9100 --name exporter -v /proc:/host/proc -v /sys:/host/sys -v /:/rootfs --net=host prom/node-exporter --path.procfs /host/proc --path.sysfs /host/sys --collector.filesystem.ignored-mount-points "^/(sys|proc|dev|host|etc)($|/)"
PS： 注意，这里使用了--net=host，这样Prometheus Server可以直接与Node-EXporter通信。
验证：打开浏览器验证结果。
http://192.168.8.10:9100
//部署安装cAdvisor，收集节点容器信息。
docker run -v /:/rootfs:ro -v /var/run:/var/run/:rw -v /sys:/sys:ro -v /var/lib/docker:/var/lib/docker:ro -p 8080:8080 --detach=true --name=cadvisor --net=host google/cadvisor
部署完成之后，打开浏览器验证。
http://192.168.8.10:8080
##### 2）在docker01上部署Prometheus Server服务。
在部署prometheus之前，我们需要对它的配置文件进行修改，所以我们先运行一个容器，先将其配置文件拷贝出来。
docker run -d -p 9090:9090 --name prometheus --net=host prom/prometheus
docker cp prometheus:/etc/prometheus/prometheus.yml /root
vim prometheus.yml
     - targets:['localhost:9090','localhost:8080','localhost:9100','192.168.8.20:8080','192.168.8.20:9100','192.168.8.30:8080','192.168.8.30:9100']
PS: 这里指定了prometheus的监控项，包括它也会监控自己收集到的数据。
修改配置文件后删除掉前面开启的prometheus容器，因为我们已经得到配置文件并按照实际修改了，后面需要重新开启新的容器
docker rm -f prometheus  容器名
//重新运行prometheus容器。
docker run -d -p 9090:9090 --name prometheus --net=host -v /root/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus
浏览器访问验证。
http://192.168.8.10:9090/targets
PS： 这里能够查看到我们各个监控项。
##### 3）在docker01上，部署grafana服务，用来展示prometheus收集到的数据。
mkdir /grafana-storage 
chmod 777 -R /grafana-storage/
docker run -d -p 3000:3000 --name grafana -v /grafana-storage:/var/lib/grafana -e "GF_SECURITY_ADMIN_PASSWORD=123.com" grafana/grafana

[root@docker1 ~]# vim alertmanager.yml
global:
  resolve_timeout: 5m
  smtp_from: '1793594335@qq.com'
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_auth_username: '1793594335@qq.com'
  smtp_auth_password: 'yuemqwmhdizdbjeg'
  smtp_require_tls: false
  smtp_hello: 'qq.com'
route:
  group_by: ['alertname']
  group_wait: 5s
  group_interval: 5s
  repeat_interval: 5m
  receiver: 'email'
receivers:
- name: 'email'
  email_configs:
  - to: '1793594335@qq.com'
    send_resolved: true
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

[root@docker1 ~]# docker rm -f alertmanager 
[root@docker01 ~]# docker run -d --name alertmanager -p 9093:9093 -v /root/alertmanager.yml:/etc/alertmanager/alertmanager.yml prom/alertmanager
### Prometheus 配置 AlertManager 告警规则

mkdir  -p /root/prometheus/rules
cd /root/prometheus/rules/
[root@docker1 rules]# vim node-up.rules 
groups:
- name: node-up
  rules:
  - alert: node-up
    expr: up{job="prometheus"} == 0
    for: 15s
    labels:
      severity: 1
      team: node
    annotations:
      summary: "{{ $labels.instance }} 已停止运行超过 15s！"

# vim /root/prometheus.yml
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - 192.168.8.10:9093
rule_files:
  - "/usr/local/prometheus/rules/*.rules"

[root@docker1 ~]# docker rm -f prometheus 
[root@docker1 ~]# docker run -d -p 9090:9090 -v /root/prometheus.yml:/etc/prometheus/prometheus.yml -v /root/prometheus/rules:/usr/local/prometheus/rules --name prometheus --net=host prom/prometheus

vim node_alerts.yml
groups:
- name: 主机状态-监控告警
  rules:
  - alert: 主机状态
    expr: up {job="kubernetes-nodes"} == 0
    for: 15s
    labels:
      status: 非常严重
    annotations:
      summary: "{{.instance}}:服务器宕机"
      description: "{{.instance}}:服务器延时超过15s"
  - alert: CPU使用情况
    expr: 100-(avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by(instance)* 100) > 60
    for: 1m
    labels:
      status: warning
    annotations:
      summary: "{{$labels.instance}}: High CPU Usage Detected"
      description: "{{$labels.instance}}: CPU usage is {{$value}}, above 60%"
  - alert: NodeFilesystemUsage
    expr: 100 - (node_filesystem_free_bytes{fstype=~"ext4|xfs"} / node_filesystem_size_bytes{fstype=~"ext4|xfs"} * 100) > 80
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{ $labels.instance }} : {{ $labels.mountpoint }} 分区使用率过高"
      description: "{{ $labels.instance }}: {{ $labels.mountpoint }} 分区使用大于80% (当前值: {{ $value }})"
  - alert: 内存使用
    expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 80
    for: 1m
    labels:
      status: 严重告警
    annotations:
      summary: "{{ $labels.instance}} 内存使用率过高！"
      description: "{{ $labels.instance }} 内存使用大于80%(目前使用:{{ $value}}%)"
  - alert: IO性能
    expr: (avg(irate(node_disk_io_time_seconds_total[1m])) by(instance)* 100) > 60
    for: 1m
    labels:
      status: 严重告警
    annotations:
      summary: "{{$labels.instance}} 流入磁盘IO使用率过高！"
      description: "{{ $labels.instance }} 流入磁盘IO大于60%(目前使用:{{ $value }})"
  - alert: 网络
    expr: ((sum(rate (node_network_receive_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance)) / 100) > 102400
    for: 1m
    labels:
      status: 严重告警
    annotations:
      summary: "{{ $labels.instance}} 流入网络带宽过高！"
      description: "{{ $labels.instance }}流入网络带宽持续2分钟高于100M. RX带宽使用率{{ $value }}"
  - alert: TCP会话
    expr: node_netstat_Tcp_CurrEstab > 1000
    for: 1m
    labels:
      status: 严重告警
    annotations:
      summary: "{{ $labels.instance }} TCP_ESTABLISHED过高！"
      description: "{{ $labels.instance }} TCP_ESTABLISHED大于1000%(目前使用:{{ $value }}%)"
vim pod_rules.yml
groups:
- name: k8s_pod.rules
  rules:
  - alert: pod-status
    expr: kube_pod_container_status_running != 1
    for: 5s
    labels:
      severity: warning
    annotations:
      description : pod-{{ $labels.pod }}故障
      summary: pod重启告警
  - alert: Pod_all_cpu_usage
    expr: (sum by(name)(rate(container_cpu_usage_seconds_total{image!=""}[5m]))*100) > 10
    for: 5m
    labels:
      severity: critical
      service: pods
    annotations:
      description: 容器 {{ $labels.name }} CPU 资源利用率大于 75% , (current value is {{ $value }})
      summary: Dev CPU 负载告警
  - alert: Pod_all_memory_usage
    expr: sort_desc(avg by(name)(irate(container_memory_usage_bytes{name!=""}[5m]))*100) > 1024*10^3*2
    for: 10m
    labels:
      severity: critical
    annotations:
      description: 容器 {{ $labels.name }} Memory 资源利用率大于 2G , (current value is {{ $value }})
      summary: Dev Memory 负载告警
  - alert: Pod_all_network_receive_usage
    expr: sum by (name)(irate(container_network_receive_bytes_total{container_name="POD"}[1m])) > 1024*1024*50
    for: 10m
    labels:
      severity: critical
    annotations:
      description: 容器 {{ $labels.name }} network_receive 资源利用率大于 50M , (current value is {{ $value }})
      summary: network_receive 负载告警
微信报警：
https://blog.csdn.net/m0_37680131/article/details/120090880