Docker 容器监控

FROM ubuntu:latest
LABEL cadvisor 0.46.0
COPY ./cadvisor-v0.46.0-linux-amd64 /usr/bin/cadvisor
RUN chmod +x /usr/bin/cadvisor
ENTRYPOINT ["/usr/bin/cadvisor"]

docker run \
--volume=/:/rootfs:ro \
--volume=/var/run:/var/run:ro \
--volume=/sys:/sys:ro \
--volume=/var/lib/docker/:/var/lib/docker:ro \
--volume=/dev/disk/:/dev/disk:ro \
--publish=8080:8080 \
--detach=true \
--name=cadvisor \
--userns=host \
--privileged \
--device=/dev/kmsg \
cadvisor:0.46.0

docker run \
--volume=/:/rootfs:ro \
--volume=/var/run:/var/run:ro \
--volume=/sys:/sys:ro \
--volume=/var/lib/docker/:/var/lib/docker:ro \
--volume=/dev/disk/:/dev/disk:ro \
--publish=8080:8080 \
--detach=true \
--name=cadvisor \
--userns=host \
--privileged \
--device=/dev/kmsg \
cadvisor:0.46.0 -disable_metrics cpu,cpuLoad

# 安装Node Exporter 来收集硬件信息
docker run -d \
--net="host" \
--pid="host" \
--userns="host" \
-v "/:/host:ro,rslave" \
--name node_exporter \
quay.io/prometheus/node-exporter:latest \
--path.rootfs=/host

docker run -d \
--net="host" \
--pid="host" \
--userns="host" \
-v "/:/host:ro,rslave" \
--name node_exporter \
quay.io/prometheus/node-exporter:latest \
--path.rootfs=/host \
--collector.disable-defaults \
--collector.arp --collector.bcache

global:
# 每20s获取一次数据指标
scrape_interval: 20s
# 获取数据超时时长 10s
scrape_timeout: 10s
# 规则评估评率，即计算指标是否有触发规则的计算频率
evaluation_interval: 20s
# 规则文件，从所有匹配的文件中读取规则和警报
rule_files:
- "alertRule.yml"
- "recordRule.yml"
# 采集配置列表
scrape_configs:
- job_name: 'cadvisor'
static_configs:
- targets:
- 192.168.239.149:8080
- job_name: 'node'
static_configs:
- targets:
- 192.168.239.149:9100
- 192.168.239.142:9100
- 192.168.239.143:9100
- job_name: 'prometheus'
static_configs:
- targets:
- 192.168.239.149:9090
# 报警管理
alerting:
alertmanagers:
- static_configs:
- targets: ['192.168.239.149:9093']

docker run -itd --name prometheus -p 9090:9090 \
-v /opt/prometheus:/etc/prometheus \
prom/prometheus --config.file=/etc/prometheus/prometheus.yml

roups:
- name: RecordCpu
rules:
- record: Cpu15mRate
expr: 100- sum(increase(node_cpu_seconds_total{mode="idle"}[15m])) by
(instance)/sum(increase(node_cpu_seconds_total[15m])) by (instance) * 100
labels:
CpuRate: 15

groups:
# 组名
- name: node_health
# 规则
rules:
# 报警名称
- alert: InstanceDown
# 基于PromQL的条件表达式
expr: up == 0
# 评估等待时间，表示，触发条件表达式后，等待一段时间发送报警信息
for: 1m
# 自定义label 标签
labels:
NodeHealth: false
# 附加信息，比如详细的描述报警情况
annotations:
# 摘要
summary: "Instance {{ $labels.instance }} down"
# 详情
description: " {{ $labels.instance }} of job {{ $labels.job }} has been
down for more than 1 minutes "
- name: node_resource
rules:
- alert: Cpu5mRate
expr: 100- sum(increase(node_cpu_seconds_total{mode="idle"}[5m])) by
(instance)/sum(increase(node_cpu_seconds_total[5m])) by (instance) * 100 > 2
labels:
CpuRate: hight
annotations:
# 摘要
summary: "Instance {{ $labels.instance }} 5分钟CPU使用率过高"
# 详情
description: " {{ $labels.instance }} of job {{ $labels.job }} 5分钟CPU使用
率过高 "
- alert: NodeLoad15
expr: node_load15 > 0.8
labels:
NodeLoad15: hight
annotations:
# 摘要
summary: "Instance {{ $labels.instance }} 15分钟平均负载过高请留意"
# 详情
description: " {{ $labels.instance }} of job {{ $labels.job }} 15分钟平均负载
过高 "
- alert: MemRate
expr: (node_memory_MemTotal_bytes - (node_memory_MemFree_bytes +
node_memory_Buffers_bytes+node_memory_Cached_bytes ))/node_memory_MemTotal_bytes
* 100 > 20
labels:
NodeMemRate: hight
annotations:
# 摘要
summary: "机器内存使用率过高"
# 详情
description: "机器内存使用率超过20%，请留意"
- alert: DiskRate
expr: node_filesystem_avail_bytes{mountpoint="/"} /
node_filesystem_size_bytes{mountpoint="/"} * 100 > 80
labels:
DiskRate: hight
annotations:
# 摘要
summary: "机器磁盘使用率过高"
# 详情
description: "机器磁盘使用率超过80%，请留意"

# 报警管理
alerting:
alertmanagers:
- static_configs:
- targets: ['192.168.239.149:9093']