Prometheus的使用总结

# my global config
global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).
# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      # - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'game'
    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.
    static_configs:
    - targets: ['10.10.20.89:8088']
      labels:
          instance: 'Allen大区'      
    - targets: ['10.10.20.18:8088']
      labels:
          instance: '黄豆本地服'

scrape_configs:
  - job_name: 'java'
    static_configs:
    - targets: ['<host>:<port>']

nohup ./prometheus  > prometheus.log 2>&1 &

/usr/local/Prometheus/prometheus --config.file=/usr/local/Prometheus/prometheus.yml &

# 配置抓取请求的TLS设置
tls_config:
  [ <tls_config> ]
# 可选的代理URL
[ proxy_url: <string> ]
# Consul服务发现配置列表
consul_sd_configs:
  [ - <consul_sd_config> ... ]
# DNS服务发现配置列表
dns_sd_configs:
  [ - <dns_sd_config> ... ]
# 亚马逊EC2服务发现的配置列表
ec2_sd_configs:
  [ - <ec2_sd_config> ... ]
# 文件服务发现配置列表
file_sd_configs:
  [ - <file_sd_config> ... ]
# google GCE服务发现配置列表
gce_sd_configs:
  [ - <gce_sd_config> ... ]
# Kubernetes服务发现配置列表
kubernetes_sd_configs:
  [ - <kubernetes_sd_config> ... ]
# Zookeeper服务发现配置列表
serverset_sd_configs:
  [ - <serverset_sd_config> ... ]
# 静态配置目标列表
static_configs:
  [ - <static_config> ... ]

api group: 'io.prometheus', name: 'simpleclient', version: '0.12.0';
    api group: 'io.prometheus', name: 'simpleclient_hotspot', version: '0.12.0';
    api group: 'io.prometheus', name: 'simpleclient_httpserver', version: '0.12.0';
    api group: 'io.prometheus', name: 'simpleclient_pushgateway', version: '0.12.0';

java -server -Dfile.encoding=UTF-8  -Xms3072m -Xmx3072m -Xmn1024m -javaagent:/cnc/run/common/jmx_prometheus_javaagent-0.13.0.jar=8088:/cnc/run/common/prometheus-jmx-config.yaml -XX:+UseG1GC -XX:-OmitStackTraceInFastThrow -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=../log/game_dump_202109261447.dat -Xloggc:../log/1.1.2.1/gc_202109261447.log -jar /cnc/run/gamesvr/gamioo-game.jar

private Counter metrics = Counter.build().name("game_command_total").help("game command statistics").labelNames("userId", "msgName").register();
//埋点
public void incr(String userId, String msgName) {
  metrics.labels(userId, msgName).inc();
}

data source:
game_command_total{instance="10.10.20.89:8088",job="game"}

increase(game_command_all_total{job="$job",instance=~"$instance"}[$__interval])

data = pd.read_csv(
    'https://labfile.oss.aliyuncs.com/courses/1283/adult.data.csv')
print(data.head())

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: game-statics   
  namespace: gamioo-demo   
spec:
  endpoints:
  - interval: 30s
    # 填写service yaml中Prometheus Exporter对应的Port的Name
    port: jmx-metrics
    # 填写Prometheus Exporter对应的Path的值，不填默认/metrics
    path: /metrics
    relabelings:
    # ** 必须要有一个 label 为 application，这里假设 k8s 有一个 label 为 app，
    # 我们通过 relabel 的 replace 动作把它替换成了 application
    - action: replace
      sourceLabels:  [__meta_kubernetes_pod_label_app]
      targetLabel: application
  # 选择要监控service所在的namespace
  namespaceSelector:
    matchNames:
    - gamioo-demo
  # 填写要监控service的Label值，以定位目标service
  selector:
    matchLabels:
      app: cnc

apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: game-pod-statics 
  namespace: gamioo-demo
spec:
  podMetricsEndpoints:
  - interval: 30s
    port: jmx-metrics  # 填写pod yaml中Prometheus Exporter对应的Port的Name
    path: /metrics  # 填写Prometheus Exporter对应的Path的值，不填默认/metrics
    relabelings:
    - action: replace
      sourceLabels:
      - __meta_kubernetes_pod_node_name
      targetLabel: instance
  namespaceSelector:   # 选择要监控pod所在的namespace
    matchNames:
    - gamioo-demo
  selector:    # 填写要监控pod的Label值，以定位目标pod
    matchExpressions:
    - key: app.kubernetes.io/name
      operator: In
      values:
      - cnc-gamesvr
      - cnc-dirsvr

PromQL ：sum(increase(game_command_all_total[1m]))  by (stream,pod,instance) >1000
告警内容：Pod {{ $labels.pod }}[{{ $labels.instance}}]  :游戏消息{{$labels.stream}}频率过高,达到了{{ printf "%.0f" $value }},阈值为1000

{{ $var := .externalURL}}{{ range $k,$v:=.alerts }}
{{if eq $v.status "resolved"}}
[恢复信息]()
>告警标题: {{$v.annotations.summary}}
>告警级别: {{$v.labels.level}}
开始时间: {{GetCSTtime $v.startsAt}}
结束时间: {{GetCSTtime $v.endsAt}}
故障主机IP: {{$v.labels.instance}}
**{{$v.annotations.description}}**
{{else}}
[告警信息]()
>告警标题: {{$v.annotations.summary}}
>告警级别: {{$v.labels.level}}
开始时间:{{GetCSTtime $v.startsAt}}
故障主机IP: {{$v.labels.instance}}
**{{$v.annotations.description}}**
{{end}}
{{ end }}

sum by(stream, pod, instance) (increase(game_command_all_total[1m]))