Prometheus 性能调优 - 什么是高基数问题以及如何解决?

topk(10, count by (__name__)({__name__=~".+"}))
PROMQL

count(count by (label_name) (metric_name))
PROMQL

count(count by (url) (apiserver_request_duration_seconds_bucket))
PROMQL

# 找到 Prometheus 的 SVC ClusterIP
kubectl get svc -n cattle-monitoring-system
export url=http://10.43.85.24:9090
export now=$(date +%s)
curl -s $url/api/v1/label/__name__/values \
| jq -r ".data[]" \
| while read metric; do
    count=$(curl -s \
        --data-urlencode 'query=count({__name__="'$metric'"})' \
        --data-urlencode "time=$now" \
        $url/api/v1/query \
    | jq -r ".data.result[0].value[1]")
    echo "$count $metric"
done
BASH

export metric=rest_client_request_duration_seconds_bucket
curl -s \
    --data-urlencode "query=$metric" \
    --data-urlencode "time=$now" \
    $url/api/v1/query \
| jq -c ".data.result[].metric"
BASH

curl -s $url/api/v1/label/__name__/values | jq -r ".data[]" | sort
BASH

curl -s $url/api/v1/labels \
| jq -r ".data[]" \
| while read label; do
    count=$(curl -s $url/api/v1/label/$label/values \
    | jq -r ".data|length")
    echo "$count $label"
  done \
| sort -n
BASH

remoteWrite:
- url: "<Your Metrics instance remote_write endpoint>"
  basicAuth:
    username:
      name: your_grafanacloud_secret
      key: your_grafanacloud_secret_username_key
    password:
      name: your_grafanacloud_secret
      key: your_grafanacloud_secret_password_key
  writeRelabelConfigs:
  - sourceLabels:
    - "__name__"
    regex: "apiserver_request_total|kubelet_node_config_error|kubelet_runtime_operations_errors_total|kubeproxy_network_programming_duration_seconds_bucket|container_cpu_usage_seconds_total|kube_statefulset_status_replicas|kube_statefulset_status_replicas_ready|node_namespace_pod_container:container_memory_swap|kubelet_runtime_operations_total|kube_statefulset_metadata_generation|node_cpu_seconds_total|kube_pod_container_resource_limits_cpu_cores|node_namespace_pod_container:container_memory_cache|kubelet_pleg_relist_duration_seconds_bucket|scheduler_binding_duration_seconds_bucket|container_network_transmit_bytes_total|kube_pod_container_resource_requests_memory_bytes|namespace_workload_pod:kube_pod_owner:relabel|kube_statefulset_status_observed_generation|process_resident_memory_bytes|container_network_receive_packets_dropped_total|kubelet_running_containers|kubelet_pod_worker_duration_seconds_bucket|scheduler_binding_duration_seconds_count|scheduler_volume_scheduling_duration_seconds_bucket|workqueue_queue_duration_seconds_bucket|container_network_transmit_packets_total|rest_client_request_duration_seconds_bucket|node_namespace_pod_container:container_memory_rss|container_cpu_cfs_throttled_periods_total|kubelet_volume_stats_capacity_bytes|kubelet_volume_stats_inodes_used|cluster_quantile:apiserver_request_duration_seconds:histogram_quantile|kube_node_status_allocatable_memory_bytes|container_memory_cache|go_goroutines|kubelet_runtime_operations_duration_seconds_bucket|kube_statefulset_replicas|kube_pod_owner|rest_client_requests_total|container_memory_swap|node_namespace_pod_container:container_memory_working_set_bytes|storage_operation_errors_total|scheduler_e2e_scheduling_duration_seconds_bucket|container_network_transmit_packets_dropped_total|kube_pod_container_resource_limits_memory_bytes|node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate|storage_operation_duration_seconds_count|node_netstat_TcpExt_TCPSynRetrans|node_netstat_Tcp_OutSegs|container_cpu_cfs_periods_total|kubelet_pod_start_duration_seconds_count|kubeproxy_network_programming_duration_seconds_count|container_network_receive_bytes_total|node_netstat_Tcp_RetransSegs|up|storage_operation_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_count|kubelet_volume_stats_available_bytes|scheduler_scheduling_algorithm_duration_seconds_bucket|kube_statefulset_status_replicas_current|code_resource:apiserver_request_total:rate5m|kube_statefulset_status_replicas_updated|process_cpu_seconds_total|kube_pod_container_resource_requests_cpu_cores|kubelet_pod_worker_duration_seconds_count|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pleg_relist_duration_seconds_count|kubeproxy_sync_proxy_rules_duration_seconds_bucket|container_memory_usage_bytes|workqueue_adds_total|container_network_receive_packets_total|container_memory_working_set_bytes|kube_resourcequota|kubelet_running_pods|kubelet_volume_stats_inodes|kubeproxy_sync_proxy_rules_duration_seconds_count|scheduler_scheduling_algorithm_duration_seconds_count|apiserver_request:availability30d|container_memory_rss|kubelet_pleg_relist_interval_seconds_bucket|scheduler_e2e_scheduling_duration_seconds_count|scheduler_volume_scheduling_duration_seconds_count|workqueue_depth|:node_memory_MemAvailable_bytes:sum|volume_manager_total_volumes|kube_node_status_allocatable_cpu_cores"
    action: "keep"
YAML

write_relabel_configs:
  - source_labels: [__name__]
    regex: "apiserver_request_duration_seconds_bucket"
    action: drop
YAML

groups:
  - interval: 3m
    name: kube-apiserver-availability.rules
    rules:
      - expr: >-
          avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) *
          24 * 30
        record: code_verb:apiserver_request_total:increase30d
      - expr: >-
          sum by (cluster, code, verb)
          (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h]))
        record: code_verb:apiserver_request_total:increase1h
      - expr: >-
          sum by (cluster, code, verb)
          (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
        record: code_verb:apiserver_request_total:increase1h
YAML

write_relabel_configs:
  - source_labels: [__name__]
    regex: "apiserver_request_duration_seconds_bucket"
    action: drop
YAML