git clone https://github.com/ClickHouse/clickhouse_exporter
cd clickhouse_exporter
go mod init
go mod vendor
go build 
ls ./clickhouse_exporter

export CLICKHOUSE_USER="user"
export CLICKHOUSE_PASSWORD="password"
nohup ./-scrape_uri=http://localhost:port/ >nohup.log 2>&1 &

> curl localhost:9116/metrics | head
# TYPE clickhouse_arena_alloc_bytes_total counter
clickhouse_arena_alloc_bytes_total 9.799096840192e+12
# HELP clickhouse_arena_alloc_chunks_total Number of ArenaAllocChunks total processed
# TYPE clickhouse_arena_alloc_chunks_total counter
clickhouse_arena_alloc_chunks_total 2.29782524e+08
# HELP clickhouse_background_move_pool_task Number of BackgroundMovePoolTask currently processed
# TYPE clickhouse_background_move_pool_task gauge
clickhouse_background_move_pool_task 0
# HELP clickhouse_background_pool_task Number of BackgroundPoolTask currently processed

git clone https://github.com/prometheus/node_exporter
make build
ls ./node_exporter

nohup ./node_exporter > nohup.log 2>&1 &

> curl localhost:9100/metrics
# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 6.3563e-05
go_gc_duration_seconds{quantile="0.25"} 7.4746e-05
go_gc_duration_seconds{quantile="0.5"} 9.0556e-05
go_gc_duration_seconds{quantile="0.75"} 0.000110677
go_gc_duration_seconds{quantile="1"} 0.004362325
go_gc_duration_seconds_sum 28.451282046
go_gc_duration_seconds_count 223479
...

global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  - ./rules/*.rules
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'clickhouse'
    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.
    static_configs:
    - targets: ['clickhouseexporter1:9116', 'clickhouseexporter2:9116', ...]

groups:
 - name: qps_too_high
   rules:
   - alert: clickhouse qps超出阈值
     expr: rate(clickhouse_query_total[1m]) > 100
     for: 2m
     labels:
      job: clickhouse-server
      severity: critical
      alertname: clickhouse qps超出阈值
     annotations:
      summary: "clickhouse qps超出阈值"
      description: "clickhouse qps超过阈值(100), qps: {{ $value }}"

nohup ./prometheus --config.file=/path/to/config --storage.tsdb.path=/path/to/storage --web.external-url=prometheus --web.enable-admin-api --web.enable-lifecycle --log.level=warn >nohup.log 2>&1 &

route:
  receiver: 'default'
  group_by: ['service','project']
receivers:
- name: "电话"
  webhook_configs:
  - url: <url>
- name: "企业微信"
  webhook_configs:
  - url: <url>
- name: "邮箱"
  webhook_configs:
  - url: <url>

nohup ./alertmanager --config.file=/path/to/config --log.level=warn >nohup.log 2>&1 &

SELECT 
    event_time, 
    user, 
    query_id AS query, 
    read_rows, 
    read_bytes, 
    result_rows, 
    result_bytes, 
    memory_usage, 
    exception
FROM clusterAllReplicas('cluster_name', system, query_log)
WHERE (event_date = today()) AND (event_time >= (now() - 60)) AND (is_initial_query = 1) AND (query NOT LIKE 'INSERT INTO%')
ORDER BY event_time DESC
LIMIT 100

SELECT 
    event_time, 
    user, 
    query_id AS query, 
    read_rows, 
    read_bytes, 
    result_rows, 
    result_bytes, 
    memory_usage, 
    exception
FROM clusterAllReplicas('cluster_name', system, query_log)
WHERE (event_date = yesterday()) AND query_duration_ms > 30000 AND (is_initial_query = 1) AND (query NOT LIKE 'INSERT INTO%')
ORDER BY query_duration_ms desc
LIMIT 100

SELECT 
    database, 
    table, 
    sum(bytes_on_disk) AS bytes_on_disk
FROM clusterAllReplicas('cluster_name', system, parts)
WHERE active AND (database != 'system')
GROUP BY 
    database, 
    table
ORDER BY bytes_on_disk DESC
LIMIT 10

SELECT 
    user, 
    count(1) AS query_times, 
    sum(read_bytes) AS query_bytes, 
    sum(read_rows) AS query_rows
FROM clusterAllReplicas('cluster_name', system, query_log)
WHERE (event_date = yesterday()) AND (is_initial_query = 1) AND (query NOT LIKE 'INSERT INTO%')
GROUP BY user
ORDER BY query_times DESC
LIMIT 10