浏览器
  → Langfuse :3100     会话追踪、token、费用
  → Grafana  :3001     显存、温度、GPU 利用率、vLLM 请求
        → Prometheus :9090
              → DCGM Exporter :9400     读 GPU
              → vLLM :8000/metrics      读推理队列

#!/bin/bash
# 已在 10.0.1.47 执行。Langfuse 页面在 3100，因为 3000 是 New-API。
# .env 已存在时不要重新生成。再跑本脚本发现容器已在会直接退出。
# 不启动、不重启 vLLM、New-API、Dify。
set -eu

if docker ps --format '{{.Names}}' | grep -qx 'langfuse-langfuse-web-1'; then
  echo "Langfuse 已在运行，退出。"
  exit 0
fi

if [ ! -f /usr/cluster/observability/langfuse/.env ]; then
  echo "缺少 /usr/cluster/observability/langfuse/.env。不要另造一套密钥。"
  exit 1
fi

# docker.langfuse.com 直连会超时。现场先从镜像站拉取，再打回编排里的名字。
# minio 的镜像站被拒绝，用的是 docker.io/minio/minio:latest。
docker pull docker.m.daocloud.io/langfuse/langfuse:4
docker tag docker.m.daocloud.io/langfuse/langfuse:4 docker.langfuse.com/langfuse/langfuse:4
docker pull docker.m.daocloud.io/langfuse/langfuse-worker:4
docker tag docker.m.daocloud.io/langfuse/langfuse-worker:4 docker.langfuse.com/langfuse/langfuse-worker:4
docker pull docker.m.daocloud.io/clickhouse/clickhouse-server:25.12
docker tag docker.m.daocloud.io/clickhouse/clickhouse-server:25.12 docker.io/clickhouse/clickhouse-server:25.12
docker pull docker.m.daocloud.io/library/postgres:17
docker tag docker.m.daocloud.io/library/postgres:17 docker.io/postgres:17
docker pull docker.m.daocloud.io/library/redis:7
docker tag docker.m.daocloud.io/library/redis:7 docker.io/redis:7
docker pull docker.io/minio/minio:latest

cd /usr/cluster/observability/langfuse
docker compose up -d
docker compose ps

# Make sure to update the credential placeholders with your own secrets.
# We mark them with # CHANGEME in the file below.
# In addition, we recommend to restrict inbound traffic on the host to langfuse-web (port 3000) and minio (port 9090) only.
# All other components are bound to localhost (127.0.0.1) to only accept connections from the local machine.
# External connections from other machines will not be able to reach these services directly.
services:
  langfuse-worker:
    image: docker.langfuse.com/langfuse/langfuse-worker:4
    restart: always
    depends_on: &langfuse-depends-on
      postgres:
        condition: service_healthy
      minio:
        condition: service_healthy
      redis:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
    ports:
      - 127.0.0.1:3030:3030
    environment: &langfuse-worker-env
      NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000}
      DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/postgres} # CHANGEME
      SALT: ${SALT:-mysalt} # CHANGEME
      ENCRYPTION_KEY: ${ENCRYPTION_KEY:-0000000000000000000000000000000000000000000000000000000000000000} # CHANGEME: generate via `openssl rand -hex 32`
      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
      LANGFUSE_MIGRATION_V4_WRITE_MODE: ${LANGFUSE_MIGRATION_V4_WRITE_MODE:-dual}
      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
      # Lets the worker reach the web service internally for in-app agent MCP
      # calls without redirecting NEXTAUTH_URL, which also builds links for
      # users. Pair it with LANGFUSE_MCP_ALLOWED_HOSTS on langfuse-web.
      LANGFUSE_MCP_BASE_URL: ${LANGFUSE_MCP_BASE_URL:-}
      LANGFUSE_LLM_CONNECTION_WHITELISTED_HOST: ${LANGFUSE_LLM_CONNECTION_WHITELISTED_HOST:-}
      LANGFUSE_LLM_CONNECTION_WHITELISTED_IPS: ${LANGFUSE_LLM_CONNECTION_WHITELISTED_IPS:-}
      LANGFUSE_LLM_CONNECTION_WHITELISTED_IP_SEGMENTS: ${LANGFUSE_LLM_CONNECTION_WHITELISTED_IP_SEGMENTS:-}
      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
      CLICKHOUSE_CLUSTER_NAME: ${CLICKHOUSE_CLUSTER_NAME:-default}
      LANGFUSE_JSON_BAD_UNICODE_ESCAPE: ${LANGFUSE_JSON_BAD_UNICODE_ESCAPE:-}
      LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false}
      LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: ${LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE:-false}
      LANGFUSE_OCI_AUTH_TYPE: ${LANGFUSE_OCI_AUTH_TYPE:-workload_identity}
      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
      LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED: ${LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED:-false}
      LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES: ${LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES:-2097152}
      LANGFUSE_S3_BATCH_EXPORT_ENABLED: ${LANGFUSE_S3_BATCH_EXPORT_ENABLED:-false}
      LANGFUSE_S3_BATCH_EXPORT_BUCKET: ${LANGFUSE_S3_BATCH_EXPORT_BUCKET:-langfuse}
      LANGFUSE_S3_BATCH_EXPORT_PREFIX: ${LANGFUSE_S3_BATCH_EXPORT_PREFIX:-exports/}
      LANGFUSE_S3_BATCH_EXPORT_REGION: ${LANGFUSE_S3_BATCH_EXPORT_REGION:-auto}
      LANGFUSE_S3_BATCH_EXPORT_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_ENDPOINT:-http://minio:9000}
      LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT:-http://localhost:9090}
      LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID: ${LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID:-minio}
      LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY: ${LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
      LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE: ${LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE:-true}
      LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-}
      LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-}
      REDIS_HOST: ${REDIS_HOST:-redis}
      REDIS_PORT: ${REDIS_PORT:-6379}
      REDIS_AUTH: ${REDIS_AUTH:-myredissecret} # CHANGEME
      LANGFUSE_BULLMQ_SKIP_REDIS_VERSION_CHECK: ${LANGFUSE_BULLMQ_SKIP_REDIS_VERSION_CHECK:-false}
      REDIS_TLS_ENABLED: ${REDIS_TLS_ENABLED:-false}
      REDIS_TLS_CA: ${REDIS_TLS_CA:-/certs/ca.crt}
      REDIS_TLS_CERT: ${REDIS_TLS_CERT:-/certs/redis.crt}
      REDIS_TLS_KEY: ${REDIS_TLS_KEY:-/certs/redis.key}
      EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-}
      SMTP_CONNECTION_URL: ${SMTP_CONNECTION_URL:-}
      # Shared AI features configuration. These apply to AI features that run
      # in both web and worker.
      LANGFUSE_IN_APP_AGENT_ENABLED: ${LANGFUSE_IN_APP_AGENT_ENABLED:-}
      LANGFUSE_EVALUATOR_MEDIA_TRANSPORT: ${LANGFUSE_EVALUATOR_MEDIA_TRANSPORT:-}
      LANGFUSE_EVALUATOR_MEDIA_INLINE_MAX_BYTES: ${LANGFUSE_EVALUATOR_MEDIA_INLINE_MAX_BYTES:-}
      LANGFUSE_AI_PROVIDER: ${LANGFUSE_AI_PROVIDER:-}
      LANGFUSE_AI_MODEL: ${LANGFUSE_AI_MODEL:-}
      LANGFUSE_AI_SMALL_MODEL: ${LANGFUSE_AI_SMALL_MODEL:-}
      LANGFUSE_AI_API_KEY: ${LANGFUSE_AI_API_KEY:-}
      LANGFUSE_AI_BASE_URL: ${LANGFUSE_AI_BASE_URL:-}
      LANGFUSE_AI_USE_RESPONSES_API: ${LANGFUSE_AI_USE_RESPONSES_API:-}
      LANGFUSE_AI_EXTRA_HEADERS: ${LANGFUSE_AI_EXTRA_HEADERS:-}
      LANGFUSE_AI_AWS_BEDROCK_REGION: ${LANGFUSE_AI_AWS_BEDROCK_REGION:-}
      LANGFUSE_AI_VERTEX_LOCATION: ${LANGFUSE_AI_VERTEX_LOCATION:-}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
      AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-}
      AWS_PROFILE: ${AWS_PROFILE:-}
      LANGFUSE_AI_FEATURES_AWS_PROFILE: ${LANGFUSE_AI_FEATURES_AWS_PROFILE:-}
      LANGFUSE_IN_APP_AGENT_AWS_PROFILE: ${LANGFUSE_IN_APP_AGENT_AWS_PROFILE:-}
      # In-app agent worker settings (ignored by the web service).
      LANGFUSE_IN_APP_AGENT_RUN_QUEUE_PROCESSING_CONCURRENCY: ${LANGFUSE_IN_APP_AGENT_RUN_QUEUE_PROCESSING_CONCURRENCY:-}
      LANGFUSE_IN_APP_AGENT_SANDBOX_PROVIDER: ${LANGFUSE_IN_APP_AGENT_SANDBOX_PROVIDER:-}
      LANGFUSE_IN_APP_AGENT_SANDBOX_AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER: ${LANGFUSE_IN_APP_AGENT_SANDBOX_AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER:-}
      LANGFUSE_IN_APP_AGENT_SANDBOX_AWS_LAMBDA_MICROVM_EXECUTION_ROLE_ARN: ${LANGFUSE_IN_APP_AGENT_SANDBOX_AWS_LAMBDA_MICROVM_EXECUTION_ROLE_ARN:-}
      LANGFUSE_IN_APP_AGENT_SANDBOX_AWS_LAMBDA_MICROVM_EGRESS_NETWORK_CONNECTOR_ARN: ${LANGFUSE_IN_APP_AGENT_SANDBOX_AWS_LAMBDA_MICROVM_EGRESS_NETWORK_CONNECTOR_ARN:-}
      LANGFUSE_IN_APP_AGENT_SANDBOX_AWS_LAMBDA_MICROVM_REGION: ${LANGFUSE_IN_APP_AGENT_SANDBOX_AWS_LAMBDA_MICROVM_REGION:-}

  langfuse-web:
    image: docker.langfuse.com/langfuse/langfuse:4
    restart: always
    depends_on: *langfuse-depends-on
    ports:
      - "3100:3000"
    environment:
      <<: *langfuse-worker-env
      NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret} # CHANGEME
      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://localhost:9090}
      LANGFUSE_S3_MEDIA_UPLOAD_INTERNAL_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_INTERNAL_ENDPOINT:-http://minio:9000}
      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
      LANGFUSE_MARKDOWN_RENDER_CHARACTER_LIMIT: ${LANGFUSE_MARKDOWN_RENDER_CHARACTER_LIMIT:-}
      # Hostnames allowed to reach the MCP endpoint, beyond NEXTAUTH_URL's own
      # host. Required when the worker uses LANGFUSE_MCP_BASE_URL.
      LANGFUSE_MCP_ALLOWED_HOSTS: ${LANGFUSE_MCP_ALLOWED_HOSTS:-}
      # In-app agent settings.
      LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_USER: ${LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_USER:-}
      LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_ORG: ${LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_ORG:-}

  clickhouse:
    image: docker.io/clickhouse/clickhouse-server:25.12
    restart: always
    user: "101:101"
    environment:
      CLICKHOUSE_DB: default
      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
    volumes:
      - langfuse_clickhouse_data:/var/lib/clickhouse
      - langfuse_clickhouse_logs:/var/log/clickhouse-server
    healthcheck:
      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
      interval: 5s
      timeout: 5s
      retries: 30
      start_period: 40s

  minio:
    image: docker.io/minio/minio:latest
    restart: always
    entrypoint: sh
    # create the 'langfuse' bucket before starting the service
    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-miniosecret} # CHANGEME
    ports:
      - "9092:9000"
    volumes:
      - langfuse_minio_data:/data
    healthcheck:
      test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"]
      interval: 5s
      timeout: 5s
      retries: 20
      start_period: 15s

  redis:
    image: docker.io/redis:7
    restart: always
    # CHANGEME: row below to secure redis password
    command: >
      --requirepass ${REDIS_AUTH:-myredissecret}
      --maxmemory-policy noeviction
    environment:
      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
    volumes:
      - langfuse_redis_data:/data
    healthcheck:
      test: ["CMD-SHELL", "redis-cli -a \"$$REDIS_AUTH\" ping | grep PONG"]
      interval: 3s
      timeout: 10s
      retries: 10

  postgres:
    image: docker.io/postgres:${POSTGRES_VERSION:-17}
    restart: always
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 3s
      timeout: 3s
      retries: 10
    environment:
      POSTGRES_USER: ${POSTGRES_USER:-postgres}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} # CHANGEME
      POSTGRES_DB: ${POSTGRES_DB:-postgres}
      TZ: UTC
      PGTZ: UTC
    volumes:
      - langfuse_postgres_data:/var/lib/postgresql/data

volumes:
  langfuse_postgres_data:
    driver: local
  langfuse_clickhouse_data:
    driver: local
  langfuse_clickhouse_logs:
    driver: local
  langfuse_minio_data:
    driver: local
  langfuse_redis_data:
    driver: local

# 现场核对过的 Langfuse 环境项。完整文件在服务器 /usr/cluster/observability/langfuse/.env，权限 600。
# 密钥、数据库口令、项目私钥只留在那份文件里。下面不抄具体值。已存在时不要重新生成，否则旧追踪解不开。

COMPOSE_PROJECT_NAME=langfuse
NEXTAUTH_URL=http://10.0.1.47:3100
TELEMETRY_ENABLED=false
POSTGRES_USER=postgres
POSTGRES_DB=postgres
CLICKHOUSE_USER=clickhouse
MINIO_ROOT_USER=minio
LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://10.0.1.47:9092
LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT=http://10.0.1.47:9092
LANGFUSE_INIT_ORG_ID=e1e3900e-7a13-4952-955c-ac8223343c04
LANGFUSE_INIT_ORG_NAME=内网大模型
LANGFUSE_INIT_PROJECT_ID=955cd00a-2949-4d51-8dbb-7b824bbbb420
LANGFUSE_INIT_PROJECT_NAME=会话追踪
LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-lf-d7a2e5c771eb542a
LANGFUSE_INIT_USER_EMAIL=admin@example.com
LANGFUSE_INIT_USER_NAME=admin
LANGFUSE_MIGRATION_V4_WRITE_MODE=dual

# 下列项已在服务器 .env 中生成，本文只列名字：
# NEXTAUTH_SECRET SALT ENCRYPTION_KEY
# POSTGRES_PASSWORD DATABASE_URL
# CLICKHOUSE_PASSWORD CLICKHOUSE_MIGRATION_URL CLICKHOUSE_URL
# REDIS_AUTH MINIO_ROOT_PASSWORD
# LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY
# LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY
# LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY
# LANGFUSE_INIT_PROJECT_SECRET_KEY LANGFUSE_INIT_USER_PASSWORD

#!/bin/bash
# 已在 10.0.1.47 执行。Grafana 在 3001，Prometheus 在 9090，DCGM 在 9400。
# 再跑本脚本发现 prometheus 容器已在会直接退出。
# 不重启 vLLM。DCGM 只读显卡，不占用算力配额。
set -eu

if docker ps --format '{{.Names}}' | grep -qx 'prometheus'; then
  echo "Prometheus 已在运行，退出。"
  exit 0
fi

docker pull docker.m.daocloud.io/prom/prometheus:v3.2.1
docker tag docker.m.daocloud.io/prom/prometheus:v3.2.1 prom/prometheus:v3.2.1
docker pull docker.m.daocloud.io/grafana/grafana:11.5.2
docker tag docker.m.daocloud.io/grafana/grafana:11.5.2 grafana/grafana:11.5.2
docker pull docker.m.daocloud.io/nvidia/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
docker tag docker.m.daocloud.io/nvidia/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04 nvidia/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04

cd /usr/cluster/observability/metrics
docker compose up -d
docker compose ps

name: observability

services:
  prometheus:
    image: prom/prometheus:v3.2.1
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    extra_hosts:
      - "host.docker.internal:host-gateway"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus_data:/prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=15d

  grafana:
    image: grafana/grafana:11.5.2
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3001:3000"
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: Gf-gpu-2026
      GF_USERS_ALLOW_SIGN_UP: "false"
      GF_SERVER_ROOT_URL: http://10.0.1.47:3001
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
      - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro

  dcgm-exporter:
    image: nvidia/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
    container_name: dcgm-exporter
    restart: unless-stopped
    cap_add:
      - SYS_ADMIN
    ports:
      - "9400:9400"
    environment:
      NVIDIA_VISIBLE_DEVICES: all
      NVIDIA_DRIVER_CAPABILITIES: compute,utility
      DCGM_EXPORTER_INTERVAL: "5000"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

volumes:
  prometheus_data:
  grafana_data:

global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets: ["localhost:9090"]

  - job_name: dcgm
    static_configs:
      - targets: ["dcgm-exporter:9400"]

  - job_name: vllm
    metrics_path: /metrics
    static_configs:
      - targets: ["host.docker.internal:8000"]

apiVersion: 1

datasources:
  - name: Prometheus
    uid: prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: false

apiVersion: 1

providers:
  - name: local
    orgId: 1
    folder: 监控
    type: file
    disableDeletion: false
    updateIntervalSeconds: 30
    options:
      path: /var/lib/grafana/dashboards

{
  "uid": "gpu-vllm",
  "title": "GPU 与 vLLM",
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "15s",
  "time": { "from": "now-1h", "to": "now" },
  "panels": [
    {
      "id": 1,
      "type": "stat",
      "title": "GPU 利用率",
      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 } },
      "targets": [{ "expr": "DCGM_FI_DEV_GPU_UTIL", "refId": "A" }]
    },
    {
      "id": 2,
      "type": "stat",
      "title": "显存已用",
      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "decmbytes" } },
      "targets": [{ "expr": "DCGM_FI_DEV_FB_USED", "refId": "A" }]
    },
    {
      "id": 3,
      "type": "stat",
      "title": "显存空闲",
      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "decmbytes" } },
      "targets": [{ "expr": "DCGM_FI_DEV_FB_FREE", "refId": "A" }]
    },
    {
      "id": 4,
      "type": "stat",
      "title": "GPU 温度",
      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "celsius" } },
      "targets": [{ "expr": "DCGM_FI_DEV_GPU_TEMP", "refId": "A" }]
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "GPU 利用率",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 } },
      "targets": [{ "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "GPU {{gpu}}", "refId": "A" }]
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "显存占用",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "decmbytes" } },
      "targets": [
        { "expr": "DCGM_FI_DEV_FB_USED", "legendFormat": "已用 {{gpu}}", "refId": "A" },
        { "expr": "DCGM_FI_DEV_FB_FREE", "legendFormat": "空闲 {{gpu}}", "refId": "B" }
      ]
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "温度与显存带宽利用率",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short" } },
      "targets": [
        { "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "温度 {{gpu}}", "refId": "A" },
        { "expr": "DCGM_FI_DEV_MEM_COPY_UTIL", "legendFormat": "显存带宽 {{gpu}}", "refId": "B" }
      ]
    },
    {
      "id": 8,
      "type": "timeseries",
      "title": "vLLM 请求与 KV 占用",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": { "defaults": { "unit": "short" } },
      "targets": [
        { "expr": "vllm:num_requests_running", "legendFormat": "运行中", "refId": "A" },
        { "expr": "vllm:num_requests_waiting", "legendFormat": "排队", "refId": "B" },
        { "expr": "vllm:kv_cache_usage_perc", "legendFormat": "KV 占用", "refId": "C" }
      ]
    }
  ]
}