#!/bin/bash
# Wait until the single-node install finishes, then move the LLM containers
# onto that node. Model files and database directories stay on disk.
set -u
export KUBECONFIG=/root/.kube/config
export DEBIAN_FRONTEND=noninteractive
MAN=/usr/cluster/k8s/manifests
STOPPED=/usr/cluster/k8s/stopped-containers.txt
log() { echo "=== $* ==="; }

check_health() {
  local label=$1 ok=1 u code
  for u in \
    http://127.0.0.1:3000/api/status \
    http://127.0.0.1:8000/health \
    http://127.0.0.1:8088/ \
    http://127.0.0.1:3100/api/public/health \
    http://127.0.0.1:3001/api/health \
    http://127.0.0.1:9090/-/ready \
    http://127.0.0.1:8090/ \
    http://127.0.0.1:9997/v1/models
  do
    code=$(curl -s -o /dev/null -m 8 -w '%{http_code}' "$u" || echo err)
    echo "HEALTH $label $code $u"
    case "$code" in
      200|301|302|307|308) ;;
      *) ok=0 ;;
    esac
  done
  if [ "$ok" -ne 1 ]; then
    return 1
  fi
  echo "HEALTH_OK $label"
}

rollback() {
  log "rollback to the stopped containers"
  kubectl delete -f "$MAN/llm/workloads.json" --ignore-not-found || true
  kubectl delete -f "$MAN/dify/workloads.json" --ignore-not-found || true
  kubectl delete -f "$MAN/langfuse/workloads.json" --ignore-not-found || true
  kubectl delete -f "$MAN/observability/workloads.json" --ignore-not-found || true
  sleep 5
  if [ -f "$STOPPED" ]; then
    while read -r name; do
      [ -n "$name" ] || continue
      docker start "$name" || true
    done < "$STOPPED"
  fi
  echo ROLLBACK_DONE
}

log "wait for the node installer"
while pgrep -f '/usr/cluster/k8s/install-node.sh' >/dev/null; do
  sleep 15
done
sleep 3

ready=$(kubectl get node prod102 -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
echo "node_ready=$ready"
if [ "$ready" != "True" ]; then
  echo NODE_NOT_READY
  kubectl get nodes -o wide || true
  journalctl -u kubelet -n 40 --no-pager || true
  exit 1
fi
kubectl get pods -A -o wide

log "gpu check"
cat > /usr/cluster/k8s/gpu-check.yaml << 'EOF'
apiVersion: v1
kind: Pod
metadata:
  name: gpu-check
  namespace: default
spec:
  restartPolicy: Never
  nodeName: prod102
  containers:
    - name: nvidia
      image: docker.m.daocloud.io/vllm/vllm-openai:latest
      imagePullPolicy: IfNotPresent
      command: ["nvidia-smi", "-L"]
      env:
        - name: NVIDIA_VISIBLE_DEVICES
          value: all
EOF
kubectl delete pod gpu-check --ignore-not-found
kubectl apply -f /usr/cluster/k8s/gpu-check.yaml
gpu_ok=0
for i in $(seq 1 60); do
  phase=$(kubectl get pod gpu-check -o jsonpath='{.status.phase}' 2>/dev/null || true)
  echo "gpu_phase $phase"
  if [ "$phase" = "Succeeded" ]; then
    gpu_ok=1
    break
  fi
  if [ "$phase" = "Failed" ]; then
    break
  fi
  sleep 5
done
kubectl logs gpu-check || true
if [ "$gpu_ok" != "1" ]; then
  kubectl describe pod gpu-check || true
  echo GPU_CHECK_FAILED
  exit 1
fi
kubectl delete pod gpu-check --ignore-not-found

log "save langfuse tool patch"
mkdir -p /usr/cluster/k8s/patches
trace_path=$(docker exec dify-worker-1 find /app -name langfuse_trace.py 2>/dev/null | head -1 || true)
if [ -n "$trace_path" ]; then
  docker cp "dify-worker-1:$trace_path" /usr/cluster/k8s/patches/langfuse_trace.py
  printf '%s' "$trace_path" > /usr/cluster/k8s/patches/langfuse_trace.path
  echo "patch $trace_path"
else
  echo "patch not found"
fi

log "render manifests"
python3 /usr/cluster/k8s/render.py

python3 - << 'PY'
from pathlib import Path
p = Path("/usr/cluster/observability/metrics/prometheus.yml")
text = p.read_text(encoding="utf-8")
text = text.replace("host.docker.internal:8000", "10.0.1.47:8000")
p.write_text(text, encoding="utf-8")
print("prometheus target updated")
PY

log "apply namespaces secrets services"
kubectl apply -f "$MAN/namespaces.json"
for ns in dify langfuse observability llm; do
  kubectl apply -f "$MAN/$ns/secrets.json"
  kubectl apply -f "$MAN/$ns/services.json"
done

python3 - << 'PY'
import json, subprocess
from pathlib import Path
rewrites = [line.split() for line in Path("/usr/cluster/k8s/manifests/dns-rewrites.txt").read_text().splitlines() if line.strip()]
raw = subprocess.check_output(["kubectl","get","cm","coredns","-n","kube-system","-o","json"], text=True)
cm = json.loads(raw)
core = cm["data"]["Corefile"]
fresh = []
for src, dst in rewrites:
    rule = "    rewrite name %s %s" % (src, dst)
    if rule not in core:
        fresh.append(rule)
if fresh:
    needle = "    prometheus"
    if needle not in core:
        needle = "    cache"
    core = core.replace(needle, "\n".join(fresh) + "\n" + needle, 1)
    cm["data"]["Corefile"] = core
    Path("/tmp/coredns-cm.json").write_text(json.dumps(cm), encoding="utf-8")
    subprocess.check_call(["kubectl","apply","-f","/tmp/coredns-cm.json"])
    subprocess.check_call(["kubectl","rollout","restart","deployment/coredns","-n","kube-system"])
    print("coredns rewrites", len(fresh))
else:
    print("coredns rewrites already present")
PY
kubectl rollout status deployment/coredns -n kube-system --timeout=120s

log "stop compose containers"
docker ps --format '{{.Names}}' | grep -v '^k8s_' > "$STOPPED"
mapfile -t names < "$STOPPED"
others=()
for name in "${names[@]}"; do
  if [ "$name" = "vllm-qwen3.6-27b" ]; then
    continue
  fi
  others+=("$name")
done
if [ "${#others[@]}" -gt 0 ]; then
  docker stop -t 25 "${others[@]}"
fi
docker stop -t 90 vllm-qwen3.6-27b || true

python3 - << 'PY'
import sqlite3
con = sqlite3.connect("/opt/new-api/data/one-api.db")
cur = con.cursor()
cur.execute("select id, base_url from channels")
changed = 0
for cid, url in cur.fetchall():
    if url and "172.17.0.1" in url:
        new = url.replace("172.17.0.1", "10.0.1.47")
        cur.execute("update channels set base_url=? where id=?", (new, cid))
        print("channel", cid, new)
        changed += 1
con.commit()
print("channels_updated", changed)
PY

log "start workloads"
for ns in dify langfuse observability llm; do
  kubectl apply -f "$MAN/$ns/workloads.json"
done

log "wait for new-api and dify"
na=0
dy=0
for i in $(seq 1 48); do
  c1=$(curl -s -o /dev/null -m 8 -w '%{http_code}' http://127.0.0.1:3000/api/status || echo err)
  c2=$(curl -s -o /dev/null -m 8 -w '%{http_code}' http://127.0.0.1:8088/ || echo err)
  echo "newapi $c1 dify $c2"
  [ "$c1" = "200" ] && na=1
  case "$c2" in
    200|301|302|307|308) dy=1 ;;
  esac
  if [ "$na" = "1" ] && [ "$dy" = "1" ]; then
    break
  fi
  sleep 10
done
if [ "$na" != "1" ] || [ "$dy" != "1" ]; then
  kubectl get pods -A -o wide || true
  rollback
  exit 1
fi

log "wait for vllm"
ok=0
for i in $(seq 1 45); do
  code=$(curl -s -o /dev/null -m 8 -w '%{http_code}' http://127.0.0.1:8000/health || echo err)
  echo "vllm $code"
  if [ "$code" = "200" ]; then
    ok=1
    break
  fi
  sleep 20
done
if [ "$ok" -ne 1 ]; then
  kubectl logs -n llm deploy/vllm --tail=60 || true
  echo VLLM_NOT_READY
  exit 1
fi

if ! check_health final; then
  kubectl get pods -A -o wide || true
  echo MIGRATE_PARTIAL
  exit 1
fi
kubectl get pods -A -o wide
echo MIGRATE_OK