feat(health): stage Contabo production migration (#360)

Adds a manual-only Contabo canary path, Traefik overlays, fail-closed migration gates, and contract tests.

Refs lingjingwanxiang/ljwx-health#390
Refs lingjingwanxiang/ljwx-health#395

.github/workflows/deploy-repo-gate.yml

@@ -114,6 +114,8 @@ jobs:
         run: |
           uvx --with pyyaml python -m unittest \
             scripts.tests.test_ljwx_health_failure_domain_scheduling -v
+          uvx --with pyyaml python -m unittest \
+            scripts.tests.test_ljwx_health_contabo_migration -v
 
       - name: Validate LJWX Health schema rollout
         run: |
@@ -121,6 +123,8 @@ jobs:
           uvx --with pyyaml python -m unittest scripts.tests.test_ljwx_health_schema_restore_drill -v
           shellcheck apps/ljwx-health/schema-restore-drill/scripts/*.sh
           kubectl kustomize apps/ljwx-health/overlays/k3s-eu >/tmp/ljwx-health-k3s-eu.yaml
+          kubectl kustomize apps/ljwx-health/overlays/contabo-prod >/tmp/ljwx-health-contabo-prod.yaml
+          kubectl kustomize apps/ljwx-health/overlays/contabo-prod-staging >/tmp/ljwx-health-contabo-prod-staging.yaml
 
       - name: Detect pgvector related changes
         id: pgvector

apps/ljwx-health/overlays/contabo-prod-staging/deployment-admin-staging.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ljwx-admin
+  namespace: ljwx-health
+spec:
+  template:
+    metadata:
+      annotations:
+        ops.ljwx.io/migration-stage: contabo-canary
+    spec:
+      containers:
+        - name: ljwx-admin
+          env:
+            - name: VITE_API_URL
+              value: https://health-admin-contabo-canary.lingjingwanxiang.cn/api

apps/ljwx-health/overlays/contabo-prod-staging/deployment-bigscreen-staging.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ljwx-bigscreen
+  namespace: ljwx-health
+spec:
+  template:
+    metadata:
+      annotations:
+        ops.ljwx.io/migration-stage: contabo-canary
+    spec:
+      containers:
+        - name: ljwx-bigscreen
+          env:
+            - name: BIGSCREEN_CUTOVER_ADMIN_BASE_URL
+              value: https://health-dashboard-contabo-canary.lingjingwanxiang.cn

apps/ljwx-health/overlays/contabo-prod-staging/deployment-boot-staging.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ljwx-boot
+  namespace: ljwx-health
+spec:
+  template:
+    metadata:
+      annotations:
+        ops.ljwx.io/migration-stage: contabo-canary
+    spec:
+      containers:
+        - name: ljwx-boot
+          env:
+            - name: LJWX_HEALTH_OUTBOX_KAFKA_ENABLED
+              value: "false"
+            - name: LJWX_HEALTH_OUTBOX_KAFKA_ALLOWED_CUSTOMER_IDS
+              value: ""
+            - name: LJWX_HEALTH_REALTIME_SCORE_ENABLED
+              value: "false"
+            - name: LJWX_HEALTH_REALTIME_SCORE_CUSTOMERS
+              value: ""

apps/ljwx-health/overlays/contabo-prod-staging/ingress-canary-patch.yaml

@@ -0,0 +1,69 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ljwx-health-traefik
+  namespace: ljwx-health
+spec:
+  $patch: replace
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - health-admin-contabo-canary.lingjingwanxiang.cn
+        - health-dashboard-contabo-canary.lingjingwanxiang.cn
+      secretName: ljwx-health-contabo-canary-tls
+  rules:
+    - host: health-admin-contabo-canary.lingjingwanxiang.cn
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-admin
+                port:
+                  number: 80
+    - host: health-dashboard-contabo-canary.lingjingwanxiang.cn
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-admin
+                port:
+                  number: 80
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ljwx-health-api-traefik
+  namespace: ljwx-health
+spec:
+  $patch: replace
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - health-admin-contabo-canary.lingjingwanxiang.cn
+        - health-dashboard-contabo-canary.lingjingwanxiang.cn
+      secretName: ljwx-health-contabo-canary-tls
+  rules:
+    - host: health-admin-contabo-canary.lingjingwanxiang.cn
+      http:
+        paths:
+          - path: /api
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-boot
+                port:
+                  number: 9998
+    - host: health-dashboard-contabo-canary.lingjingwanxiang.cn
+      http:
+        paths:
+          - path: /api
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-boot
+                port:
+                  number: 9998

apps/ljwx-health/overlays/contabo-prod-staging/kustomization.yaml

@@ -0,0 +1,213 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: ljwx-health
+resources:
+  - ../contabo-prod
+patches:
+  - target:
+      version: v1
+      kind: ConfigMap
+      namespace: ljwx-health
+      name: ljwx-health-cloudflared-config
+    patch: |-
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: ljwx-health-cloudflared-config
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      namespace: ljwx-health
+      name: ljwx-health-cloudflared
+    patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: ljwx-health-cloudflared
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      version: v1
+      kind: Service
+      namespace: ljwx-health
+      name: ljwx-health-cloudflared
+    patch: |-
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: ljwx-health-cloudflared
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: policy
+      version: v1
+      kind: PodDisruptionBudget
+      namespace: ljwx-health
+      name: ljwx-health-cloudflared
+    patch: |-
+      apiVersion: policy/v1
+      kind: PodDisruptionBudget
+      metadata:
+        name: ljwx-health-cloudflared
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: monitoring.coreos.com
+      version: v1
+      kind: ServiceMonitor
+      namespace: ljwx-health
+      name: ljwx-health-cloudflared
+    patch: |-
+      apiVersion: monitoring.coreos.com/v1
+      kind: ServiceMonitor
+      metadata:
+        name: ljwx-health-cloudflared
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: networking.k8s.io
+      version: v1
+      kind: NetworkPolicy
+      namespace: ljwx-health
+      name: ljwx-health-cloudflared
+    patch: |-
+      apiVersion: networking.k8s.io/v1
+      kind: NetworkPolicy
+      metadata:
+        name: ljwx-health-cloudflared
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: batch
+      version: v1
+      kind: Job
+      namespace: ljwx-health
+      name: health-shadow-kafka-init
+    patch: |-
+      apiVersion: batch/v1
+      kind: Job
+      metadata:
+        name: health-shadow-kafka-init
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: batch
+      version: v1
+      kind: Job
+      namespace: ljwx-health
+      name: health-shadow-clickhouse-init
+    patch: |-
+      apiVersion: batch/v1
+      kind: Job
+      metadata:
+        name: health-shadow-clickhouse-init
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      namespace: ljwx-health
+      name: health-writer
+    patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: health-writer
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      namespace: ljwx-health
+      name: health-recon
+    patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: health-recon
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      version: v1
+      kind: Service
+      namespace: ljwx-health
+      name: health-writer
+    patch: |-
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: health-writer
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      version: v1
+      kind: Service
+      namespace: ljwx-health
+      name: health-recon
+    patch: |-
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: health-recon
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: monitoring.coreos.com
+      version: v1
+      kind: ServiceMonitor
+      namespace: ljwx-health
+      name: ljwx-health-shadow
+    patch: |-
+      apiVersion: monitoring.coreos.com/v1
+      kind: ServiceMonitor
+      metadata:
+        name: ljwx-health-shadow
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: monitoring.coreos.com
+      version: v1
+      kind: PrometheusRule
+      namespace: ljwx-health
+      name: ljwx-health-shadow
+    patch: |-
+      apiVersion: monitoring.coreos.com/v1
+      kind: PrometheusRule
+      metadata:
+        name: ljwx-health-shadow
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: monitoring.coreos.com
+      version: v1
+      kind: PrometheusRule
+      namespace: ljwx-health
+      name: ljwx-health-slo
+    patch: |-
+      apiVersion: monitoring.coreos.com/v1
+      kind: PrometheusRule
+      metadata:
+        name: ljwx-health-slo
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      version: v1
+      kind: ConfigMap
+      namespace: ljwx-health
+      name: ljwx-health-shadow-clickhouse-schema
+    patch: |-
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: ljwx-health-shadow-clickhouse-schema
+        namespace: ljwx-health
+      $patch: delete
+  - path: ingress-canary-patch.yaml
+  - path: deployment-boot-staging.yaml
+  - path: deployment-admin-staging.yaml
+  - path: deployment-bigscreen-staging.yaml

apps/ljwx-health/overlays/contabo-prod/cloudflared-config-patch.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ljwx-health-cloudflared-config
+data:
+  config.yaml: |
+    tunnel: 70479f3e-caaf-4528-b713-413cc27e8ad0
+    credentials-file: /etc/cloudflared/credentials/credentials.json
+    metrics: 0.0.0.0:2000
+    loglevel: info
+    ingress:
+      - hostname: health-admin.omniverseai.net
+        service: http://traefik.kube-system.svc.cluster.local:80
+        originRequest:
+          httpHostHeader: health-admin.omniverseai.net
+          connectTimeout: 10s
+      - hostname: health-dashboard.omniverseai.net
+        service: http://traefik.kube-system.svc.cluster.local:80
+        originRequest:
+          httpHostHeader: health-dashboard.omniverseai.net
+          connectTimeout: 10s
+      - service: http_status:404

apps/ljwx-health/overlays/contabo-prod/cloudflared-networkpolicy-patch.yaml

@@ -0,0 +1,37 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ljwx-health-cloudflared
+spec:
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik
+      ports:
+        - protocol: TCP
+          port: 80
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 10.0.0.0/8
+              - 172.16.0.0/12
+              - 192.168.0.0/16
+      ports:
+        - protocol: TCP
+          port: 7844
+        - protocol: UDP
+          port: 7844

apps/ljwx-health/overlays/contabo-prod/ingress-api-traefik.yaml

@@ -0,0 +1,55 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ljwx-health-api-traefik
+  annotations:
+    cert-manager.io/cluster-issuer: dnspod-letsencrypt
+    traefik.ingress.kubernetes.io/router.middlewares: ljwx-health-health-api-chain@kubernetescrd
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - health-admin.lingjingwanxiang.cn
+        - health-dashboard.lingjingwanxiang.cn
+      secretName: ljwx-health-contabo-prod-tls
+  rules:
+    - host: health-admin.lingjingwanxiang.cn
+      http:
+        paths:
+          - path: /api
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-boot
+                port:
+                  number: 9998
+    - host: health-dashboard.lingjingwanxiang.cn
+      http:
+        paths:
+          - path: /api
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-boot
+                port:
+                  number: 9998
+    - host: health-admin.omniverseai.net
+      http:
+        paths:
+          - path: /api
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-boot
+                port:
+                  number: 9998
+    - host: health-dashboard.omniverseai.net
+      http:
+        paths:
+          - path: /api
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-boot
+                port:
+                  number: 9998

apps/ljwx-health/overlays/contabo-prod/ingress-networkpolicy-patch.yaml

@@ -0,0 +1,28 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ljwx-health-ingress
+  namespace: ljwx-health
+spec:
+  ingress:
+    - from:
+        - podSelector: {}
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: infra
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: ingress-nginx
+              app.kubernetes.io/component: controller
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: observability
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: prometheus

apps/ljwx-health/overlays/contabo-prod/ingress-traefik.yaml

@@ -0,0 +1,55 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ljwx-health-traefik
+  annotations:
+    cert-manager.io/cluster-issuer: dnspod-letsencrypt
+    traefik.ingress.kubernetes.io/router.middlewares: ljwx-health-health-cors@kubernetescrd
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - health-admin.lingjingwanxiang.cn
+        - health-dashboard.lingjingwanxiang.cn
+      secretName: ljwx-health-contabo-prod-tls
+  rules:
+    - host: health-admin.lingjingwanxiang.cn
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-admin
+                port:
+                  number: 80
+    - host: health-dashboard.lingjingwanxiang.cn
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-admin
+                port:
+                  number: 80
+    - host: health-admin.omniverseai.net
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-admin
+                port:
+                  number: 80
+    - host: health-dashboard.omniverseai.net
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: ljwx-admin
+                port:
+                  number: 80

apps/ljwx-health/overlays/contabo-prod/kustomization.yaml

@@ -0,0 +1,38 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: ljwx-health
+resources:
+  - ../k3s-eu
+  - middleware-traefik.yaml
+  - ingress-traefik.yaml
+  - ingress-api-traefik.yaml
+patches:
+  - target:
+      group: networking.k8s.io
+      version: v1
+      kind: Ingress
+      namespace: ljwx-health
+      name: ljwx-health-ingress
+    patch: |-
+      apiVersion: networking.k8s.io/v1
+      kind: Ingress
+      metadata:
+        name: ljwx-health-ingress
+        namespace: ljwx-health
+      $patch: delete
+  - target:
+      group: networking.k8s.io
+      version: v1
+      kind: Ingress
+      namespace: ljwx-health
+      name: ljwx-health-api-ingress
+    patch: |-
+      apiVersion: networking.k8s.io/v1
+      kind: Ingress
+      metadata:
+        name: ljwx-health-api-ingress
+        namespace: ljwx-health
+      $patch: delete
+  - path: cloudflared-config-patch.yaml
+  - path: cloudflared-networkpolicy-patch.yaml
+  - path: ingress-networkpolicy-patch.yaml

apps/ljwx-health/overlays/contabo-prod/middleware-traefik.yaml

@@ -0,0 +1,52 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: health-cors
+spec:
+  headers:
+    accessControlAllowCredentials: true
+    accessControlAllowHeaders:
+      - DNT
+      - X-CustomHeader
+      - Keep-Alive
+      - User-Agent
+      - X-Requested-With
+      - If-Modified-Since
+      - Cache-Control
+      - Content-Type
+      - Authorization
+    accessControlAllowMethods:
+      - GET
+      - POST
+      - PUT
+      - PATCH
+      - DELETE
+      - OPTIONS
+    accessControlAllowOriginList:
+      - https://health-admin.lingjingwanxiang.cn
+      - https://health-dashboard.lingjingwanxiang.cn
+      - https://health-admin.omniverseai.net
+      - https://health-dashboard.omniverseai.net
+      - https://health-admin-contabo-canary.lingjingwanxiang.cn
+      - https://health-dashboard-contabo-canary.lingjingwanxiang.cn
+    accessControlMaxAge: 600
+    addVaryHeader: true
+---
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: health-strip-api
+spec:
+  stripPrefix:
+    prefixes:
+      - /api
+---
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: health-api-chain
+spec:
+  chain:
+    middlewares:
+      - name: health-strip-api
+      - name: health-cors

cluster-prod/kustomization.yaml

@@ -7,4 +7,5 @@ resources:
   - registry-pull-secret-sync-rbac.yaml
   - registry-pull-secret-sync-cronjob.yaml
   - website-prod-application.yaml
+  - ljwx-health-prod-application.yaml
   - deploy-promoter-prod-cronjob.yaml

cluster-prod/ljwx-health-prod-application.yaml

@@ -0,0 +1,22 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ljwx-health-contabo-prod
+  namespace: argocd
+  annotations:
+    # The application is intentionally manual until the migration runbook's
+    # storage, Secret, restore, canary and failure-domain gates all pass.
+    ops.ljwx.io/cutover-gate: manual-preflight
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/lingjingwanxiang/ljwx-deploy.git
+    targetRevision: main
+    path: apps/ljwx-health/overlays/contabo-prod-staging
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: ljwx-health
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+      - PruneLast=true

docs/operations/ljwx-health-contabo-migration.md

@@ -0,0 +1,217 @@
+# LJWX Health Contabo staged migration and failure-domain acceptance
+
+Issues: `lingjingwanxiang/ljwx-health#390`, `#395`
+
+This runbook moves LJWX Health from the current single-node production cluster
+to the existing three-node Contabo k3s cluster without changing the release or
+schema sources of truth. The destination Argo CD Application is deliberately
+manual and initially renders the `contabo-prod-staging` overlay.
+
+## Non-negotiable safety boundaries
+
+- Do not delete or rewrite unrelated Contabo workloads, Docker volumes, Ollama
+  models, database files, or object-store data to recover disk space. Expand
+  the affected disk, or obtain explicit asset-owner approval for a separately
+  backed-up archive operation.
+- Do not sync the destination Application until every preflight gate below is
+  recorded. Creating the Application alone does not authorize a sync.
+- Do not run two production cloudflared connector sets for the same tunnel
+  during canary. The staging overlay removes all cloudflared resources.
+- Do not create credentials in Git or place decoded Secret data in evidence.
+  Provision destination Secrets through the protected operator path and record
+  only names and resource versions.
+- Do not treat hostnames, public IPs, or node count as proof of independent
+  physical failure domains. Infrastructure must record the provider placement
+  evidence used to assign each `topology.kubernetes.io/zone` label.
+- Do not promote if a backup, restore, schema fingerprint, immutable image,
+  canary, node-loss, or rollback check fails.
+
+## Source-of-truth contract
+
+One protected `ljwx-health` commit produces the signed Release Manifest,
+digest-pinned images, signed schema bundle and expected production schema
+fingerprint. One protected `ljwx-deploy` commit selects those immutable
+artifacts. Migration changes only the observed destination cluster.
+
+For each attempt, record all of the following in one evidence directory:
+
+- Health source SHA and Release Manifest digest;
+- Deploy source SHA and Argo CD observed revision;
+- expected and observed image digests for Boot, Admin, Bigscreen,
+  schema-runner, writer and recon;
+- schema source SHA, schema manifest digest, expected production fingerprint
+  and the restored database's observed fingerprint;
+- source backup label, start/stop LSN, stop time, manifest checksum and object
+  repository identity;
+- destination node names, declared zones, Ready/schedulable state and provider
+  placement evidence;
+- canary/final HTTP checks, workload placement, drill timeline, RTO/RPO and
+  rollback outcome.
+
+Any mismatch is a stop condition. A matching branch name, image tag or table
+count is not equality evidence.
+
+## Stage 0: freeze identities and prepare evidence
+
+Use explicit contexts and verify them before every write:
+
+```bash
+export LJWX_ACTIVE_CONTEXT='<current-production-context>'
+export LJWX_CONTABO_CONTEXT='<contabo-context>'
+export LJWX_EVIDENCE_DIR="test-results/contabo-migration/$(date -u +%Y%m%dT%H%M%SZ)"
+mkdir -p "${LJWX_EVIDENCE_DIR}"
+
+kubectl --context "${LJWX_ACTIVE_CONTEXT}" cluster-info
+kubectl --context "${LJWX_CONTABO_CONTEXT}" cluster-info
+git rev-parse HEAD | tee "${LJWX_EVIDENCE_DIR}/deploy-source-sha.txt"
+kubectl kustomize apps/ljwx-health/overlays/contabo-prod-staging \
+  >"${LJWX_EVIDENCE_DIR}/staging-render.yaml"
+```
+
+Before a source backup, stop write-producing jobs and establish the declared
+write-freeze window. Record the authenticated API smoke immediately before the
+freeze. Take a pgBackRest full or differential backup, require a successful
+`check`, and copy the immutable backup set to a destination that does not share
+the source database node or storage failure domain.
+
+## Stage 1: repair and qualify the destination storage plane
+
+The 2026-08-02 preflight found `vmi2822146` at 100% root usage with
+`DiskPressure`; the dominant consumer was an unrelated Ollama container's
+writable model layer. MinIO, PostgreSQL and MySQL local-path volumes were pinned
+to that node, leaving replacement pods Pending. Repair must preserve those
+assets.
+
+Require all of these before migration:
+
+```bash
+kubectl --context "${LJWX_CONTABO_CONTEXT}" get nodes
+kubectl --context "${LJWX_CONTABO_CONTEXT}" describe nodes \
+  | grep -E '^(Name:|  DiskPressure|Taints:)'
+kubectl --context "${LJWX_CONTABO_CONTEXT}" get pods -A \
+  --field-selector=status.phase!=Running,status.phase!=Succeeded
+kubectl --context "${LJWX_CONTABO_CONTEXT}" get pv,pvc -A -o wide
+```
+
+Acceptance:
+
+- every node has at least 20% free root/image filesystem capacity;
+- no node has `DiskPressure` or an unplanned storage taint;
+- each moved volume has a checksum-verified copy and a documented rollback
+  location before the original is changed;
+- PostgreSQL, MinIO and any required shared dependency have Running endpoints
+  and pass application-level read/write checks;
+- no data-bearing volume is counted as an independent backup of itself.
+
+## Stage 2: declare verified failure domains and provision prerequisites
+
+After Infrastructure records provider placement evidence, apply stable zone
+labels. The example values are placeholders, not proof:
+
+```bash
+kubectl --context "${LJWX_CONTABO_CONTEXT}" label node <node-a> \
+  topology.kubernetes.io/zone=<verified-zone-a> --overwrite
+kubectl --context "${LJWX_CONTABO_CONTEXT}" label node <node-b> \
+  topology.kubernetes.io/zone=<verified-zone-b> --overwrite
+kubectl --context "${LJWX_CONTABO_CONTEXT}" label node <node-c> \
+  topology.kubernetes.io/zone=<verified-zone-c> --overwrite
+```
+
+Provision the destination registry pull Secret, runtime Secret, schema
+migrator Secret and cloudflared credential Secret through the protected secret
+path. Validate Traefik, cert-manager, the `dnspod-letsencrypt` ClusterIssuer,
+Prometheus Operator CRDs and required dependency Services before syncing.
+Staging intentionally disables Kafka/ClickHouse shadow writers; they must be
+restored or explicitly accepted before final production cutover.
+
+## Stage 3: restore and prove schema identity
+
+Restore the frozen backup into PostgreSQL 18 on the destination. Run the signed
+schema bundle verify path against the restored database. If upgrade is needed,
+run only the digest-pinned bundle selected by the same Release Manifest, then
+verify again.
+
+Acceptance requires exact equality between the Release Manifest's expected
+production fingerprint and the observed normalized fingerprint. Also compare
+database system identifier policy, extensions, public table/column/sequence
+counts, RLS policies, role/privilege boundary and representative tenant data.
+Preserve the restore logs and machine-readable schema evidence.
+
+## Stage 4: create and manually sync the canary
+
+After this GitOps change is merged to protected `main`, create only the manual
+Application and inspect the diff:
+
+```bash
+kubectl --context "${LJWX_CONTABO_CONTEXT}" apply \
+  -f cluster-prod/ljwx-health-prod-application.yaml
+argocd app diff ljwx-health-contabo-prod
+argocd app sync ljwx-health-contabo-prod --prune
+argocd app wait ljwx-health-contabo-prod --sync --health --operation
+```
+
+The only public hosts allowed during this stage are:
+
+- `health-admin-contabo-canary.lingjingwanxiang.cn`
+- `health-dashboard-contabo-canary.lingjingwanxiang.cn`
+
+Require authenticated API/UI smoke, zero unexpected restarts, both Boot and
+Admin replicas Ready, and actual placement across at least two verified zones.
+Run the Health repository's exact-main `scripts/ops_readiness.py --live` against
+the destination and preserve the result.
+
+## Stage 5: node-loss, entrypoint and restore drills
+
+Drain one workload node at a time. Never drain the next node until the previous
+node is uncordoned, Ready and all critical replicas have reconverged:
+
+```bash
+kubectl --context "${LJWX_CONTABO_CONTEXT}" drain <node> \
+  --ignore-daemonsets --delete-emptydir-data --timeout=10m
+# run authenticated canary/API, alert delivery and placement checks
+kubectl --context "${LJWX_CONTABO_CONTEXT}" uncordon <node>
+```
+
+Repeat with an actual node stop/restart if Infrastructure can safely control
+the VM. During each loss, require the canary UI and core API to remain within
+the declared RTO, record all failed requests, and verify connector/alert paths.
+
+Independently restore the destination backup copy into an isolated volume that
+does not mount production data. Require pgBackRest checksums, the same schema
+fingerprint, tenant-data checks and application smoke. Record RPO and RTO. The
+restore target and evidence must remain usable when the database node is
+unavailable.
+
+## Stage 6: final cutover
+
+Final cutover is a separate reviewed GitOps change that switches the
+Application source path from `contabo-prod-staging` to `contabo-prod`. Before
+merge, restore the Kafka/ClickHouse shadow path or record the approved
+production-equivalent design; the staging disablement is not an implicit
+waiver.
+
+At the write-freeze boundary, replay/restore any final WAL, rerun schema and
+data equality checks, stop the old production connector set, then allow the
+destination cloudflared connector. Verify both production domains, authenticated
+API/UI smoke, alerts, metrics, traces, image digests and workload placement.
+Keep the old production cluster read-only and rollback-capable until the agreed
+soak period completes.
+
+## Rollback
+
+- Before final WAL replay: stop/delete only the destination Application and
+  retain destination data for diagnosis. The current cluster remains primary.
+- After final replay but before traffic cutover: keep writes frozen, compare
+  source/destination LSNs and return to the source only if no destination write
+  was accepted.
+- After destination writes begin: do not perform an unplanned reverse switch.
+  Freeze writes, reconcile data, make an incident decision and record the data
+  authority before changing traffic.
+- For a manifest regression, revert the final GitOps path change to staging or
+  the recorded N-1 deploy SHA. Do not reverse a successful schema migration.
+
+Close `#390` only after the machine-readable readiness result has no
+failure-domain FAIL, both node-loss drills and the isolated restore pass, and
+all evidence identifies immutable sources. Close `#395` only after `#392` also
+has distinct real-human primary/backup acknowledgement and escalation drill
+evidence.

scripts/tests/test_ljwx_health_contabo_migration.py

@@ -0,0 +1,157 @@
+import subprocess
+import unittest
+from pathlib import Path
+
+import yaml
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+class LjwxHealthContaboMigrationTest(unittest.TestCase):
+    def load_yaml(self, relative_path: str) -> dict:
+        return yaml.safe_load(
+            (REPO_ROOT / relative_path).read_text(encoding="utf-8")
+        )
+
+    def render(self, overlay: str) -> list[dict]:
+        result = subprocess.run(
+            ["kubectl", "kustomize", str(REPO_ROOT / overlay)],
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(
+            0,
+            result.returncode,
+            f"failed to render {overlay}:\n{result.stderr}",
+        )
+        return [
+            document
+            for document in yaml.safe_load_all(result.stdout)
+            if isinstance(document, dict)
+        ]
+
+    def test_prod_bootstrap_registers_a_manual_preflight_application(self):
+        bootstrap = self.load_yaml("cluster-prod/kustomization.yaml")
+        self.assertIn(
+            "ljwx-health-prod-application.yaml", bootstrap["resources"]
+        )
+
+        application = self.load_yaml(
+            "cluster-prod/ljwx-health-prod-application.yaml"
+        )
+        self.assertEqual("Application", application["kind"])
+        self.assertEqual(
+            "ljwx-health-contabo-prod", application["metadata"]["name"]
+        )
+        self.assertEqual(
+            "manual-preflight",
+            application["metadata"]["annotations"]
+            ["ops.ljwx.io/cutover-gate"],
+        )
+        self.assertEqual(
+            "apps/ljwx-health/overlays/contabo-prod-staging",
+            application["spec"]["source"]["path"],
+        )
+        self.assertNotIn(
+            "automated", application["spec"].get("syncPolicy", {})
+        )
+
+    def test_staging_overlay_is_canary_only_and_has_no_tunnel_connector(self):
+        documents = self.render(
+            "apps/ljwx-health/overlays/contabo-prod-staging"
+        )
+        by_key = {
+            (document.get("kind"), document.get("metadata", {}).get("name")):
+            document
+            for document in documents
+        }
+
+        cloudflared_keys = {
+            key
+            for key in by_key
+            if key[1] and "cloudflared" in key[1]
+        }
+        self.assertEqual(set(), cloudflared_keys)
+
+        ingress_documents = [
+            document for document in documents if document.get("kind") == "Ingress"
+        ]
+        self.assertEqual(2, len(ingress_documents))
+        expected_hosts = {
+            "health-admin-contabo-canary.lingjingwanxiang.cn",
+            "health-dashboard-contabo-canary.lingjingwanxiang.cn",
+        }
+        for ingress in ingress_documents:
+            self.assertEqual("traefik", ingress["spec"]["ingressClassName"])
+            self.assertEqual(
+                expected_hosts,
+                {rule["host"] for rule in ingress["spec"]["rules"]},
+            )
+            self.assertFalse(
+                any(
+                    key.startswith("nginx.ingress.kubernetes.io/")
+                    for key in ingress.get("metadata", {})
+                    .get("annotations", {})
+                )
+            )
+
+        admin = by_key[("Deployment", "ljwx-admin")]
+        admin_env = {
+            item["name"]: item.get("value")
+            for item in admin["spec"]["template"]["spec"]["containers"][0]
+            .get("env", [])
+        }
+        self.assertEqual(
+            "https://health-admin-contabo-canary.lingjingwanxiang.cn/api",
+            admin_env["VITE_API_URL"],
+        )
+
+        boot = by_key[("Deployment", "ljwx-boot")]
+        boot_env = {
+            item["name"]: item.get("value")
+            for item in boot["spec"]["template"]["spec"]["containers"][0]
+            .get("env", [])
+        }
+        self.assertEqual("false", boot_env["LJWX_HEALTH_OUTBOX_KAFKA_ENABLED"])
+        self.assertEqual("false", boot_env["LJWX_HEALTH_REALTIME_SCORE_ENABLED"])
+
+    def test_final_overlay_retains_production_tunnel_with_traefik_origin(self):
+        documents = self.render("apps/ljwx-health/overlays/contabo-prod")
+        by_key = {
+            (document.get("kind"), document.get("metadata", {}).get("name")):
+            document
+            for document in documents
+        }
+        connector = by_key[("ConfigMap", "ljwx-health-cloudflared-config")]
+        self.assertIn(
+            "http://traefik.kube-system.svc.cluster.local:80",
+            connector["data"]["config.yaml"],
+        )
+        self.assertIn(
+            ("Deployment", "ljwx-health-cloudflared"), by_key
+        )
+
+        ingress_policy = by_key[("NetworkPolicy", "ljwx-health-ingress")]
+        ingress_sources = [
+            source
+            for rule in ingress_policy["spec"]["ingress"]
+            for source in rule.get("from", [])
+        ]
+        self.assertTrue(
+            any(
+                source.get("namespaceSelector", {})
+                .get("matchLabels", {})
+                .get("kubernetes.io/metadata.name")
+                == "kube-system"
+                and source.get("podSelector", {})
+                .get("matchLabels", {})
+                .get("app.kubernetes.io/name")
+                == "traefik"
+                for source in ingress_sources
+            )
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()