feat(observability): implement enterprise-grade k3s monitoring guardrails

.github/workflows/observability-guardrails.yml

@@ -0,0 +1,72 @@
+name: Observability Guardrails
+
+on:
+  pull_request:
+    paths:
+      - "deploy/monitoring/**"
+      - "scripts/check-observability-k3s.sh"
+      - "scripts/deploy-observability.sh"
+      - "docs/04-operations/monitoring/**"
+  push:
+    branches: [main]
+    paths:
+      - "deploy/monitoring/**"
+      - "scripts/check-observability-k3s.sh"
+      - "scripts/deploy-observability.sh"
+      - "docs/04-operations/monitoring/**"
+
+jobs:
+  validate-observability:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Validate shell scripts syntax
+        run: |
+          bash -n scripts/check-observability-k3s.sh
+          bash -n scripts/deploy-observability.sh
+
+      - name: Validate dashboard JSON
+        run: |
+          jq -e . deploy/monitoring/grafana/provisioning/dashboards/ljwx-boot-dashboard.json >/dev/null
+          jq -e . deploy/monitoring/grafana/provisioning/dashboards/ljwx-bigscreen-dashboard.json >/dev/null
+          jq -e . deploy/monitoring/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json >/dev/null
+
+      - name: Validate YAML structure and required resources
+        run: |
+          python - <<'PY'
+          import sys, yaml, pathlib
+
+          files = [
+              "deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml",
+              "deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml",
+              "deploy/monitoring/grafana/grafana.yaml",
+          ]
+          for f in files:
+              p = pathlib.Path(f)
+              if not p.exists():
+                  raise SystemExit(f"missing required file: {f}")
+              with p.open("r", encoding="utf-8") as fp:
+                  docs = [d for d in yaml.safe_load_all(fp) if d]
+              if not docs:
+                  raise SystemExit(f"empty yaml: {f}")
+
+          with open("deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml", "r", encoding="utf-8") as fp:
+              rule = yaml.safe_load(fp)
+          if rule.get("kind") != "PrometheusRule":
+              raise SystemExit("ljwx-health-k3s-prometheusrule.yaml kind must be PrometheusRule")
+          groups = rule.get("spec", {}).get("groups", [])
+          names = {g.get("name") for g in groups}
+          required = {
+              "ljwx_health_k3s_availability",
+              "ljwx_health_k3s_latency_and_errors",
+              "ljwx_health_k3s_dependencies",
+              "ljwx_health_k3s_recording_rules",
+              "ljwx_health_k3s_slo_alerts",
+          }
+          missing = required - names
+          if missing:
+              raise SystemExit(f"PrometheusRule missing groups: {sorted(missing)}")
+          print("observability yaml validation passed")
+          PY

deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml

@@ -0,0 +1,54 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: ljwx-health-routing
+  namespace: monitoring
+  labels:
+    release: monitoring
+    app.kubernetes.io/name: ljwx-health
+spec:
+  route:
+    groupBy:
+      - alertname
+      - service
+      - category
+      - severity
+    groupWait: 30s
+    groupInterval: 5m
+    repeatInterval: 2h
+    receiver: ljwx-health-warning
+    routes:
+      - matchers:
+          - name: severity
+            value: critical
+            matchType: "="
+        receiver: ljwx-health-critical
+        continue: false
+      - matchers:
+          - name: severity
+            value: warning
+            matchType: "="
+        receiver: ljwx-health-warning
+        continue: false
+      - matchers:
+          - name: severity
+            value: info
+            matchType: "="
+        receiver: ljwx-health-info
+        continue: false
+  receivers:
+    - name: ljwx-health-critical
+      webhookConfigs:
+        - url: http://alert-webhook.monitoring.svc.cluster.local:8080/critical
+          sendResolved: true
+          maxAlerts: 0
+    - name: ljwx-health-warning
+      webhookConfigs:
+        - url: http://alert-webhook.monitoring.svc.cluster.local:8080/warning
+          sendResolved: true
+          maxAlerts: 0
+    - name: ljwx-health-info
+      webhookConfigs:
+        - url: http://alert-webhook.monitoring.svc.cluster.local:8080/info
+          sendResolved: true
+          maxAlerts: 0

deploy/monitoring/grafana/README.md

@@ -83,6 +83,11 @@ kubectl create configmap grafana-dashboard-ljwx-bigscreen \
   --from-file=ljwx-bigscreen-dashboard.json=deploy/monitoring/grafana/provisioning/dashboards/ljwx-bigscreen-dashboard.json \
   -n infra \
   --dry-run=client -o yaml | kubectl apply -f -
+
+kubectl create configmap grafana-dashboard-ljwx-health-enterprise-slo \
+  --from-file=ljwx-health-enterprise-slo-dashboard.json=deploy/monitoring/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json \
+  -n infra \
+  --dry-run=client -o yaml | kubectl apply -f -
 ```
 
 ### 方法 2: 更新 Grafana Deployment

deploy/monitoring/grafana/grafana.yaml

@@ -217,6 +217,12 @@ spec:
               subPath: ljwx-bigscreen-dashboard.json
               readOnly: true
 
+            # 挂载 Enterprise SLO Dashboard JSON
+            - name: dashboard-ljwx-health-enterprise-slo
+              mountPath: /etc/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json
+              subPath: ljwx-health-enterprise-slo-dashboard.json
+              readOnly: true
+
             # 挂载数据持久化目录（可选）
             - name: grafana-storage
               mountPath: /var/lib/grafana
@@ -272,6 +278,12 @@ spec:
             name: grafana-dashboard-ljwx-bigscreen
             defaultMode: 0644
 
+        # Enterprise SLO Dashboard ConfigMap
+        - name: dashboard-ljwx-health-enterprise-slo
+          configMap:
+            name: grafana-dashboard-ljwx-health-enterprise-slo
+            defaultMode: 0644
+
         # 数据持久化（可选，使用 emptyDir 或 PVC）
         - name: grafana-storage
           emptyDir: {}

deploy/monitoring/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json

@@ -0,0 +1,157 @@
+{
+  "id": null,
+  "uid": "ljwx-health-enterprise-slo",
+  "title": "LJWX Health - Enterprise SLO",
+  "tags": ["ljwx", "slo", "enterprise"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Boot Error Ratio (5m)",
+      "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "ljwx_health:boot:http_error_ratio:rate5m",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 0.001},
+              {"color": "red", "value": 0.005}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {"reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""}}
+    },
+    {
+      "type": "stat",
+      "title": "Boot Request Rate (5m)",
+      "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "ljwx_health:boot:http_requests:rate5m",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
+      "options": {"reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""}}
+    },
+    {
+      "type": "stat",
+      "title": "Boot P95 Latency",
+      "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "ljwx_health:boot:http_latency_p95:5m",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 0.5},
+              {"color": "red", "value": 1.0}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {"reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""}}
+    },
+    {
+      "type": "stat",
+      "title": "Critical Alerts Firing",
+      "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"critical\",namespace=\"monitoring\",alertname=~\"LjwxHealth.*\"})",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "red", "value": 1}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {"reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""}}
+    },
+    {
+      "type": "timeseries",
+      "title": "SLO Error Ratio Trends",
+      "gridPos": {"h": 9, "w": 12, "x": 0, "y": 6},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "ljwx_health:boot:http_error_ratio:rate5m",
+          "legendFormat": "5m",
+          "refId": "A"
+        },
+        {
+          "expr": "ljwx_health:boot:http_error_ratio:rate1h",
+          "legendFormat": "1h",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {"displayMode": "table", "placement": "bottom"},
+        "tooltip": {"mode": "multi"}
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Availability/Saturation Signals",
+      "gridPos": {"h": 9, "w": 12, "x": 12, "y": 6},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "kube_deployment_status_replicas_available{namespace=\"ljwx-health\",deployment=~\"ljwx-health-(boot|admin|bigscreen)\"}",
+          "legendFormat": "{{deployment}} available",
+          "refId": "A"
+        },
+        {
+          "expr": "kube_deployment_spec_replicas{namespace=\"ljwx-health\",deployment=~\"ljwx-health-(boot|admin|bigscreen)\"}",
+          "legendFormat": "{{deployment}} desired",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
+      "options": {
+        "legend": {"displayMode": "table", "placement": "bottom"},
+        "tooltip": {"mode": "multi"}
+      }
+    }
+  ]
+}

deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml

@@ -154,3 +154,82 @@ spec:
             summary: "ljwx-health 副本未达到期望值"
             description: "Deployment {{ $labels.deployment }} 可用副本持续低于期望值。"
             runbook: "docs/04-operations/monitoring/observability-review-checklist-k3s.md"
+
+    - name: ljwx_health_k3s_recording_rules
+      interval: 30s
+      rules:
+        - record: ljwx_health:boot:http_requests:rate5m
+          expr: sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[5m]))
+
+        - record: ljwx_health:boot:http_errors_5xx:rate5m
+          expr: sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[5m]))
+
+        - record: ljwx_health:boot:http_error_ratio:rate5m
+          expr: |
+            ljwx_health:boot:http_errors_5xx:rate5m
+            /
+            clamp_min(ljwx_health:boot:http_requests:rate5m, 0.001)
+
+        - record: ljwx_health:boot:http_error_ratio:rate1h
+          expr: |
+            sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[1h]))
+            /
+            clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[1h])), 0.001)
+
+        - record: ljwx_health:boot:http_latency_p95:5m
+          expr: |
+            histogram_quantile(
+              0.95,
+              sum(rate(http_server_requests_seconds_bucket{namespace="ljwx-health",service="ljwx-health-boot"}[5m])) by (le)
+            )
+
+    - name: ljwx_health_k3s_slo_alerts
+      interval: 30s
+      rules:
+        - alert: LjwxHealthBootSLOErrorBudgetBurnFast
+          expr: |
+            (
+              sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[5m]))
+              /
+              clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[5m])), 0.001)
+            ) > (14.4 * (1 - 0.999))
+            and
+            (
+              sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[1h]))
+              /
+              clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[1h])), 0.001)
+            ) > (14.4 * (1 - 0.999))
+          for: 2m
+          labels:
+            severity: critical
+            category: slo
+            service: ljwx-health-boot
+            slo_target: "99.9"
+          annotations:
+            summary: "ljwx-health-boot 错误预算快速燃烧"
+            description: "5m/1h 多窗口 burn-rate 超标，SLO(99.9%) 可能快速失守。"
+            runbook: "docs/04-operations/monitoring/observability-review-checklist-k3s.md"
+
+        - alert: LjwxHealthBootSLOErrorBudgetBurnSlow
+          expr: |
+            (
+              sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[30m]))
+              /
+              clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[30m])), 0.001)
+            ) > (6 * (1 - 0.999))
+            and
+            (
+              sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[6h]))
+              /
+              clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[6h])), 0.001)
+            ) > (6 * (1 - 0.999))
+          for: 15m
+          labels:
+            severity: warning
+            category: slo
+            service: ljwx-health-boot
+            slo_target: "99.9"
+          annotations:
+            summary: "ljwx-health-boot 错误预算持续燃烧"
+            description: "30m/6h 多窗口 burn-rate 超标，需在值班窗口内处理。"
+            runbook: "docs/04-operations/monitoring/observability-review-checklist-k3s.md"

docs/04-operations/monitoring/observability-enterprise-implementation.md

@@ -0,0 +1,88 @@
+# 企业级可观测性实施方案（LJWX Health）
+
+## 目标
+
+- 建立企业级三支柱能力：Metrics、Logs、Traces
+- 建立 SLO 与错误预算治理机制（99.9% 可用性目标）
+- 建立值班可执行机制：分级告警、路由、Runbook、门禁脚本
+
+## 已落地资产
+
+- Prometheus 告警与记录规则：
+  - `deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml`
+- Alertmanager 路由配置（CRD）：
+  - `deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml`
+- SLO 总览仪表盘：
+  - `deploy/monitoring/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json`
+- 发布门禁脚本：
+  - `scripts/check-observability-k3s.sh`
+- 监控变更 CI 守护：
+  - `.github/workflows/observability-guardrails.yml`
+
+## 企业级能力映射
+
+### 1) 可用性治理
+
+- Deployment 不可用告警（boot/admin/bigscreen）
+- CrashLoopBackOff 告警
+- 副本覆盖率不足告警
+
+### 2) 质量治理（错误率与延迟）
+
+- 5xx 错误率告警（warning/critical）
+- P95 延迟告警
+- SLI recording rules：
+  - 请求率
+  - 错误率（5m/1h）
+  - P95 延迟
+
+### 3) SLO 与错误预算
+
+- SLO 目标：可用性 99.9%
+- 多窗口 burn-rate 告警：
+  - Fast burn：5m/1h（critical）
+  - Slow burn：30m/6h（warning）
+
+### 4) 值班分级路由
+
+- severity = critical/warning/info 分流
+- 支持 resolved 回调，减少人工噪音
+
+## 部署步骤（生产/准生产）
+
+```bash
+# 1. 应用告警和记录规则
+kubectl apply -f deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml
+
+# 2. 应用 Alertmanager 路由
+kubectl apply -f deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml
+
+# 3. 部署/刷新 Grafana（含 enterprise SLO dashboard）
+bash scripts/deploy-observability.sh
+
+# 4. 运行门禁验收
+./scripts/check-observability-k3s.sh
+```
+
+## 发布门禁（必须全部通过）
+
+- ArgoCD：`Synced + Healthy + selfHeal=true`
+- 业务核心服务：`boot/admin/bigscreen` 均为 `Ready`
+- PrometheusRule 存在并可被 Prometheus Operator 校验
+- 对外健康探针返回 `HTTP 200`
+
+## 运行期管理建议
+
+- 值班流程：
+  - Critical：5 分钟内响应，必须回填事件编号
+  - Warning：30 分钟内确认影响与处置
+  - Info：日内归档处理
+- 周期审计：
+  - 每周审计 Top 10 告警噪音来源
+  - 每周审计 SLO burn 事件与根因分类
+  - 每月审计阈值是否与业务峰值匹配
+
+## 已知边界
+
+- AlertmanagerConfig 中 webhook 地址为占位服务，需要替换成企业内部通知网关。
+- 若集群未安装 `AlertmanagerConfig` CRD，路由对象不会生效，但不会影响 PrometheusRule。

docs/04-operations/monitoring/observability-guide.md

@@ -7,6 +7,7 @@
 - 评审清单: `docs/04-operations/monitoring/observability-review-checklist-k3s.md`
 - 门禁脚本: `scripts/check-observability-k3s.sh`
 - K3s 告警规则: `deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml`
+- 企业级实施方案: `docs/04-operations/monitoring/observability-enterprise-implementation.md`
 
 ## 概述
 

scripts/apply-observability-enterprise.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "[1/4] Apply PrometheusRule"
+kubectl apply -f deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml
+
+echo "[2/4] Apply AlertmanagerConfig"
+kubectl apply -f deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml
+
+echo "[3/4] Deploy/refresh Grafana observability assets"
+bash scripts/deploy-observability.sh
+
+echo "[4/4] Run observability gate"
+./scripts/check-observability-k3s.sh
+
+echo "Observability enterprise rollout completed."

scripts/check-observability-k3s.sh

@@ -4,6 +4,7 @@ set -u
 
 NS_APP="${NS_APP:-ljwx-health}"
 NS_ARGO="${NS_ARGO:-argocd}"
+NS_GRAFANA="${NS_GRAFANA:-monitoring}"
 APP_NAME="${APP_NAME:-ljwx-health-k3s-eu}"
 API_HEALTH_URL="${API_HEALTH_URL:-https://health-admin.lingjingwanxiang.cn/api/actuator/health}"
 
@@ -74,7 +75,26 @@ else
   bad "ServiceMonitor 数量不足：$sm_count"
 fi
 
-info "4) 对外健康检查"
+info "4) 规则与路由资源"
+if kubectl -n monitoring get prometheusrule ljwx-health-k3s-alerts >/dev/null 2>&1; then
+  ok "PrometheusRule ljwx-health-k3s-alerts 存在"
+else
+  bad "PrometheusRule ljwx-health-k3s-alerts 缺失"
+fi
+
+if kubectl -n monitoring get alertmanagerconfig ljwx-health-routing >/dev/null 2>&1; then
+  ok "AlertmanagerConfig ljwx-health-routing 存在"
+else
+  warn "AlertmanagerConfig ljwx-health-routing 缺失（若未启用该 CRD 可忽略）"
+fi
+
+if kubectl -n "$NS_GRAFANA" get configmap grafana-dashboard-ljwx-health-enterprise-slo >/dev/null 2>&1; then
+  ok "Grafana enterprise SLO dashboard ConfigMap 存在"
+else
+  warn "Grafana enterprise SLO dashboard ConfigMap 缺失（请运行 scripts/deploy-observability.sh）"
+fi
+
+info "5) 对外健康检查"
 http_code="$(curl -skI --max-time 8 "$API_HEALTH_URL" | awk '/^HTTP/{print $2}' | tail -n 1)"
 if [[ "$http_code" == "200" ]]; then
   ok "API health 返回 HTTP 200"
@@ -82,7 +102,7 @@ else
   bad "API health 返回 HTTP ${http_code:-N/A}"
 fi
 
-info "5) 近 10 分钟 boot 错误日志扫描"
+info "6) 近 10 分钟 boot 错误日志扫描"
 err_lines="$(kubectl -n "$NS_APP" logs deploy/ljwx-health-boot --since=10m 2>/dev/null | grep -E 'ERROR|Exception' | wc -l | tr -d ' ')"
 if [[ "$err_lines" -eq 0 ]]; then
   ok "boot 最近 10 分钟无 ERROR/Exception"

scripts/deploy-observability.sh

@@ -165,6 +165,16 @@ deploy_grafana() {
             log_warning "  ✗ ljwx-bigscreen-dashboard.json 未找到"
         fi
 
+        if [ -f "$DASHBOARD_DIR/ljwx-health-enterprise-slo-dashboard.json" ]; then
+            kubectl create configmap grafana-dashboard-ljwx-health-enterprise-slo \
+                --from-file=ljwx-health-enterprise-slo-dashboard.json="$DASHBOARD_DIR/ljwx-health-enterprise-slo-dashboard.json" \
+                -n $NAMESPACE_INFRA \
+                --dry-run=client -o yaml | kubectl apply -f -
+            log_success "  ✓ ljwx-health Enterprise SLO Dashboard ConfigMap 已创建"
+        else
+            log_warning "  ✗ ljwx-health-enterprise-slo-dashboard.json 未找到"
+        fi
+
         # 2. 部署 Grafana Deployment 和 Service
         log_info "部署 Grafana Deployment 和 Service..."
         kubectl apply -f "$MONITORING_DIR/grafana/grafana.yaml" -n $NAMESPACE_INFRA