feat(health): add bigscreen trial observability

apps/ljwx-health/base/grafana/ljwx-health-slo-dashboard.json

@@ -62,15 +62,132 @@
       "targets": [{"expr": "sum(hikaricp_connections_active{namespace=\"ljwx-health\",service=\"ljwx-boot\"}) / clamp_min(sum(hikaricp_connections_max{namespace=\"ljwx-health\",service=\"ljwx-boot\"}), 1)", "legendFormat": "pool", "refId": "A"}],
       "title": "Database pool saturation",
       "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "reqps", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}]}}, "overrides": []},
+      "gridPos": {"h": 5, "w": 6, "x": 0, "y": 13},
+      "id": 7,
+      "targets": [{"expr": "ljwx_health:bigscreen_bff_qps:5m{tenant=~\"$tenant\"}", "legendFormat": "tenant {{tenant}}", "refId": "A"}],
+      "title": "Bigscreen BFF QPS",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1, "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 0.995}]}}, "overrides": []},
+      "gridPos": {"h": 5, "w": 6, "x": 6, "y": 13},
+      "id": 8,
+      "targets": [{"expr": "ljwx_health:bigscreen_bff_success:ratio5m{tenant=~\"$tenant\"}", "legendFormat": "tenant {{tenant}}", "refId": "A"}],
+      "title": "BFF success ratio (target 99.5%)",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "red", "value": 0.5}]}}, "overrides": []},
+      "gridPos": {"h": 5, "w": 6, "x": 12, "y": 13},
+      "id": 9,
+      "targets": [
+        {"expr": "ljwx_health:bigscreen_bff_latency_p95:5m{tenant=~\"$tenant\"}", "legendFormat": "P95 {{tenant}}", "refId": "A"},
+        {"expr": "ljwx_health:bigscreen_bff_latency_p99:5m{tenant=~\"$tenant\"}", "legendFormat": "P99 {{tenant}}", "refId": "B"}
+      ],
+      "title": "BFF P95 / P99 latency",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1, "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 0.7}]}}, "overrides": []},
+      "gridPos": {"h": 5, "w": 6, "x": 18, "y": 13},
+      "id": 10,
+      "targets": [{"expr": "ljwx_health:bigscreen_cache_hit:ratio5m{tenant=~\"$tenant\"}", "legendFormat": "tenant {{tenant}}", "refId": "A"}],
+      "title": "Redis cache hit ratio (target 70%)",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1, "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 0.3}]}}, "overrides": []},
+      "gridPos": {"h": 5, "w": 6, "x": 0, "y": 18},
+      "id": 11,
+      "targets": [{"expr": "ljwx_health:bigscreen_snapshot_304:ratio5m{tenant=~\"$tenant\"}", "legendFormat": "tenant {{tenant}}", "refId": "A"}],
+      "title": "Snapshot 304 ratio (target 30%)",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "red", "value": 0.5}]}}, "overrides": []},
+      "gridPos": {"h": 5, "w": 6, "x": 6, "y": 18},
+      "id": 12,
+      "targets": [{"expr": "ljwx_health:bigscreen_snapshot_build_p95:5m{tenant=~\"$tenant\"}", "legendFormat": "tenant {{tenant}}", "refId": "A"}],
+      "title": "Snapshot DB aggregation P95",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1, "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 0.995}]}}, "overrides": []},
+      "gridPos": {"h": 5, "w": 6, "x": 12, "y": 18},
+      "id": 13,
+      "targets": [{"expr": "ljwx_health:bigscreen_kiosk_refresh_success:ratio5m{tenant=~\"$tenant\"}", "legendFormat": "tenant {{tenant}}", "refId": "A"}],
+      "title": "Kiosk refresh success ratio",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "red", "value": 2}]}}, "overrides": []},
+      "gridPos": {"h": 5, "w": 6, "x": 18, "y": 18},
+      "id": 14,
+      "targets": [{"expr": "ljwx_health:bigscreen_kiosk_update_delay_p95:5m{tenant=~\"$tenant\"}", "legendFormat": "tenant {{tenant}}", "refId": "A"}],
+      "title": "generatedAt to kiosk update P95",
+      "type": "stat"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "red", "value": 300}]}}, "overrides": []},
+      "gridPos": {"h": 7, "w": 8, "x": 0, "y": 23},
+      "id": 15,
+      "targets": [
+        {"expr": "ljwx_health:bigscreen_kiosk_heartbeat_age:seconds{tenant=~\"$tenant\"}", "legendFormat": "heartbeat age {{tenant}}", "refId": "A"},
+        {"expr": "ljwx_health:bigscreen_kiosk_recovery_p95:1h{tenant=~\"$tenant\"}", "legendFormat": "recovery P95 {{tenant}}", "refId": "B"}
+      ],
+      "title": "Kiosk heartbeat and recovery",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "short", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "red", "value": 1}]}}, "overrides": []},
+      "gridPos": {"h": 7, "w": 8, "x": 8, "y": 23},
+      "id": 16,
+      "targets": [
+        {"expr": "sum by (tenant, type) (increase(bigscreen_kiosk_js_errors_total{tenant=~\"$tenant\"}[$__range]))", "legendFormat": "{{type}} {{tenant}}", "refId": "A"},
+        {"expr": "sum by (tenant, event) (increase(bigscreen_kiosk_connectivity_total{tenant=~\"$tenant\"}[$__range]))", "legendFormat": "{{event}} {{tenant}}", "refId": "B"}
+      ],
+      "title": "JS errors and offline/recovery events",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "bytes", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "red", "value": 157286400}]}}, "overrides": []},
+      "gridPos": {"h": 7, "w": 8, "x": 16, "y": 23},
+      "id": 17,
+      "targets": [{"expr": "max by (tenant) (max_over_time(bigscreen_kiosk_js_heap_bytes{tenant=~\"$tenant\"}[24h])) - min by (tenant) (min_over_time(bigscreen_kiosk_js_heap_bytes{tenant=~\"$tenant\"}[24h]))", "legendFormat": "24h growth {{tenant}}", "refId": "A"}],
+      "title": "Kiosk 24h JS heap growth",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
+      "gridPos": {"h": 7, "w": 24, "x": 0, "y": 30},
+      "id": 18,
+      "targets": [{"expr": "ALERTS{alertname=~\"LjwxHealthBigscreen.*\",tenant=~\"$tenant\",alertstate=\"firing\"}", "format": "table", "instant": true, "refId": "A"}],
+      "title": "Active bigscreen alerts",
+      "type": "table"
     }
   ],
   "refresh": "30s",
   "schemaVersion": 39,
-  "tags": ["ljwx-health", "slo", "operations"],
-  "templating": {"list": []},
-  "time": {"from": "now-6h", "to": "now"},
+  "tags": ["ljwx-health", "slo", "operations", "bigscreen", "trial"],
+  "templating": {"list": [{"name": "tenant", "label": "Tenant", "type": "query", "datasource": {"type": "prometheus", "uid": "prometheus"}, "query": {"query": "label_values(bigscreen_bff_requests_total, tenant)", "refId": "StandardVariableQuery"}, "definition": "label_values(bigscreen_bff_requests_total, tenant)", "includeAll": true, "allValue": ".*", "multi": true, "refresh": 1, "current": {"text": "All", "value": "$__all", "selected": true}}]},
+  "time": {"from": "now-24h", "to": "now"},
   "timezone": "browser",
-  "title": "LJWX Health / SLO and Golden Signals",
+  "title": "LJWX Health / SLO and Bigscreen Trial",
   "uid": "ljwx-health-slo",
-  "version": 1
+  "version": 2
 }

apps/ljwx-health/base/prometheusrule.yaml

@@ -255,3 +255,194 @@ spec:
             summary: "LJWX Health message response P95 is above five minutes"
             description: "The send-to-response P95 has remained above 300 seconds for fifteen minutes."
             runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
+
+    - name: ljwx_health_bigscreen_trial
+      interval: 30s
+      rules:
+        - record: ljwx_health:bigscreen_bff_qps:5m
+          expr: sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime"}[5m]))
+
+        - record: ljwx_health:bigscreen_bff_success:ratio5m
+          expr: |
+            sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime",result=~"success|not_modified"}[5m]))
+            /
+            clamp_min(sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime"}[5m])), 0.001)
+
+        - record: ljwx_health:bigscreen_bff_latency_p95:5m
+          expr: |
+            histogram_quantile(
+              0.95,
+              sum by (tenant, le) (rate(bigscreen_bff_request_latency_seconds_bucket{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime"}[5m]))
+            )
+
+        - record: ljwx_health:bigscreen_bff_latency_p99:5m
+          expr: |
+            histogram_quantile(
+              0.99,
+              sum by (tenant, le) (rate(bigscreen_bff_request_latency_seconds_bucket{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime"}[5m]))
+            )
+
+        - record: ljwx_health:bigscreen_cache_hit:ratio5m
+          expr: |
+            sum by (tenant) (rate(bigscreen_snapshot_cache_access_total{namespace="ljwx-health",service="ljwx-boot",result="hit"}[5m]))
+            /
+            clamp_min(sum by (tenant) (rate(bigscreen_snapshot_cache_access_total{namespace="ljwx-health",service="ljwx-boot",result=~"hit|miss"}[5m])), 0.001)
+
+        - record: ljwx_health:bigscreen_snapshot_304:ratio5m
+          expr: |
+            sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint="snapshot",result="not_modified"}[5m]))
+            /
+            clamp_min(sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint="snapshot"}[5m])), 0.001)
+
+        - record: ljwx_health:bigscreen_snapshot_build_p95:5m
+          expr: |
+            histogram_quantile(
+              0.95,
+              sum by (tenant, le) (rate(bigscreen_snapshot_build_latency_seconds_bucket{namespace="ljwx-health",service="ljwx-boot"}[5m]))
+            )
+
+        - record: ljwx_health:bigscreen_kiosk_refresh_success:ratio5m
+          expr: |
+            sum by (tenant) (rate(bigscreen_kiosk_refresh_total{namespace="ljwx-health",service="ljwx-boot",result=~"success|not_modified"}[5m]))
+            /
+            clamp_min(sum by (tenant) (rate(bigscreen_kiosk_refresh_total{namespace="ljwx-health",service="ljwx-boot"}[5m])), 0.001)
+
+        - record: ljwx_health:bigscreen_kiosk_update_delay_p95:5m
+          expr: |
+            histogram_quantile(
+              0.95,
+              sum by (tenant, le) (rate(bigscreen_kiosk_last_update_delay_seconds_bucket{namespace="ljwx-health",service="ljwx-boot",result="success"}[5m]))
+            )
+
+        - record: ljwx_health:bigscreen_kiosk_recovery_p95:1h
+          expr: |
+            histogram_quantile(
+              0.95,
+              sum by (tenant, le) (rate(bigscreen_kiosk_recovery_latency_seconds_bucket{namespace="ljwx-health",service="ljwx-boot"}[1h]))
+            )
+
+        - record: ljwx_health:bigscreen_kiosk_heartbeat_age:seconds
+          expr: time() - max by (tenant) (bigscreen_kiosk_last_heartbeat_timestamp{namespace="ljwx-health",service="ljwx-boot"})
+
+        - alert: LjwxHealthBigscreenBffSuccessLow
+          expr: ljwx_health:bigscreen_bff_success:ratio5m < 0.995 and ljwx_health:bigscreen_bff_qps:5m > 0
+          for: 5m
+          labels:
+            severity: critical
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen BFF success ratio is below 99.5%"
+            description: "Tenant {{ $labels.tenant }} has missed the trial success target for five minutes."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
+
+        - alert: LjwxHealthBigscreenBffP95High
+          expr: ljwx_health:bigscreen_bff_latency_p95:5m > 0.5
+          for: 5m
+          labels:
+            severity: warning
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen BFF P95 latency is above 500ms"
+            description: "Tenant {{ $labels.tenant }} has exceeded the P95 trial target for five minutes."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
+
+        - alert: LjwxHealthBigscreenBffP99High
+          expr: ljwx_health:bigscreen_bff_latency_p99:5m > 1
+          for: 5m
+          labels:
+            severity: warning
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen BFF P99 latency is above one second"
+            description: "Tenant {{ $labels.tenant }} has exceeded the P99 trial target for five minutes."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
+
+        - alert: LjwxHealthBigscreenCacheHitLow
+          expr: |
+            ljwx_health:bigscreen_cache_hit:ratio5m < 0.70
+            and
+            sum by (tenant) (rate(bigscreen_snapshot_cache_access_total{namespace="ljwx-health",service="ljwx-boot",result=~"hit|miss"}[5m])) > 0
+          for: 10m
+          labels:
+            severity: warning
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen snapshot cache hit ratio is below 70%"
+            description: "Tenant {{ $labels.tenant }} has missed the Redis cache hit target for ten minutes."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
+
+        - alert: LjwxHealthBigscreenSnapshot304Low
+          expr: |
+            ljwx_health:bigscreen_snapshot_304:ratio5m < 0.30
+            and
+            sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint="snapshot"}[5m])) > 0
+          for: 15m
+          labels:
+            severity: warning
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen snapshot 304 ratio is below 30%"
+            description: "Tenant {{ $labels.tenant }} has missed the HTTP cache reuse target for fifteen minutes."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
+
+        - alert: LjwxHealthBigscreenKioskRefreshLow
+          expr: |
+            ljwx_health:bigscreen_kiosk_refresh_success:ratio5m < 0.995
+            and
+            sum by (tenant) (rate(bigscreen_kiosk_refresh_total{namespace="ljwx-health",service="ljwx-boot"}[5m])) > 0
+          for: 5m
+          labels:
+            severity: critical
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen kiosk refresh success ratio is below 99.5%"
+            description: "Tenant {{ $labels.tenant }} has missed the kiosk refresh target for five minutes."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
+
+        - alert: LjwxHealthBigscreenKioskUpdateDelayed
+          expr: ljwx_health:bigscreen_kiosk_update_delay_p95:5m > 2
+          for: 5m
+          labels:
+            severity: warning
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen kiosk last-update delay is above two seconds"
+            description: "Tenant {{ $labels.tenant }} has exceeded the generatedAt-to-render target for five minutes."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
+
+        - alert: LjwxHealthBigscreenKioskJsError
+          expr: sum by (tenant) (increase(bigscreen_kiosk_js_errors_total{namespace="ljwx-health",service="ljwx-boot"}[15m])) > 0
+          labels:
+            severity: critical
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen kiosk reported a JavaScript error"
+            description: "Tenant {{ $labels.tenant }} reported at least one JavaScript error in fifteen minutes."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
+
+        - alert: LjwxHealthBigscreenKioskContinuousOffline
+          expr: |
+            ljwx_health:bigscreen_kiosk_heartbeat_age:seconds > 300
+            or
+            max by (tenant) (bigscreen_kiosk_browser_online{namespace="ljwx-health",service="ljwx-boot"}) == 0
+            or
+            absent(bigscreen_kiosk_last_heartbeat_timestamp{namespace="ljwx-health",service="ljwx-boot",tenant="1"})
+          for: 2m
+          labels:
+            severity: critical
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen kiosk has been continuously offline"
+            description: "Tenant {{ $labels.tenant }} has no fresh online heartbeat."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
+
+        - alert: LjwxHealthBigscreenKioskRecoverySlow
+          expr: ljwx_health:bigscreen_kiosk_recovery_p95:1h > 30
+          for: 5m
+          labels:
+            severity: warning
+            category: bigscreen
+          annotations:
+            summary: "Bigscreen kiosk recovery P95 is above 30 seconds"
+            description: "Tenant {{ $labels.tenant }} has missed the offline recovery target."
+            runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"