449 lines
22 KiB
YAML
449 lines
22 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: ljwx-health-slo
|
|
namespace: ljwx-health
|
|
labels:
|
|
release: monitoring
|
|
app.kubernetes.io/name: ljwx-health
|
|
app.kubernetes.io/component: observability
|
|
spec:
|
|
groups:
|
|
- name: ljwx_health_availability
|
|
interval: 30s
|
|
rules:
|
|
- alert: LjwxHealthDeploymentUnavailable
|
|
expr: kube_deployment_status_replicas_available{namespace="ljwx-health",deployment=~"ljwx-(boot|admin)"} < 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
category: availability
|
|
annotations:
|
|
summary: "LJWX Health deployment is unavailable"
|
|
description: "{{ $labels.deployment }} has had no available replica for more than two minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- alert: LjwxHealthBootMetricsTargetMissing
|
|
expr: absent(up{namespace="ljwx-health",service="ljwx-boot"})
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
category: observability
|
|
service: ljwx-boot
|
|
annotations:
|
|
summary: "Prometheus has no ljwx-boot target"
|
|
description: "The application may be healthy, but its metrics target has been absent for five minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- alert: LjwxHealthBootMetricsTargetDown
|
|
expr: up{namespace="ljwx-health",service="ljwx-boot"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
category: availability
|
|
service: ljwx-boot
|
|
annotations:
|
|
summary: "Prometheus cannot scrape ljwx-boot"
|
|
description: "The ljwx-boot metrics target has been down for more than two minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- alert: LjwxHealthPodRestarting
|
|
expr: increase(kube_pod_container_status_restarts_total{namespace="ljwx-health",container=~"ljwx-(boot|admin)"}[30m]) > 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
category: availability
|
|
annotations:
|
|
summary: "LJWX Health pod is restarting repeatedly"
|
|
description: "{{ $labels.pod }} restarted more than twice in 30 minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- name: ljwx_health_golden_signals
|
|
interval: 30s
|
|
rules:
|
|
- record: ljwx_health:http_requests:rate5m
|
|
expr: sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-boot",uri!~"/actuator.*"}[5m]))
|
|
|
|
- record: ljwx_health:http_5xx:ratio5m
|
|
expr: |
|
|
sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-boot",status=~"5..",uri!~"/actuator.*"}[5m]))
|
|
/
|
|
clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-boot",uri!~"/actuator.*"}[5m])), 0.001)
|
|
|
|
- record: ljwx_health:http_5xx:ratio1h
|
|
expr: |
|
|
sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-boot",status=~"5..",uri!~"/actuator.*"}[1h]))
|
|
/
|
|
clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-boot",uri!~"/actuator.*"}[1h])), 0.001)
|
|
|
|
- record: ljwx_health:http_5xx:ratio30m
|
|
expr: |
|
|
sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-boot",status=~"5..",uri!~"/actuator.*"}[30m]))
|
|
/
|
|
clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-boot",uri!~"/actuator.*"}[30m])), 0.001)
|
|
|
|
- record: ljwx_health:http_5xx:ratio6h
|
|
expr: |
|
|
sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-boot",status=~"5..",uri!~"/actuator.*"}[6h]))
|
|
/
|
|
clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-boot",uri!~"/actuator.*"}[6h])), 0.001)
|
|
|
|
- record: ljwx_health:http_latency_p95:5m
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum(rate(http_server_requests_seconds_bucket{namespace="ljwx-health",service="ljwx-boot",uri!~"/actuator.*"}[5m])) by (le)
|
|
)
|
|
|
|
- alert: LjwxHealthBootHighErrorRate
|
|
expr: ljwx_health:http_5xx:ratio5m > 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
category: errors
|
|
service: ljwx-boot
|
|
annotations:
|
|
summary: "ljwx-boot 5xx ratio is above 5%"
|
|
description: "The five-minute server error ratio has remained above 5%."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- alert: LjwxHealthBootHighP95Latency
|
|
expr: ljwx_health:http_latency_p95:5m > 1
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
category: latency
|
|
service: ljwx-boot
|
|
annotations:
|
|
summary: "ljwx-boot P95 latency is above one second"
|
|
description: "The five-minute P95 latency has remained above one second."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- alert: LjwxHealthBootJvmHeapHigh
|
|
expr: sum(jvm_memory_used_bytes{namespace="ljwx-health",service="ljwx-boot",area="heap"}) / clamp_min(sum(jvm_memory_max_bytes{namespace="ljwx-health",service="ljwx-boot",area="heap"}), 1) > 0.85
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
category: saturation
|
|
service: ljwx-boot
|
|
annotations:
|
|
summary: "ljwx-boot JVM heap is above 85%"
|
|
description: "Heap saturation has remained above 85% for ten minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- alert: LjwxHealthBootDbPoolSaturated
|
|
expr: sum(hikaricp_connections_active{namespace="ljwx-health",service="ljwx-boot"}) / clamp_min(sum(hikaricp_connections_max{namespace="ljwx-health",service="ljwx-boot"}), 1) > 0.9
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
category: saturation
|
|
service: ljwx-boot
|
|
annotations:
|
|
summary: "ljwx-boot database pool is above 90%"
|
|
description: "The database connection pool has remained saturated for five minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- name: ljwx_health_slo_burn_rate
|
|
interval: 30s
|
|
rules:
|
|
- alert: LjwxHealthSloFastBurn
|
|
expr: ljwx_health:http_5xx:ratio5m > 0.0144 and ljwx_health:http_5xx:ratio1h > 0.0144
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
category: slo
|
|
service: ljwx-boot
|
|
slo_target: "99.9"
|
|
annotations:
|
|
summary: "LJWX Health is consuming error budget rapidly"
|
|
description: "The 5m and 1h windows exceed the 14.4x burn threshold for the 99.9% SLO."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- alert: LjwxHealthSloSlowBurn
|
|
expr: ljwx_health:http_5xx:ratio30m > 0.006 and ljwx_health:http_5xx:ratio6h > 0.006
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
category: slo
|
|
service: ljwx-boot
|
|
slo_target: "99.9"
|
|
annotations:
|
|
summary: "LJWX Health is steadily consuming error budget"
|
|
description: "The 30m and 6h windows exceed the 6x burn threshold for the 99.9% SLO."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- name: ljwx_health_edge_connectivity
|
|
interval: 30s
|
|
rules:
|
|
- alert: LjwxHealthCloudflaredUnavailable
|
|
expr: kube_deployment_status_replicas_available{namespace="ljwx-health",deployment="ljwx-health-cloudflared"} < 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
category: edge
|
|
annotations:
|
|
summary: "LJWX Health Cloudflare Tunnel is unavailable"
|
|
description: "No in-cluster cloudflared connector has been available for two minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/edge-tunnel.md"
|
|
|
|
- alert: LjwxHealthCloudflaredRedundancyLost
|
|
expr: sum(cloudflared_tunnel_ha_connections{namespace="ljwx-health"}) < 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
category: edge
|
|
annotations:
|
|
summary: "LJWX Health Cloudflare Tunnel lost connector redundancy"
|
|
description: "Fewer than two Cloudflare edge connections have been active for five minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/edge-tunnel.md"
|
|
|
|
- alert: LjwxHealthCloudflaredMetricsMissing
|
|
expr: absent(up{namespace="ljwx-health",service="ljwx-health-cloudflared"})
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
category: observability
|
|
annotations:
|
|
summary: "LJWX Health Cloudflare Tunnel metrics are missing"
|
|
description: "Prometheus has not discovered cloudflared metrics for five minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/edge-tunnel.md"
|
|
|
|
- name: ljwx_health_message_chain
|
|
interval: 30s
|
|
rules:
|
|
- alert: LjwxHealthMessageOutboxBacklog
|
|
expr: max(ljwx_message_outbox_queue_size{namespace="ljwx-health",service="ljwx-boot"}) > 1000
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
category: message
|
|
service: ljwx-boot
|
|
annotations:
|
|
summary: "LJWX Health message outbox backlog is high"
|
|
description: "The message outbox has contained more than 1000 pending events for ten minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- alert: LjwxHealthMessageOutboxDeliveryFailureRate
|
|
expr: |
|
|
sum(rate(ljwx_message_outbox_delivery_total{namespace="ljwx-health",service="ljwx-boot",status="failure"}[5m]))
|
|
/
|
|
clamp_min(sum(rate(ljwx_message_outbox_delivery_total{namespace="ljwx-health",service="ljwx-boot"}[5m])), 0.001) > 0.05
|
|
and
|
|
sum(rate(ljwx_message_outbox_delivery_total{namespace="ljwx-health",service="ljwx-boot"}[5m])) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
category: message
|
|
service: ljwx-boot
|
|
annotations:
|
|
summary: "LJWX Health message outbox delivery is failing"
|
|
description: "More than 5% of message outbox deliveries have failed for ten minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- alert: LjwxHealthMessageResponseLatencyHigh
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum(rate(ljwx_message_response_latency_seconds_bucket{namespace="ljwx-health",service="ljwx-boot"}[10m])) by (le)
|
|
) > 300
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
category: message
|
|
service: ljwx-boot
|
|
annotations:
|
|
summary: "LJWX Health message response P95 is above five minutes"
|
|
description: "The send-to-response P95 has remained above 300 seconds for fifteen minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/service-degradation.md"
|
|
|
|
- name: ljwx_health_bigscreen_trial
|
|
interval: 30s
|
|
rules:
|
|
- record: ljwx_health:bigscreen_bff_qps:5m
|
|
expr: sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime"}[5m]))
|
|
|
|
- record: ljwx_health:bigscreen_bff_success:ratio5m
|
|
expr: |
|
|
sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime",result=~"success|not_modified"}[5m]))
|
|
/
|
|
clamp_min(sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime"}[5m])), 0.001)
|
|
|
|
- record: ljwx_health:bigscreen_bff_latency_p95:5m
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum by (tenant, le) (rate(bigscreen_bff_request_latency_seconds_bucket{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime"}[5m]))
|
|
)
|
|
|
|
- record: ljwx_health:bigscreen_bff_latency_p99:5m
|
|
expr: |
|
|
histogram_quantile(
|
|
0.99,
|
|
sum by (tenant, le) (rate(bigscreen_bff_request_latency_seconds_bucket{namespace="ljwx-health",service="ljwx-boot",endpoint=~"snapshot|realtime"}[5m]))
|
|
)
|
|
|
|
- record: ljwx_health:bigscreen_cache_hit:ratio5m
|
|
expr: |
|
|
sum by (tenant) (rate(bigscreen_snapshot_cache_access_total{namespace="ljwx-health",service="ljwx-boot",result="hit"}[5m]))
|
|
/
|
|
clamp_min(sum by (tenant) (rate(bigscreen_snapshot_cache_access_total{namespace="ljwx-health",service="ljwx-boot",result=~"hit|miss"}[5m])), 0.001)
|
|
|
|
- record: ljwx_health:bigscreen_snapshot_304:ratio5m
|
|
expr: |
|
|
sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint="snapshot",result="not_modified"}[5m]))
|
|
/
|
|
clamp_min(sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint="snapshot"}[5m])), 0.001)
|
|
|
|
- record: ljwx_health:bigscreen_snapshot_build_p95:5m
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum by (tenant, le) (rate(bigscreen_snapshot_build_latency_seconds_bucket{namespace="ljwx-health",service="ljwx-boot"}[5m]))
|
|
)
|
|
|
|
- record: ljwx_health:bigscreen_kiosk_refresh_success:ratio5m
|
|
expr: |
|
|
sum by (tenant) (rate(bigscreen_kiosk_refresh_total{namespace="ljwx-health",service="ljwx-boot",result=~"success|not_modified"}[5m]))
|
|
/
|
|
clamp_min(sum by (tenant) (rate(bigscreen_kiosk_refresh_total{namespace="ljwx-health",service="ljwx-boot"}[5m])), 0.001)
|
|
|
|
- record: ljwx_health:bigscreen_kiosk_update_delay_p95:5m
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum by (tenant, le) (rate(bigscreen_kiosk_last_update_delay_seconds_bucket{namespace="ljwx-health",service="ljwx-boot",result="success"}[5m]))
|
|
)
|
|
|
|
- record: ljwx_health:bigscreen_kiosk_recovery_p95:1h
|
|
expr: |
|
|
histogram_quantile(
|
|
0.95,
|
|
sum by (tenant, le) (rate(bigscreen_kiosk_recovery_latency_seconds_bucket{namespace="ljwx-health",service="ljwx-boot"}[1h]))
|
|
)
|
|
|
|
- record: ljwx_health:bigscreen_kiosk_heartbeat_age:seconds
|
|
expr: time() - max by (tenant) (bigscreen_kiosk_last_heartbeat_timestamp{namespace="ljwx-health",service="ljwx-boot"})
|
|
|
|
- alert: LjwxHealthBigscreenBffSuccessLow
|
|
expr: ljwx_health:bigscreen_bff_success:ratio5m < 0.995 and ljwx_health:bigscreen_bff_qps:5m > 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen BFF success ratio is below 99.5%"
|
|
description: "Tenant {{ $labels.tenant }} has missed the trial success target for five minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|
|
|
|
- alert: LjwxHealthBigscreenBffP95High
|
|
expr: ljwx_health:bigscreen_bff_latency_p95:5m > 0.5
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen BFF P95 latency is above 500ms"
|
|
description: "Tenant {{ $labels.tenant }} has exceeded the P95 trial target for five minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|
|
|
|
- alert: LjwxHealthBigscreenBffP99High
|
|
expr: ljwx_health:bigscreen_bff_latency_p99:5m > 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen BFF P99 latency is above one second"
|
|
description: "Tenant {{ $labels.tenant }} has exceeded the P99 trial target for five minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|
|
|
|
- alert: LjwxHealthBigscreenCacheHitLow
|
|
expr: |
|
|
ljwx_health:bigscreen_cache_hit:ratio5m < 0.70
|
|
and
|
|
sum by (tenant) (rate(bigscreen_snapshot_cache_access_total{namespace="ljwx-health",service="ljwx-boot",result=~"hit|miss"}[5m])) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen snapshot cache hit ratio is below 70%"
|
|
description: "Tenant {{ $labels.tenant }} has missed the Redis cache hit target for ten minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|
|
|
|
- alert: LjwxHealthBigscreenSnapshot304Low
|
|
expr: |
|
|
ljwx_health:bigscreen_snapshot_304:ratio5m < 0.30
|
|
and
|
|
sum by (tenant) (rate(bigscreen_bff_requests_total{namespace="ljwx-health",service="ljwx-boot",endpoint="snapshot"}[5m])) > 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen snapshot 304 ratio is below 30%"
|
|
description: "Tenant {{ $labels.tenant }} has missed the HTTP cache reuse target for fifteen minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|
|
|
|
- alert: LjwxHealthBigscreenKioskRefreshLow
|
|
expr: |
|
|
ljwx_health:bigscreen_kiosk_refresh_success:ratio5m < 0.995
|
|
and
|
|
sum by (tenant) (rate(bigscreen_kiosk_refresh_total{namespace="ljwx-health",service="ljwx-boot"}[5m])) > 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen kiosk refresh success ratio is below 99.5%"
|
|
description: "Tenant {{ $labels.tenant }} has missed the kiosk refresh target for five minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|
|
|
|
- alert: LjwxHealthBigscreenKioskUpdateDelayed
|
|
expr: ljwx_health:bigscreen_kiosk_update_delay_p95:5m > 2
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen kiosk last-update delay is above two seconds"
|
|
description: "Tenant {{ $labels.tenant }} has exceeded the generatedAt-to-render target for five minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|
|
|
|
- alert: LjwxHealthBigscreenKioskJsError
|
|
expr: sum by (tenant) (increase(bigscreen_kiosk_js_errors_total{namespace="ljwx-health",service="ljwx-boot"}[15m])) > 0
|
|
labels:
|
|
severity: critical
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen kiosk reported a JavaScript error"
|
|
description: "Tenant {{ $labels.tenant }} reported at least one JavaScript error in fifteen minutes."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|
|
|
|
- alert: LjwxHealthBigscreenKioskContinuousOffline
|
|
expr: |
|
|
ljwx_health:bigscreen_kiosk_heartbeat_age:seconds > 300
|
|
or
|
|
max by (tenant) (bigscreen_kiosk_browser_online{namespace="ljwx-health",service="ljwx-boot"}) == 0
|
|
or
|
|
absent(bigscreen_kiosk_last_heartbeat_timestamp{namespace="ljwx-health",service="ljwx-boot",tenant="1"})
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen kiosk has been continuously offline"
|
|
description: "Tenant {{ $labels.tenant }} has no fresh online heartbeat."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|
|
|
|
- alert: LjwxHealthBigscreenKioskRecoverySlow
|
|
expr: ljwx_health:bigscreen_kiosk_recovery_p95:1h > 30
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
category: bigscreen
|
|
annotations:
|
|
summary: "Bigscreen kiosk recovery P95 is above 30 seconds"
|
|
description: "Tenant {{ $labels.tenant }} has missed the offline recovery target."
|
|
runbook_url: "https://github.com/lingjingwanxiang/ljwx-health/blob/main/docs/04-operations/runbooks/bigscreen-observability.md"
|