ljwx-health/ljwx-boot/k8s/prometheusrule.yml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: ljwx-boot-p0-alerts
  namespace: ljwx-health
  labels:
    app: ljwx-boot
    release: prometheus
    monitoring: p0
spec:
  groups:
  # ========== 服务可用性告警 ==========
  - name: ljwx-boot.availability
    interval: 30s
    rules:
    - alert: LjwxBootDown
      expr: up{job="ljwx-boot"} == 0
      for: 1m
      labels:
        severity: critical
        component: ljwx-boot
      annotations:
        summary: "LJWX Boot服务不可用"
        description: "LJWX Boot服务 {{ $labels.pod }} 已停止响应超过1分钟"
        runbook_url: "https://wiki.ljwx.com/runbooks/ljwx-boot-down"

    - alert: LjwxBootHighErrorRate
      expr: |
        rate(http_server_requests_seconds_count{status=~"5..",job="ljwx-boot"}[5m])
        /
        rate(http_server_requests_seconds_count{job="ljwx-boot"}[5m]) > 0.05
      for: 5m
      labels:
        severity: warning
        component: ljwx-boot
      annotations:
        summary: "LJWX Boot错误率过高"
        description: "Pod {{ $labels.pod }} 5xx错误率超过5%已持续5分钟"

  # ========== P0-3 Redis去重告警 ==========
  - name: ljwx-boot.p0-3-redis-dedup
    interval: 30s
    rules:
    - alert: RedisConnectionFailed
      expr: health_dedup_mode{job="ljwx-boot"} == 0
      for: 2m
      labels:
        severity: critical
        component: redis
        feature: p0-3
      annotations:
        summary: "Redis连接失败，已降级到内存模式"
        description: "Pod {{ $labels.pod }} Redis连接失败，去重功能已切换到内存模式，可能影响去重准确性"
        action: "检查Redis服务状态和网络连接"

    - alert: RedisMemoryHigh
      expr: |
        redis_memory_used_bytes{job="ljwx-boot"}
        /
        redis_memory_max_bytes > 0.8
      for: 10m
      labels:
        severity: warning
        component: redis
        feature: p0-3
      annotations:
        summary: "Redis内存使用率超过80%"
        description: "Redis实例 {{ $labels.instance }} 内存使用率 {{ $value | humanizePercentage }}，去重Key可能被驱逐"
        action: "考虑增加Redis内存或调整TTL策略"

    - alert: RedisKeyExplosion
      expr: |
        rate(redis_db_keys{db="database1",job="ljwx-boot"}[15m]) > 1000
      for: 5m
      labels:
        severity: warning
        component: redis
        feature: p0-3
      annotations:
        summary: "Redis去重Key增长异常"
        description: "Redis Key增长速度超过1000/分钟，可能存在数据异常或攻击"
        action: "检查数据上传日志和用户行为"

  # ========== P0-4 缓存失效告警 ==========
  - name: ljwx-boot.p0-4-cache-invalidation
    interval: 30s
    rules:
    - alert: CacheInvalidationDisabled
      expr: health_cache_invalidation_enabled{job="ljwx-boot"} == 0
      for: 1m
      labels:
        severity: warning
        component: cache
        feature: p0-4
      annotations:
        summary: "缓存自动失效功能已禁用"
        description: "Pod {{ $labels.pod }} 缓存自动失效功能处于禁用状态，可能导致数据不一致"
        action: "检查配置 health.cache.auto-invalidation"

    - alert: CacheInvalidationHighFailureRate
      expr: |
        rate(health_cache_invalidation_total{result="failure",job="ljwx-boot"}[5m])
        /
        rate(health_cache_invalidation_total{job="ljwx-boot"}[5m]) > 0.05
      for: 10m
      labels:
        severity: warning
        component: cache
        feature: p0-4
      annotations:
        summary: "缓存失效失败率过高"
        description: "Pod {{ $labels.pod }} 缓存失效失败率 {{ $value | humanizePercentage }} 超过5%"
        action: "检查Redis连接和缓存服务日志"

    - alert: CacheInvalidationSlowResponse
      expr: |
        histogram_quantile(0.95,
          rate(health_cache_invalidation_duration_seconds_bucket{job="ljwx-boot"}[5m])
        ) > 0.5
      for: 10m
      labels:
        severity: warning
        component: cache
        feature: p0-4
      annotations:
        summary: "缓存失效响应时间过长"
        description: "Pod {{ $labels.pod }} P95缓存失效耗时超过500ms"
        action: "检查Redis性能和网络延迟"

  # ========== JVM资源告警 ==========
  - name: ljwx-boot.jvm-resources
    interval: 30s
    rules:
    - alert: JvmMemoryHigh
      expr: |
        jvm_memory_used_bytes{area="heap",job="ljwx-boot"}
        /
        jvm_memory_max_bytes{area="heap"} > 0.85
      for: 10m
      labels:
        severity: warning
        component: jvm
      annotations:
        summary: "JVM堆内存使用率过高"
        description: "Pod {{ $labels.pod }} 堆内存使用率 {{ $value | humanizePercentage }} 超过85%"
        action: "检查内存泄漏或考虑增加JVM内存"

    - alert: JvmGcTimeHigh
      expr: |
        rate(jvm_gc_pause_seconds_sum{job="ljwx-boot"}[5m])
        /
        rate(jvm_gc_pause_seconds_count{job="ljwx-boot"}[5m]) > 0.1
      for: 10m
      labels:
        severity: warning
        component: jvm
      annotations:
        summary: "JVM GC时间占比过高"
        description: "Pod {{ $labels.pod }} 平均GC停顿时间 {{ $value | humanizeDuration }} 超过100ms"

  # ========== 数据库连接池告警 ==========
  - name: ljwx-boot.database
    interval: 30s
    rules:
    - alert: DatabaseConnectionPoolExhausted
      expr: |
        hikaricp_connections_active{job="ljwx-boot"}
        /
        hikaricp_connections{job="ljwx-boot"} > 0.9
      for: 5m
      labels:
        severity: critical
        component: database
      annotations:
        summary: "数据库连接池即将耗尽"
        description: "Pod {{ $labels.pod }} 活跃连接占比 {{ $value | humanizePercentage }} 超过90%"
        action: "检查慢查询或考虑增加连接池大小"

    - alert: DatabaseConnectionPoolLow
      expr: hikaricp_connections_idle{job="ljwx-boot"} == 0
      for: 5m
      labels:
        severity: warning
        component: database
      annotations:
        summary: "数据库连接池无空闲连接"
        description: "Pod {{ $labels.pod }} 连接池无空闲连接已持续5分钟"

  # ========== 应用性能告警 ==========
  - name: ljwx-boot.performance
    interval: 30s
    rules:
    - alert: ApiResponseTimeSlow
      expr: |
        histogram_quantile(0.95,
          rate(http_server_requests_seconds_bucket{job="ljwx-boot",uri!~"/actuator.*"}[5m])
        ) > 1.0
      for: 10m
      labels:
        severity: warning
        component: api
      annotations:
        summary: "API响应时间过慢"
        description: "Pod {{ $labels.pod }} P95响应时间超过1秒"
        action: "检查慢接口和数据库性能"

    - alert: HighConcurrentRequests
      expr: |
        http_server_requests_active_seconds_count{job="ljwx-boot"} > 100
      for: 5m
      labels:
        severity: warning
        component: api
      annotations:
        summary: "并发请求数过高"
        description: "Pod {{ $labels.pod }} 并发请求数 {{ $value }} 超过100"

  # ========== Pod资源告警 ==========
  - name: ljwx-boot.pod-resources
    interval: 30s
    rules:
    - alert: PodCpuHigh
      expr: |
        rate(container_cpu_usage_seconds_total{pod=~"ljwx-boot.*",namespace="ljwx-health"}[5m]) > 0.8
      for: 10m
      labels:
        severity: warning
        component: pod
      annotations:
        summary: "Pod CPU使用率过高"
        description: "Pod {{ $labels.pod }} CPU使用率超过80%已持续10分钟"

    - alert: PodMemoryHigh
      expr: |
        container_memory_usage_bytes{pod=~"ljwx-boot.*",namespace="ljwx-health"}
        /
        container_spec_memory_limit_bytes > 0.85
      for: 10m
      labels:
        severity: warning
        component: pod
      annotations:
        summary: "Pod内存使用率过高"
        description: "Pod {{ $labels.pod }} 内存使用率 {{ $value | humanizePercentage }} 超过85%"

    - alert: PodRestartFrequent
      expr: |
        increase(kube_pod_container_status_restarts_total{pod=~"ljwx-boot.*",namespace="ljwx-health"}[1h]) > 3
      for: 0m
      labels:
        severity: critical
        component: pod
      annotations:
        summary: "Pod重启频繁"
        description: "Pod {{ $labels.pod }} 在过去1小时内重启超过3次"
        action: "检查Pod日志和健康检查配置"