Configure Alerting Rules¶

groups:
  # ── Access-Decay Score Alerts ─────────────────────────────────
  - name: verity.scores
    rules:
      - alert: HighDecayScoreDetected
        expr: verity_grant_decay_score > 80
        for: 10m
        labels:
          severity: warning
          team: identity-security
        annotations:
          summary: "High decay score on {{ $labels.principal }}→{{ $labels.asset }}"
          description: >-
            Grant {{ $labels.grant_id }} has a decay score of
            {{ $value | printf "%.1f" }} (threshold: 80).
            A review should be triggered automatically.
          runbook_url: https://mjtpena.github.io/verity/operations/runbooks/#high-decay-score

      - alert: CriticalDecayScore
        expr: verity_grant_decay_score > 95
        for: 5m
        labels:
          severity: critical
          team: identity-security
        annotations:
          summary: "Critical decay score — immediate review required"
          description: >-
            Grant {{ $labels.grant_id }} scored {{ $value | printf "%.1f" }}.
            Verify that the review generator has created a review and that
            an assignee is notified.

  # ── Review SLA Alerts ─────────────────────────────────────────
  - name: verity.reviews
    rules:
      - alert: ReviewSLABreached
        expr: >-
          (time() - verity_review_created_timestamp)
          > on(review_id) group_left() verity_review_sla_seconds
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: "Review {{ $labels.review_id }} has breached its SLA"
          description: >-
            Review for grant {{ $labels.grant_id }} exceeded the configured
            SLA. Check escalation status in the Workflow Engine.

      - alert: ReviewBacklogGrowing
        expr: verity_reviews_pending_total > 200
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Pending review backlog exceeds 200"

  # ── Service Health Alerts ─────────────────────────────────────
  - name: verity.health
    rules:
      - alert: ConnectorIngestStalled
        expr: rate(verity_events_ingested_total[15m]) == 0
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Connector {{ $labels.connector }} has stopped ingesting"
          runbook_url: https://mjtpena.github.io/verity/operations/runbooks/#connector-stalled

      - alert: KafkaConsumerLag
        expr: verity_kafka_consumer_lag > 10000
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Kafka consumer lag above 10 000 for {{ $labels.group_id }}"

      - alert: DecayEngineLatencyHigh
        expr: >-
          histogram_quantile(0.99, rate(verity_score_computation_duration_seconds_bucket[5m])) > 2
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "p99 decay-score computation exceeds 2 s"

global:
  resolve_timeout: 5m

route:
  receiver: default-slack
  group_by: ["alertname", "team"]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

  routes:
    # Critical → PagerDuty
    - match:
        severity: critical
      receiver: pagerduty-critical
      repeat_interval: 15m

    # Warning → Slack
    - match:
        severity: warning
      receiver: verity-slack-warnings

receivers:
  - name: default-slack
    slack_configs:
      - api_url: "https://hooks.slack.com/services/T00/B00/xxxx"
        channel: "#verity-alerts"
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ "\n" }}{{ end }}'

  - name: pagerduty-critical
    pagerduty_configs:
      - service_key: "<PAGERDUTY_INTEGRATION_KEY>"
        severity: critical

  - name: verity-slack-warnings
    slack_configs:
      - api_url: "https://hooks.slack.com/services/T00/B00/yyyy"
        channel: "#verity-warnings"