Alert arrives at Integration URL
  → Routing template (Jinja2, first True wins) selects escalation chain
  → Grouping ID template consolidates related alerts
  → Escalation chain fires: wait → notify schedule → wait → notify team lead
  → Users: acknowledge / resolve / silence from Slack, mobile, or web

# alertmanager.yml
receivers:
  - name: grafana-oncall
    webhook_configs:
      - url: https://your-oncall.grafana.net/integrations/v1/alertmanager/[id]/
        send_resolved: true
        max_alerts: 100   # prevent oversized payloads

route:
  receiver: grafana-oncall
  group_by: [alertname, cluster]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

# Send alert via formatted webhook
curl -X POST https://your-oncall.grafana.net/integrations/v1/formatted_webhook/[id]/ \
  -H "Content-Type: application/json" \
  -d '{
    "alert_uid": "incident-123",
    "title": "Database CPU High",
    "state": "alerting",
    "message": "db-prod-01 CPU at 95% for 10 minutes",
    "link_to_upstream_details": "https://grafana.example.com/d/abc123"
  }'

# Resolve the alert
curl -X POST https://your-oncall.grafana.net/integrations/v1/formatted_webhook/[id]/ \
  -H "Content-Type: application/json" \
  -d '{"alert_uid": "incident-123", "state": "ok"}'

{# Route critical alerts to PagerDuty escalation #}
{{ payload.labels.severity == "critical" }}

{# Route by team label #}
{{ payload.labels.team == "platform" }}

{# Route database alerts to DBA on-call #}
{{ "database" in payload.labels.get("component", "") }}

{# Default catch-all (always True) #}
{{ true }}

{{ payload.labels.alertname }}-{{ payload.labels.instance }}

{{ payload.field | b64decode }}                          # Decode base64
{{ "pattern" | regex_match(payload.message) }}           # Regex matching
{{ datetimeformat_as_timezone(payload.startsAt, "UTC") }} # Timezone display
{{ payload.values | tojson_pretty }}                     # Pretty-print JSON

Step 1: Notify users from schedule "Primary On-Call" (Important Notifications)
Step 2: Wait 5 minutes
Step 3: Notify users from schedule "Primary On-Call" (Default Notifications)
Step 4: Wait 10 minutes
Step 5: Notify whole team "Platform"
Step 6: Trigger webhook (PagerDuty, ticket system, etc.)

# API: create schedule from iCal
curl -X POST https://your-oncall.grafana.net/api/v1/schedules/ \
  -H "Authorization: your-api-key" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "Platform On-Call",
    "ical_url_primary": "https://calendar.example.com/platform-oncall.ics",
    "ical_url_overrides": "https://calendar.example.com/overrides.ics",
    "slack": {
      "channel_id": "C123456ABC",
      "user_group_id": "S123456ABC"
    }
  }'

resource "grafana_oncall_schedule" "platform" {
  name = "Platform On-Call"
  type = "calendar"

  shifts = [
    grafana_oncall_on_call_shift.weekday.id,
    grafana_oncall_on_call_shift.weekend.id,
  ]
}

resource "grafana_oncall_on_call_shift" "weekday" {
  name       = "Weekday"
  type       = "rolling_users"
  start      = "2024-01-01T09:00:00"
  duration   = 3600 * 8    # 8 hours
  frequency  = "weekly"
  users_per_slot = 1
  rolling_users  = [["user-id-1"], ["user-id-2"], ["user-id-3"]]
}

TOKEN=your-api-key

# List integrations
curl "$BASE/integrations/" -H "Authorization: $TOKEN"

# Create escalation chain
curl -X POST "$BASE/escalation_chains/" \
  -H "Authorization: $TOKEN" \
  -H "Content-Type: application/json" \
  -d '{"name": "Platform Critical", "team_id": "team-id"}'

# List schedules
curl "$BASE/schedules/" -H "Authorization: $TOKEN"

# List alert groups
curl "$BASE/alert_groups/?page=1&perpage=25" -H "Authorization: $TOKEN"

# Who is on-call right now
curl "$BASE/schedules/{schedule_id}/next_shifts/" -H "Authorization: $TOKEN"