Incident Runbook Agent Blueprint

You are an SRE incident responder. Your role is to assist on-call engineers
during incidents by applying runbook procedures. Follow this protocol:

1. THOUGHT: What type of incident is this? What systems are affected?
2. ACTION: Read the relevant runbook, analyze the incident details
3. Classify the incident: severity (SEV1-SEV4), impact scope, affected services
4. Match remediation steps from the runbook to the incident symptoms
5. Generate a timeline: detection → diagnosis → mitigation → resolution
6. After resolution, draft a blameless postmortem
7. FINAL_RESPONSE: severity classification + matched runbook steps +
   timeline + postmortem draft

Rules:
- Always classify severity before suggesting actions
- If symptoms don't match any runbook, suggest the closest match and flag it
- Timeline format: timestamp, action taken, outcome
- Postmortem must be blameless — focus on systems, process, and prevention
- If unsure about severity or the correct runbook, escalate to human — don't guess
- Never suggest running destructive commands (drop table, rm -rf, force push)

### Severity Matrix

```json
{
  "severity_levels": [
    {
      "level": "SEV1",
      "label": "Critical",
      "definition": "Complete service outage affecting all users. Revenue loss, data loss, or security breach in progress.",
      "response_time": "Immediate (5 min)",
      "escalation": "VP Engineering, CTO",
      "examples": ["Production database down", "Auth service returning 500 for all users", "Data breach confirmed"]
    },
    {
      "level": "SEV2",
      "label": "Major",
      "definition": "Significant degradation affecting many users. Core feature unavailable but partial workarounds exist.",
      "response_time": "15 min",
      "escalation": "Engineering Manager, On-call lead",
      "examples": ["Checkout flow failing for 30% of users", "Search returning stale results", "API latency > 5s p99"]
    },
    {
      "level": "SEV3",
      "label": "Minor",
      "definition": "Partial degradation affecting a subset of users or a non-critical feature.",
      "response_time": "1 hour",
      "escalation": "On-call engineer (no manager escalation)",
      "examples": ["Admin dashboard slow to load", "Email notifications delayed 10 min", "Non-critical cron job failing"]
    },
    {
      "level": "SEV4",
      "label": "Cosmetic",
      "definition": "No user impact. Visual glitch, typo, or internal tool issue.",
      "response_time": "Next business day",
      "escalation": "None (track in backlog)",
      "examples": ["Logo misaligned on settings page", "Typo in notification email", "Internal wiki page broken"]
    }
  ]
}

# tools.py
import os
import json
import re
from datetime import datetime

RUNBOOK_DIR = None

def list_runbooks():
    if not os.path.exists(RUNBOOK_DIR):
        return f"ERROR: Runbook directory not found: {RUNBOOK_DIR}"
    files = []
    for f in sorted(os.listdir(RUNBOOK_DIR)):
        if f.endswith(".md"):
            full = os.path.join(RUNBOOK_DIR, f)
            with open(full, "r") as fh:
                first_line = fh.readline().strip()
                title = first_line.lstrip("# ").strip() if first_line.startswith("#") else "(no title)"
            files.append(f"{f} — {title}")
    return "\n".join(files) if files else f"No .md runbooks found in {RUNBOOK_DIR}"

def read_runbook(path):
    full = os.path.join(RUNBOOK_DIR, path)
    if not os.path.exists(full):
        return f"ERROR: Runbook not found: {path}"
    with open(full, "r") as f:
        content = f.read()
    return content[:8000] if len(content) > 8000 else content

def classify_severity(incident_text, severity_matrix_path="severity_matrix.json"):
    if not os.path.exists(severity_matrix_path):
        return f"WARNING: Severity matrix not found. Cannot classify."
    with open(severity_matrix_path) as f:
        matrix = json.load(f)["severity_levels"]

    incident_lower = incident_text.lower()
    # Score each severity level by keyword matches from its examples
    scores = {}
    for level in matrix:
        score = 0
        for example in level.get("examples", []):
            example_words = set(example.lower().split())
            for word in example_words:
                if len(word) > 3 and word in incident_lower:
                    score += 1
        scores[level["level"]] = score

    best = max(scores, key=scores.get)
    if scores[best] > 0:
        reason = f"Matched {scores[best]} keywords from {best} examples"
    else:
        reason = "No keyword matches — defaulting to SEV4"

    return json.dumps({
        "severity": best,
        "reason": reason,
        "scores": scores
    }, indent=2)

def match_remediation(incident_text, runbook_text):
    steps = []
    # Extract all remediation steps (numbered or bulleted lists after "Remediation" or "Steps")
    sections = re.split(r'(?=^#{1,3}\s)', runbook_text, flags=re.MULTILINE)
    for section in sections:
        if re.search(r'(remediation|steps|resolution|mitigation)', section, re.IGNORECASE):
            lines = section.strip().split("\n")
            for line in lines:
                match = re.match(r'(?:\d+\.|\-|\*)\s+(.+)', line.strip())
                if match:
                    steps.append(match.group(1))

    if not steps:
        steps = ["No structured remediation steps found in runbook. Review manually."]

    return json.dumps({
        "matched_steps": steps,
        "step_count": len(steps),
        "warning": "Steps matched by heading proximity. Human verification required."
    }, indent=2)

def generate_timeline(events):
    timeline = []
    for i, event in enumerate(events):
        timestamp = event.get("time", datetime.now().isoformat())
        action = event.get("action", "Unknown action")
        outcome = event.get("outcome", "Unknown outcome")
        timeline.append(f"[{timestamp}] {action} → {outcome}")
    return json.dumps({"timeline": timeline, "event_count": len(timeline)}, indent=2)

def draft_postmortem(client, model, incident_text, timeline, resolution):
    prompt = f"""Write a blameless postmortem for this incident. Use the following structure:

# Incident Postmortem

## Summary
(2-3 sentences — what happened, impact, duration)

## Impact
- Users affected:
- Services affected:
- Duration:
- Revenue impact (if known):

## Timeline
{timeline}

## Root Cause
(What system or process failure caused this? Focus on systems, not individuals.)

## Remediation
{resolution}

## Prevention
(Specific, actionable changes to prevent recurrence: monitoring, alerts, runbook updates, code changes, process changes.)

Incident details:
{incident_text}

Keep the tone blameless and constructive. Focus on what we can change about our systems and processes."""

    response = client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": prompt}],
        temperature=0.2
    )
    return response.choices[0].message.content

# agent.py
import json
import os
import argparse
from openai import OpenAI
import tools as agent_tools

TOOL_SCHEMAS = [
    {
        "type": "function",
        "function": {
            "name": "list_runbooks",
            "description": "List all available runbooks in the runbook directory",
            "parameters": {"type": "object", "properties": {}, "required": []}
        }
    },
    {
        "type": "function",
        "function": {
            "name": "read_runbook",
            "description": "Read a specific runbook's full markdown content",
            "parameters": {
                "type": "object",
                "properties": {"path": {"type": "string"}},
                "required": ["path"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "classify_severity",
            "description": "Classify incident severity (SEV1-SEV4) based on impact and symptoms",
            "parameters": {
                "type": "object",
                "properties": {"incident_text": {"type": "string"}},
                "required": ["incident_text"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "match_remediation",
            "description": "Match incident symptoms to runbook remediation steps",
            "parameters": {
                "type": "object",
                "properties": {
                    "incident_text": {"type": "string"},
                    "runbook_text": {"type": "string"}
                },
                "required": ["incident_text", "runbook_text"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "generate_timeline",
            "description": "Generate an incident timeline from events",
            "parameters": {
                "type": "object",
                "properties": {
                    "events": {"type": "array", "items": {
                        "type": "object",
                        "properties": {
                            "time": {"type": "string"},
                            "action": {"type": "string"},
                            "outcome": {"type": "string"}
                        }
                    }}
                },
                "required": ["events"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "draft_postmortem",
            "description": "Generate a blameless postmortem document",
            "parameters": {
                "type": "object",
                "properties": {
                    "incident_text": {"type": "string"},
                    "timeline": {"type": "string"},
                    "resolution": {"type": "string"}
                },
                "required": ["incident_text", "timeline", "resolution"]
            }
        }
    }
]

SYSTEM_PROMPT = """You are an SRE incident responder. Your role is to assist on-call
engineers during incidents by applying runbook procedures. Follow this protocol:

1. THOUGHT: What type of incident is this? What systems are affected?
2. ACTION: Read the relevant runbook, analyze the incident details
3. Classify the incident: severity (SEV1-SEV4), impact scope, affected services
4. Match remediation steps from the runbook to the incident symptoms
5. Generate a timeline: detection → diagnosis → mitigation → resolution
6. After resolution, draft a blameless postmortem
7. FINAL_RESPONSE: severity classification + matched runbook steps +
   timeline + postmortem draft

Rules:
- Always classify severity before suggesting actions
- If symptoms don't match any runbook, suggest the closest match and flag it
- Timeline format: timestamp, action taken, outcome
- Postmortem must be blameless — focus on systems, process, and prevention
- If unsure about severity or runbook match, escalate to human — don't guess
- Never suggest running destructive commands"""


def run_agent(incident_text: str, config: dict, runbook_path: str = None):
    client = OpenAI(api_key=config["openai_api_key"])
    model = config.get("model", "gpt-4o")

    agent_tools.RUNBOOK_DIR = config.get("runbook_directory", "./runbooks")
    severity_path = config.get("severity_matrix_path", "severity_matrix.json")

    query = f"Analyze this incident:\n\n{incident_text}"
    if runbook_path:
        query += f"\n\nUse the runbook at: {runbook_path}"

    messages = [
        {"role": "system", "content": SYSTEM_PROMPT},
        {"role": "user", "content": query}
    ]

    for i in range(config.get("max_iterations", 6)):
        response = client.chat.completions.create(
            model=model,
            messages=messages,
            tools=TOOL_SCHEMAS,
            temperature=0.2
        )

        msg = response.choices[0].message
        messages.append(msg)

        if msg.content and "FINAL_RESPONSE:" in msg.content:
            return msg.content.split("FINAL_RESPONSE:", 1)[1].strip()

        if not msg.tool_calls:
            messages.append({
                "role": "user",
                "content": "Continue the incident analysis. Classify severity, match runbook steps, generate timeline, and provide FINAL_RESPONSE."
            })
            continue

        for tool_call in msg.tool_calls:
            func_name = tool_call.function.name
            func_args = json.loads(tool_call.function.arguments)

            if func_name == "list_runbooks":
                result = agent_tools.list_runbooks()
            elif func_name == "read_runbook":
                result = agent_tools.read_runbook(func_args.get("path", ""))
            elif func_name == "classify_severity":
                result = agent_tools.classify_severity(
                    func_args.get("incident_text", ""), severity_path)
            elif func_name == "match_remediation":
                result = agent_tools.match_remediation(
                    func_args.get("incident_text", ""),
                    func_args.get("runbook_text", ""))
            elif func_name == "generate_timeline":
                result = agent_tools.generate_timeline(
                    func_args.get("events", []))
            elif func_name == "draft_postmortem":
                result = agent_tools.draft_postmortem(
                    client, model,
                    func_args.get("incident_text", ""),
                    func_args.get("timeline", ""),
                    func_args.get("resolution", ""))
            else:
                result = f"Unknown tool: {func_name}"

            messages.append({
                "role": "tool",
                "tool_call_id": tool_call.id,
                "content": result
            })

    return "Agent reached max iterations."


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--incident", required=True, help="Path to incident details file or quoted text")
    parser.add_argument("--runbook", help="Specific runbook file to use (optional)")
    parser.add_argument("--config", default="config.json")
    args = parser.parse_args()

    with open(args.config) as f:
        config = json.load(f)

    # Read incident from file or treat as raw text
    incident_text = args.incident
    if os.path.exists(args.incident):
        with open(args.incident) as f:
            incident_text = f.read()

    result = run_agent(incident_text, config, args.runbook)
    print(result)