ROLE: Software Engineer
Auto-remediate production incidents with policies, parallel diagnosis, and human sign-off for deploys
halo
// ============================================================
// ROLE: Software Engineer
// PURPOSE: Auto-remediate production incidents with policies,
// parallel diagnosis, and human sign-off for deploys
// ============================================================
// --- Policies ---
policy ChangeSafety : Block(DestructiveOps) & Require(Approver)
policy CostControl : MaxSpend(5.00)
// --- Tools ---
tool GitHub : POST("api.github.com/repos", apiKey: Env.GH_TOKEN)
tool PagerDuty : POST("api.pagerduty.com/incidents")
tool SlackDev : POST("hooks.slack.com/services/eng-oncall")
tool RunbookDB : PostgreSQL(Env.DB_URI, schema: "runbooks")
// --- Store ---
tool PastIncidents : VectorStore(Env.PINECONE_KEY, namespace: "incident-history")
tool CodebaseIndex : VectorStore(Env.PINECONE_KEY, namespace: "code-embeddings")
// --- Agents ---
// Cheap triage — runs fast on every alert
agent Triage [openai/gpt-4o-mini, temp: 0.0]
system """
You are an incident triage specialist. Rapidly classify
incoming alerts by severity and known-issue patterns.
Respond with: category, severity (P0-P4), and confidence.
"""
requires CostControl
// Deep analyzer — correlates logs, metrics, and code
@shadow(Triage)
agent Diagnoser [anthropic/claude-sonnet-4, temp: 0.05]
system """
You are a senior SRE diagnostician. Identify root cause
by correlating error logs, metrics anomalies, and recent
code changes. Always include a confidence score and list
the evidence chain that led to your diagnosis.
"""
equips GitHub, CodebaseIndex, PastIncidents
requires CostControl
// Code reviewer — validates any generated fix
agent CodeReviewer [anthropic/claude-sonnet-4, temp: 0.0]
system """
You are an automated code review agent. Evaluate diffs
for correctness, safety, and regression risk. Flag any
destructive operations. Approve only safe, minimal changes.
"""
equips GitHub
requires ChangeSafety, CostControl
// --- The Pipeline ---
@timeout(15m)
@retry(2, 5s)
pipeline AutoRemediate(incident: PagerDuty.Incident) -> Resolution:
incident
// Step 1: Fast classification — is this a known issue type?
|> Triage.classify(
context.data.incident.summary,
context.data.incident.last_50_errors
) as triage
// Step 2: Fan-out — parallel investigation across three signals
|* {
Diagnoser.analyze_logs(
context.data.incident.error_window,
context.data.incident.service_name
) as diagnosis,
Diagnoser.search_codebase(
CodebaseIndex,
context.data.incident.affected_endpoint,
10
) as codebase,
Diagnoser.search_history(
PastIncidents,
context.data.incident.summary,
0.8
) as history
}
// Step 3: Gate — only auto-fix if confidence > 85%
|? (context.data.diagnosis.confidence > 85)
// Step 4: Generate the fix
|> Diagnoser.generate_fix(
context.data.diagnosis,
context.data.codebase.matches,
context.data.incident.service_language
) as fix
// Step 5: Code review gate — must pass automated review
|> CodeReviewer.review(
context.data.fix.diff,
context.data.incident.service_name
) as review
// Step 6: Gate on review approval
|? (context.data.review.approved == true)
// Step 7: Create PR and notify on-call engineer
|> GitHub.create_pull_request(
context.data.incident.repo,
context.data.fix.title,
context.data.fix.description
) as github
// Step 8: Human-in-the-loop — engineer must merge
|! yield(
SlackDev,
context.data.fix.summary,
[context.data.github.pr_url, context.data.fix.diff],
1h
)
// Step 9: Wait for engineer action
~> listen(SlackDev.Reaction) as engineer
// Step 10: Gate on merge confirmation
|? (context.data.engineer.action == "merged")
|> System.ResolveIncident(
context.data.incident.id,
context.data.fix.title
)