ai-incidents 1.0.0

examples/config.toml

@@ -0,0 +1,93 @@
+# ai-incidents: full configuration reference.
+#
+# Default location: ~/.config/ai-incidents/config.toml (or $AI_INCIDENTS_CONFIG, or -c PATH).
+# Every key is optional except [ledger] dir. Unknown keys are an error, so a typo cannot silently
+# switch a setting off. Paths accept ~ and $VARIABLES.
+
+# --- Where the transcripts are ---------------------------------------------------------------
+# Read-only, always. Omit every [[source]] to read both defaults below.
+# Give two sources of the same type distinct names, e.g. a second Claude Code config directory.
+
+[[source]]
+type = "claude-code"
+path = "~/.claude/projects"                      # default; honours $CLAUDE_CONFIG_DIR
+# name = "claude-code"                           # default: the type
+
+[[source]]
+type = "opencode"
+path = "~/.local/share/opencode/opencode*.db"    # default; a glob, to catch channel builds
+
+# --- The pre-filter --------------------------------------------------------------------------
+
+[scan]
+max_candidates = 50          # most candidates shown to the judge in one run; the rest wait
+max_per_session = 12         # most candidates from any one session
+first_run = "baseline"       # "baseline": record existing sessions as judged, without judging
+                             # "backfill": judge them, max_candidates at a time
+exclude = []                 # fnmatch globs on the transcript path, e.g. ["*/-home-me-scratch*"]
+extra_destructive = []       # extra regexes for commands that count as destructive
+extra_benign = []            # extra regexes for destructive-looking commands that are routine
+extra_alarm = []             # extra regexes for your own alarms, e.g. ["MyJobOverdue"]
+
+# --- The judge -------------------------------------------------------------------------------
+# One model call per run, and only when there are candidates. It gets the candidates on stdin and
+# has no tools. The candidates leave this machine for whichever model you choose here.
+
+[judge]
+backend = "claude"           # "claude" | "openai" | "command"
+model = "sonnet"
+timeout = 900                # seconds
+prompt_file = ""             # replace the built-in rubric (see src/ai_incidents/prompts/judge.md)
+
+# backend = "claude": the Claude Code CLI, run with every tool and MCP server disabled.
+binary = "claude"
+extra_args = []              # appended to the claude command line
+
+# backend = "openai": any OpenAI-compatible endpoint. This is the local-model option.
+# base_url = "http://localhost:8080/v1"        # llama.cpp server
+# base_url = "http://localhost:11434/v1"       # Ollama
+# model = "qwen3:32b"
+# api_key_env = "LOCAL_LLM_KEY"                # name of an env var; never the key itself
+# json_mode = true                             # send response_format = json_object
+# max_tokens = 16000
+
+# backend = "command": any program that reads the prompt on stdin and prints the answer.
+# command = ["ollama", "run", "qwen3:32b"]
+
+# --- The output ------------------------------------------------------------------------------
+
+[ledger]
+dir = "~/ai-incidents-ledger"    # required
+order = "severity"               # "severity": most severe first | "newest"
+title = "AI incidents"           # heading for a new incidents.md
+
+[git]
+enabled = true               # commit if ledger.dir is a git repository; ignored otherwise
+push = false
+remote = "origin"
+branch = ""                  # refuse to run on any other branch; empty = whatever is checked out
+expected_remote_url = ""     # refuse to push if the remote has been repointed
+author_name = ""             # commit identity; empty = git's own configuration
+author_email = ""
+
+[state]
+file = "~/.local/state/ai-incidents/state.json"   # default ($XDG_STATE_HOME)
+seen_list = ""               # optional plain-text export of judged Claude Code transcripts, one
+                             # `<file name>:<mtime>:<size>` per line, for a cleanup job that must
+                             # not delete what has not been judged. Read back on start.
+
+[privacy]
+redact = true                # mask secret-shaped strings before the judge and before the ledger
+extra_redact_patterns = []   # extra regexes to mask
+
+# --- Hooks -----------------------------------------------------------------------------------
+# Commands, split like a shell would but not run by one. They get the run summary on stdin and in
+# AI_INCIDENTS_* environment variables. A failing hook is logged and ignored.
+
+[hooks]
+notify_cmd = ""              # after a run that filed something; quiet runs are silent
+on_success_cmd = ""          # after every successful run (a heartbeat for a dead-man's switch)
+on_failure_cmd = ""          # after a failed run
+timeout = 60
+# notify_cmd = "curl -s -H 'Title: ai-incidents' --data-binary @- https://ntfy.sh/your-topic"
+# on_success_cmd = "curl -fsS -m 10 https://hc-ping.com/your-uuid"

examples/cron.txt

@@ -0,0 +1,12 @@
+# crontab -e
+#
+# Nightly at 02:30. cron's PATH is minimal: give the full path to ai-incidents, and make sure the
+# judge's CLI (claude, by default) is reachable too. Output goes to a log you can read later; the
+# exit code is 0 for a successful run (quiet or not), 1 for a failed one, 3 if a run was already
+# in progress.
+
+PATH=/home/you/.local/bin:/usr/local/bin:/usr/bin:/bin
+30 2 * * * ai-incidents run >> "$HOME/.local/state/ai-incidents/cron.log" 2>&1
+
+# With a notification when something is filed, and a heartbeat on every successful run:
+# 30 2 * * * ai-incidents run --notify-cmd "curl -s --data-binary @- https://ntfy.sh/your-topic" >> "$HOME/.local/state/ai-incidents/cron.log" 2>&1

examples/sample-ledger.md

@@ -0,0 +1,33 @@
+# AI incidents
+
+A curated excerpt from a real ledger kept by `ai-incidents`: five of its entries, with host names,
+project names, people and infrastructure details generalized. The format is exactly what the tool
+writes. Most severe first; newest first within a severity.
+
+## Cleanup deleted the user's own agent sessions, unrecoverably, twice in one day · 2026-09-01 · HIGH
+- **What:** cleaning up its own smoke-test sessions, the agent filtered a global session list by name and by time window and piped the result into a delete; the filter also matched the user's real sessions, including the one open in their terminal, and after reporting the damage it ran the same filter again later that day.
+- **Cost:** at least five sessions destroyed with nothing recoverable; the user's active session was cut off mid-use and unrelated work was lost.
+- **Lesson:** never choose what to delete by name or time window over a shared list; delete only what you can prove you created (a recorded ID list, a tagged prefix), and after a filter has deleted someone else's data once, redesign it before running it again.
+
+## An hourly agent's cost, once measured: ~47.6 billion tokens a year · 2026-07-14 · HIGH
+- **What:** an ingestion pipeline ran an unattended agent every hour; measured over a week, each run used about 5 million tokens, some ran past 200 turns, and 11 of 27 runs found no work at all.
+- **Cost:** roughly 47.6 billion tokens a year, drawn from the same subscription quota as interactive work, so the bill was paid invisibly in rate-limit headroom.
+- **Lesson:** gate an unattended agent behind a deterministic check so a model is loaded only when there is work, price the model to the work, and treat cadence as a cost multiplier.
+
+## Deleted source data after mistaking its own bug for corruption · 2026-07-11 · HIGH
+- **What:** decided a price-history file was corrupt ("905 missing trading days") and deleted it; the corruption was the agent's own validation bug, because a parquet round-trip had renamed the date column and broken its comparison.
+- **Cost:** the file could not be fetched again (the free source returned nothing and the paid one was rate-limited), so the data loss stood.
+- **Lesson:** never delete source data to exclude it; quarantine it or filter at read time, and check a corruption diagnosis against a known-good sample before acting on it.
+
+## Committed 14 embedded git repositories to `main` with `git add -A` · 2026-07-14 · MEDIUM
+- **What:** ran `git add -A && git commit && git push` in a worktree where a scheduled review job had left 14 throwaway clones; git staged each one as an embedded repository and the commit went to `main`, while printing `warning: adding embedded git repository` 14 times from a command that exited 0.
+- **Cost:** `main` carried 14 broken submodule references that broke cloning until a follow-up commit removed them; it also exposed that the review job had been cloning into its own checkout on every run.
+- **Lesson:** `git add -A` is not a review step: stage explicit paths anywhere a tool may have written, and read the warnings from commands that succeed.
+
+## Monitoring went blind: an inline comment corrupted a metric value · 2026-07-14 · MEDIUM
+- **What:** wrote `MaxAge=9d   # comment` into a config file; a homegrown metrics generator read the comment as part of the value, and the metrics exporter rejected the whole file. The ready explanation, that the exporter was being strict about comments, was wrong.
+- **Cost:** every job-health metric vanished from monitoring until it was caught in the same session, so any job failure in that window would have gone unnoticed.
+- **Lesson:** keep comments off value lines in key=value files, test the parser against trailing content, and when a tool rejects your output, read what your own code wrote before blaming the tool.
+
+## Recurring patterns
+- **Matching by name or time window instead of an ID you own, before deleting or killing.** A loose match over a shared namespace (sessions, processes, files) eventually catches something that is not yours.

examples/sample-report.md

@@ -0,0 +1,24 @@
+# AI-incidents run · 2026-07-19
+
+## What the pre-filter found
+
+- **claude-code:** 390 new session(s), 376 clean, 14 with candidates
+- **Candidates:** 50
+- **Judge:** claude CLI, model sonnet
+
+## Confirmed incidents (1)
+
+### Committed 14 embedded git repositories to `main` with `git add -A` · 2026-07-14 · MEDIUM
+**Category:** destructive-action · **Sessions:** `a41c07e2` (claude-code, 2026-07-14)
+
+Clears the bar: `main` carried 14 broken submodule references until a follow-up commit removed them, and the warning that proves it came from a command that exited 0.
+
+## Excluded candidates (5)
+
+- **C03** `5d2b9f10` (claude-code, 2026-07-15): `rm -f` of a previous attempt's own scratch files before writing a clean replacement. Own-scratch cleanup, not an incident.
+- **C11, C12** `e8f41a77` (claude-code, 2026-07-17): two self-described mistakes during a test-suite reduction, both caught by the compiler and fixed before anything was committed. No cost survived.
+- **C19** `0c9d3e55` (claude-code, 2026-07-16): a memory-leak diagnosis that ended up blaming an upstream project. Checked against the misattribution bar, but the session cited a specific, real upstream report and took no action on its earlier wrong theory. Not filed.
+- **C27** `9b61aa04` (claude-code, 2026-07-17): a long library migration with several self-caught missteps, each verified back to a correct final state. Routine iteration.
+
+*This is a curated excerpt of a real run report, generalized the same way as `sample-ledger.md`;
+the candidate list at the end of a full report is omitted.*

examples/systemd/ai-incidents.service

@@ -0,0 +1,24 @@
+# A systemd *user* service. Install with:
+#   cp ai-incidents.service ai-incidents.timer ~/.config/systemd/user/
+#   systemctl --user daemon-reload
+#   systemctl --user enable --now ai-incidents.timer
+#   loginctl enable-linger "$USER"     # so it runs while you are logged out
+#
+# Check on it with:
+#   systemctl --user list-timers ai-incidents.timer
+#   journalctl --user -u ai-incidents.service
+
+[Unit]
+Description=ai-incidents: judge new agent sessions and update the ledger
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+# pipx / uv tool installs land in ~/.local/bin, which is not on a user unit's PATH.
+Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
+ExecStart=%h/.local/bin/ai-incidents run
+# A quiet run makes no model call and finishes in seconds. A busy one waits on the judge
+# ([judge] timeout, default 900s); leave headroom over it.
+TimeoutStartSec=1200
+Nice=10

examples/systemd/ai-incidents.timer

@@ -0,0 +1,13 @@
+[Unit]
+Description=Run ai-incidents nightly
+
+[Timer]
+# Nightly is plenty: the state file makes runs incremental, and a night with nothing to judge
+# costs no model call. Run it more often than your transcripts are cleaned up (Claude Code's
+# cleanupPeriodDays defaults to 30), or sessions can be deleted before they are judged.
+OnCalendar=*-*-* 02:30:00
+Persistent=true
+RandomizedDelaySec=600
+
+[Install]
+WantedBy=timers.target