docs: refresh the TrueNAS compatibility matrix

Reviewed-on: #12

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: sudolulo
+ko_fi: sudolulo

.github/workflows/ci.yml

@@ -43,20 +43,21 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: install dev deps
-        run: python -m pip install --upgrade pip pytest ruff
+      # uv-managed interpreters instead of actions/setup-python: the prebuilt-CPython
+      # download path setup-python relies on does not work on the self-hosted Gitea
+      # runner (all three matrix jobs failed at setup there while passing on GitHub);
+      # uv works identically on both.
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
 
       - name: ruff
-        run: ruff check patch tests tools
+        # Pinned: an unpinned ruff means any upstream release can turn main red
+        # with no code change.
+        run: uvx ruff@0.16.1 check patch tests tools
 
       - name: pytest
-        run: pytest tests -v
+        run: uvx --python ${{ matrix.python }} pytest tests -v
 
       - name: verify injected middleware blocks compile
         # Belt-and-braces: the *_BLOCK strings are appended into live middlewared
         # modules. A syntax error there would break the box at boot.
-        run: pytest tests/test_apply_blocks.py -v
+        run: uvx --python ${{ matrix.python }} pytest tests/test_apply_blocks.py -v

.github/workflows/compat.yml

@@ -24,10 +24,18 @@ on:
     paths:
       # The manifest itself changed -- re-check immediately rather than waiting a day.
       - "tools/compat.py"
+      # ...and so did the thing that PUBLISHES the finding. This was missing, and it
+      # showed: the commit teaching the bot to refresh a stale report body touched only
+      # compat_publish.py, so no run fired, and the report stayed stale until the next
+      # scheduled one. A fix nobody runs is a fix nobody has.
+      - "tools/compat_publish.py"
       - ".github/workflows/compat.yml"
 
 permissions:
-  contents: read
+  # write, because the matrix refresh pushes a branch and opens a PR. It does NOT get
+  # to move `main` -- that is the whole reason it is a PR. See the refresh step below.
+  contents: write
+  pull-requests: write
   issues: write
 
 jobs:
@@ -130,17 +138,27 @@ jobs:
       - name: matrix
         run: cat /tmp/matrix.md
 
-      # Keep the README's table true. A support matrix that quietly goes stale is not
-      # a stale doc -- it is a false promise to somebody deciding whether to trust
-      # this with their backups.
+      # Keep the README's table true — as a PULL REQUEST, on the CANONICAL forge.
       #
-      # Only ever touches the block between the COMPAT MATRIX markers, and only on
-      # the canonical host (Gitea) so the two forges cannot race each other. The
-      # `paths:` trigger above does not include README.md, so this cannot re-trigger
-      # itself; and a README change is documentation-only, which by design raises no
-      # update alert on anyone's box.
-      - name: refresh the README matrix
+      # Two things this gets right that the obvious version gets wrong:
+      #
+      #  1. It is a PR, not a push to main. This used to `git push origin HEAD:main`
+      #     from CI. An unattended write to main is exactly what the release barrier
+      #     exists to prevent — a bot that can move main can move it somewhere nobody
+      #     looked. Nothing lands by itself.
+      #
+      #  2. It runs on GITEA, not GitHub. GitHub is a one-way MIRROR: a PR merged there
+      #     would be silently clobbered by the next `fleet-repos mirror` push from Gitea.
+      #     A bot opening PRs against a mirror is a bot doing nothing, slowly.
+      #
+      # A stale support matrix is not a stale doc — it is a false promise to somebody
+      # deciding whether to trust this with their backups. So it is refreshed daily; it
+      # just asks first.
+      - name: refresh the README matrix (PR on the canonical forge)
         if: ${{ github.event_name == 'schedule' && !contains(github.server_url, 'github.com') }}
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
         run: |
           python3 - <<'PY'
           import json, sys
@@ -151,87 +169,73 @@ jobs:
           print("changed" if compat.update_readme(rows) else "unchanged")
           PY
 
-          if ! git diff --quiet -- README.md; then
-            git config user.name  "truecloud-patch bot"
-            git config user.email "bot@onetick.ninja"
-            git add README.md
-            git commit -m "docs: refresh the TrueNAS compatibility matrix"
-            git push origin HEAD:main
-          fi
+          git diff --quiet -- README.md && { echo "matrix unchanged — nothing to propose"; exit 0; }
 
-      # A broken SHIPPED release is an outage: users are on it right now.
+          git config user.name  "truecloud-patch bot"
+          git config user.email "bot@onetick.ninja"
+
+          BRANCH=bot/compat-matrix
+          git checkout -B "$BRANCH"
+          git add README.md
+          git commit -m "docs: refresh the TrueNAS compatibility matrix"
+          git push -f origin "$BRANCH"
+
+          # ONE long-lived PR, force-pushed in place — not a new one every morning.
+          # (A daily PR is the same mistake as a daily comment, wearing a hat.)
+          BRANCH="$BRANCH" python3 - <<'PY'
+          import json, os, urllib.error, urllib.request
+
+          api, token, branch = os.environ["API"], os.environ["TOKEN"], os.environ["BRANCH"]
+          h = {"Authorization": f"token {token}", "Content-Type": "application/json"}
+
+          def call(url, method="GET", data=None):
+              r = urllib.request.Request(
+                  url, method=method, headers=h,
+                  data=json.dumps(data).encode() if data else None)
+              with urllib.request.urlopen(r) as resp:  # noqa: S310
+                  return json.load(resp) if resp.length != 0 else {}
+
+          existing = [
+              p for p in call(f"{api}/pulls?state=open")
+              if p["head"]["ref"] == branch
+          ]
+          if existing:
+              print(f"PR #{existing[0]['number']} already open; the force-push updated it")
+          else:
+              pr = call(f"{api}/pulls", "POST", {
+                  "head": branch, "base": "main",
+                  "title": "docs: refresh the TrueNAS compatibility matrix",
+                  "body": (
+                      "The daily compatibility check found that the support matrix in "
+                      "the README no longer matches iXsystems' actual middleware.\n\n"
+                      "This only touches the block between the `COMPAT MATRIX` markers. "
+                      "It is regenerated by `tools/compat.py --matrix --update-readme` "
+                      "and force-pushed, so it always reflects the latest run."
+                  ),
+              })
+              print(f"opened PR #{pr['number']}")
+          PY
+
+      # ONE bug report, kept in sync. It is edited in place when the findings change and
+      # says NOTHING when they do not.
+      #
+      # The first version commented on every run and left 11 identical 3,000-character
+      # comments on one issue in a single day. A bot that repeats itself daily gets
+      # muted, and then the next real finding is scrolled past — which defeats the whole
+      # reason for building it.
+      #
+      # Runs on whichever forge it lands on; compat_publish.py handles both, so the two
+      # cannot drift.
+      - name: file / update / close the bug report
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
+          API: ${{ contains(github.server_url, 'github.com') && 'https://api.github.com' || format('{0}/api/v1', github.server_url) }}/repos/${{ github.repository }}
+        run: python3 tools/compat_publish.py --api "$API" --token "$TOKEN" --matrix /tmp/matrix.json
+
+      # A broken SHIPPED release is an outage: users are on it right now. Fails LAST, so
+      # the report is filed before the job goes red.
       - name: fail if a shipped release is broken
         if: ${{ steps.check.outputs.shipped_broken != '0' }}
         run: |
           echo "::error::The patch is broken on a SHIPPED TrueNAS release."
           exit 1
-
-      - name: file a bug report (GitHub)
-        if: ${{ steps.report.outputs.broken == '1' && contains(github.server_url, 'github.com') }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-          TITLE: "TrueNAS compatibility: the patch's assumptions no longer hold"
-        run: |
-          # One issue per set of broken refs, reopened/updated rather than duplicated
-          # daily -- a bot that files the same issue every morning gets muted, and
-          # then it is not a warning system any more.
-          # Lowest-numbered match, for the same reason as the Gitea step below.
-          existing="$(gh issue list --state all --search "$TITLE" \
-                        --json number,title \
-                        --jq '[.[] | select(.title == env.TITLE) | .number] | min // empty')"
-
-          if [ -n "$existing" ]; then
-            gh issue comment "$existing" --body-file /tmp/issue.md
-            gh issue reopen "$existing" 2>/dev/null || true
-          else
-            gh issue create --title "$TITLE" --body-file /tmp/issue.md
-          fi
-
-      - name: file a bug report (Gitea)
-        if: ${{ steps.report.outputs.broken == '1' && !contains(github.server_url, 'github.com') }}
-        env:
-          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
-          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
-          TITLE: "TrueNAS compatibility: the patch's assumptions no longer hold"
-        run: |
-          # python3, not jq: jq is not guaranteed on a self-hosted runner, and a bug
-          # report that dies on a missing tool is a warning system that does not warn.
-          python3 - <<'PY'
-          import json, os, urllib.error, urllib.request
-
-          api, token, title = os.environ["API"], os.environ["TOKEN"], os.environ["TITLE"]
-          with open("/tmp/issue.md", encoding="utf-8") as fh:
-              body = fh.read()
-          headers = {"Authorization": f"token {token}",
-                     "Content-Type": "application/json"}
-
-          def call(url, method, data=None):
-              req = urllib.request.Request(
-                  url, method=method, headers=headers,
-                  data=json.dumps(data).encode() if data else None)
-              with urllib.request.urlopen(req) as r:  # noqa: S310
-                  return json.load(r) if r.length != 0 else {}
-
-          # Same title => same issue. Comment on it rather than filing a new one every
-          # morning: a bot that duplicates itself daily gets muted, and then it is not
-          # a warning system any more.
-          # LOWEST-numbered match, not "whichever the API returns first". Two issues
-          # with the same title already existed once (the old title embedded the ref
-          # list, so the identity changed when that set changed), and an
-          # order-dependent pick would have alternated between them, reopening one and
-          # commenting on the other. Lowest number is stable no matter what the API
-          # sorts by.
-          issues = call(f"{api}/issues?state=all&type=issues", "GET")
-          matches = sorted((i for i in issues if i["title"] == title),
-                           key=lambda i: i["number"])
-          match = matches[0] if matches else None
-
-          if match:
-              n = match["number"]
-              call(f"{api}/issues/{n}/comments", "POST", {"body": body})
-              call(f"{api}/issues/{n}", "PATCH", {"state": "open"})
-              print(f"commented on and reopened issue #{n}")
-          else:
-              made = call(f"{api}/issues", "POST", {"title": title, "body": body})
-              print(f"filed issue #{made['number']}")
-          PY

.github/workflows/release.yml

@@ -124,7 +124,7 @@ jobs:
           echo "--- release body ---"
           cat /tmp/notes.md
 
-      # This repo is canonically hosted on Gitea (git.onetick.ninja) and mirrored to
+      # This repo is canonically hosted on Gitea (git.arch.fyi) and mirrored to
       # GitHub, and BOTH run this workflow -- Gitea reads .github/workflows too. So
       # the publish step has to work on whichever forge it lands on. Everything
       # above is forge-agnostic; only the "create a release" API differs.

CHANGELOG.md

@@ -6,6 +6,248 @@ is deliberate: see [Releasing](docs/releasing.md). Twelve releases were cut on
 live, every one of those interrupts every user. An alert people learn to ignore is
 worse than no alert, because one day it carries a security fix.
 
+## Unreleased
+### Changed
+
+- **CI's python matrix is green on the self-hosted Gitea runner again.** The real
+  failure was that the Gitea runner image executes jobs as root, and the two
+  unreadable-sidecar tests build their scenario with `chmod(0)` — which cannot make
+  a file unreadable for root (`CAP_DAC_OVERRIDE`). Those two tests now skip as root
+  with that reason; GitHub's non-root runner still exercises them. The matrix also
+  moved to uv-managed interpreters (one toolchain across both runners) and ruff is
+  pinned to 0.16.1 so an upstream ruff release can't turn `main` red without a code
+  change.
+- **README badges point at the public GitHub mirror** (workflow status and
+  releases) instead of the private forge. The release badge had also been reading
+  the stale Gitea v0.6.1 release instead of the current v0.7.0 on GitHub.
+
+- **`master` is now labelled `27-dev`, because it is not the next release.** iX
+  branches each major onto its own `release/` line and master rolls straight on to the
+  one after — on 2026-07-14 every recent commit on master targeted `27.0.0-BETA.1`
+  while 26 was still in beta. So a **BROKEN** master row, rendered as
+  "master _(unreleased)_", read as *"the version you are about to install is broken"*
+  when the breakage was a major release away on a line nobody can download. In a table
+  whose entire job is helping somebody decide whether to trust this with their backups,
+  that is a false alarm in the worst possible place. The label is derived from the
+  newest major in the matrix plus one, so it rolls over to `28-dev` by itself once 27
+  branches.
+
+  For the record, the breakage is `NAS-141498` (2026-06-24), "Convert cloud_backup
+  plugin to the typesafe pattern": it re-signatures `restic_backup` and
+  `get_restic_config`, splitting `entry`/`credentials` out of the `cloud_backup` dict.
+  It is deliberately not being chased while the 27 line is still churning.
+
+### Fixed
+
+- **The next maintenance release was never checked, and it is the one that reaches
+  users.** Shipped versions were discovered from `TS-*` tags and unreleased ones from
+  `release/*` branches carrying `-BETA`/`-RC`. A branched-but-untagged *maintenance*
+  release is neither: `release/25.10.5` has no tag, and its line has already shipped,
+  so the "a prerelease of a shipped line is history" filter discarded it. It was
+  invisible — and it is precisely what a 25.10.4 box gets on its next update. A break
+  there would have reached real users before the daily check ever looked at it, on the
+  only line anybody is actually running.
+
+  A plain `release/X.Y.Z` branch is now checked when its line **has** shipped and it
+  sorts **newer** than that line's newest tag. Both things that must stay out fall out
+  of the same rule: `release/24.10-RC.2` sorts older than `TS-24.10.2.4` (history, not
+  a warning), and iX's typo branch `release/25.20.2.2` is on a line that has no tag at
+  all, so it is not a release line. This immediately surfaced two refs that had never
+  been checked — `release/25.10.5` and `release/24.10.2.5` — both of which pass.
+
+  `is_unreleased()` now keys off where a ref came from (branch = not yet shipped)
+  rather than looking for `-BETA`/`-RC` in its name. Otherwise `release/25.10.5` would
+  count as shipped and a break in it would fail the build as a live outage — on a
+  version nobody is running yet.
+
+- **An unchanged fingerprint froze the bug report's body, not just its comments.** Two
+  questions were sharing one answer. *Have the findings changed?* gates **comments** —
+  they notify, and a daily "still broken, same as yesterday" is what teaches everyone
+  to ignore the one that finally matters. *Is the body still true?* gates the **body** —
+  and editing an issue body notifies nobody on either forge, so keeping it honest is
+  free. Conflated, the report could never be corrected while the findings held steady,
+  and the fingerprint deliberately ignores everything that moves on its own — healthy
+  rows, the hardware-verified column, point releases, and how a row is labelled. The
+  `master` → `27-dev` relabel above would have reached the README and never the issue
+  anybody actually opens. The body is now rewritten whenever it is out of date (after
+  normalising line endings, so a forge round-tripping `\r\n` does not cause a rewrite
+  every run) and comments remain strictly a changelog of real changes.
+
+- **A change to the publisher did not re-run the check.** `compat.yml`'s `push:` paths
+  listed `tools/compat.py` but not `tools/compat_publish.py` — so the very commit that
+  taught the bot to refresh a stale report body triggered no run, and the report stayed
+  stale until the next scheduled one. A fix nobody runs is a fix nobody has.
+
+- **The compatibility bot filed a new duplicate bug report on every Gitea run.**
+  `find_issue()` skipped pull requests by testing for the *presence* of the
+  `pull_request` key. GitHub omits that key on a plain issue; Gitea sends it as
+  `null`. So on Gitea every issue was discarded as a PR, the lookup always came back
+  empty, and the bot took the "nothing filed yet" branch and opened a fresh report
+  each run — **nine copies on the canonical forge**, four of them filed *after* the
+  commit that was meant to stop precisely this. The mirror was fine, which is why it
+  went unnoticed: GitHub's payload shape is the one the filter was written against.
+
+  It is the same failure the anti-spam fix was written to prevent, moved from
+  comments to issues, and it survived because `find_issue` was the only function in
+  `compat_publish.py` with no test. It now has one, per forge, and the daily cron —
+  which had not yet run once — no longer accumulates a report a day.
+
+  The issue list is also requested with **both** paging parameters (`per_page` for
+  GitHub, `limit` for Gitea). Each forge ignores the other's, and Gitea's default page
+  is 30, so the lookup would have started missing the report again once the pile it
+  was creating grew past one page.
+
+## v0.7.0 — 2026-07-14
+### Added
+
+- **TrueNAS 26 support, verified on a real TrueNAS 26 install.** 26 deletes
+  `plugins/zfs_/` outright, taking the private `zfs.dataset.query`,
+  `zfs.snapshot.query` and `zfs.snapshot.delete` with it. Every one of those was on
+  the nested module's critical path, so nested snapshots were **BROKEN** on 26 and
+  `apply.sh` correctly refused to apply the module there.
+
+  Snapshot **deletion** now resolves its namespace at runtime — `pool.snapshot` on
+  25.10 and 26, `zfs.snapshot` on 24.10 and 25.04, because no single namespace spans
+  every supported release. `tools/compat.py` checks the same list the runtime uses,
+  so what CI verifies and what runs cannot drift apart.
+
+  Hardware-verified on TrueNAS 26.0.0-BETA.1: a 274-snapshot recursive backup of a
+  292-dataset pool, then a **byte-identical restore of a four-level-deep child
+  dataset**.
+
+### Fixed
+
+- **Enumeration no longer trusts middleware's dataset and snapshot queries — they
+  are filtered.** This is the important one, and it is the bug that a test VM caught
+  and no amount of source analysis ever could have.
+
+  The obvious port of the deleted private `zfs.dataset.query` was the public
+  `pool.dataset.query`. It exists, it is documented, it is covered by iX's
+  deprecation policy — and it is **not a like-for-like replacement**. It applies a
+  *visibility policy*: it hides the datasets TrueNAS considers its own — `ix-apps/*`,
+  `.system/*`, `.ix-virt/*`. On a real pool that is **84 of 270 datasets**, and
+  `ix-apps` holds **live application data**.
+
+  Staging from that view would have silently omitted every one of them. Worse,
+  `plan_staging()` would never have seen them, so they would not have appeared in its
+  `skipped` list either — no warning, no failure, just a green backup quietly missing
+  data. That is precisely the failure this module exists to prevent. The snapshot
+  query lies the same way (205 of 274), so the sweep would have orphaned one snapshot
+  per hidden dataset, on every run, forever.
+
+  The module now **reads the truth from ZFS and makes changes through middleware**:
+  enumeration is `zfs list`, which no policy can filter and which behaves identically
+  on every release; mutation stays a middleware call, so TrueNAS's own bookkeeping
+  stays consistent. A failing `zfs list` raises rather than returning an empty list —
+  "no datasets" and "the command broke" must never look the same.
+
+  **No shipped release is affected.** v0.6.1 and earlier call the *private*
+  `zfs.dataset.query`, which returns all 270 datasets. The bug existed only in the
+  unreleased TrueNAS 26 port.
+
+- **The patch now owns the snapshot sweep even when it does not stage anything.**
+  Stock decides whether to take a *recursive* snapshot by its own rule, and on
+  TrueNAS 26 that rule stopped being ours.
+
+  Up to 25.10, stock's `create_snapshot` called `get_dataset_recursive()` — the same
+  function this module vendors — so "stock went recursive" and "we have something to
+  stage" were the *same question*, and stock's non-recursive delete was correct for
+  everything the patch declined to stage. TrueNAS 26 uses `filesystem.statfs`:
+  `recursive = (path == the dataset's mountpoint)`. The two rules now disagree for a
+  dataset whose only descendants are **ZVOLs** or **legacy/none-mountpoint** datasets
+  — stock snapshots it recursively, while the patch sees nothing to stage.
+
+  The patch then handed the snapshot back to stock, which destroys the parent only.
+  With no staging tree there was no sidecar, and the garbage collector only ever ran
+  from the staging path — so nothing on the box would ever have found the children.
+  Reproduced on the test VM: one orphaned snapshot per zvol, on every run, forever,
+  with the backup reporting success. Ownership of the sweep is no longer conditional
+  on staging.
+
+- **The runtime resolved a *namespace*; the checker verified a *method*.** Those are
+  different questions, and the gap is a false "ok". `get_service()` only proves a
+  namespace is registered — it says nothing about whether `delete` still exists on it.
+  So if iX guts the method while keeping the service (they have already done exactly
+  that to `pool.snapshot.do_update` on master), `tools/compat.py` would fall through
+  to `zfs.snapshot`, report the box healthy, and let the patch apply — while the
+  runtime picked `pool.snapshot` and failed *every* delete, orphaning the whole tree.
+  Both sides now ask the same question, and a test binds the two lists together.
+
+- `query_filesystems()` **dropped malformed `zfs list` rows silently** — the last
+  remaining silent-omission path, and a direct contradiction of this module's cardinal
+  rule. It raises now. A missing `zfs` binary raised `FileNotFoundError` rather than
+  `ZfsError`; also fixed.
+
+- The snapshot retry loop **discarded the delete error** and reported every survivor
+  as "(still busy?)" — naming the one cause that is benign and self-healing, and
+  hiding the ones that are permanent. It keeps and reports the real error.
+
+- The staging-failure handler could **lose the original exception** if its own cleanup
+  sweep raised. An error handler must not be able to lose the error.
+
+- **A snapshot delete that returns cleanly is not proof that anything was deleted.**
+  The recursive sweep's fast path took the call's word for it and returned "no
+  survivors" — so `cleanup_task` read that as a clean sweep and removed the sidecar,
+  the only record the tree ever existed. Roughly 250 snapshots would have been orphaned
+  on every run, with nothing left able to find them, and the backup reporting success.
+
+  This is not a hypothetical about a well-behaved API: iX has already gutted
+  `pool.snapshot.do_update` on master into a no-op whose body is commented out and
+  which returns `None`. A source check still sees the `def`; a runtime check still sees
+  a callable method. Only asking ZFS can tell. The sweep now confirms against ZFS, and
+  where it *cannot* confirm it keeps owning the tree rather than claiming success — a
+  false survivor self-heals on the next run, a lost record never does.
+
+- `_write_sidecar` **swallowed `OSError`**. The sidecar is the only thing that survives
+  a middlewared restart; failing to write it is not fatal, but it must never be
+  invisible. `_read_sidecar` had the mirror bug — it conflated "there is no sidecar"
+  with "I could not read the sidecar", and `cleanup_task` then took the empty branch
+  and **unlinked the only record** of a tree it had failed to read.
+
+- **A dataset from another tree, mounted inside the backup path, was omitted
+  silently.** The staging plan scopes by dataset *name*, which is correct — a dataset
+  with no mountpoint cannot be scoped by path at all. But ZFS lets any dataset mount
+  anywhere, so one from an unrelated tree can sit inside the path:
+
+      Tank/photos   mountpoint=/mnt/Tap/apps/photos
+
+  It holds data inside the backed-up path, and `zfs snapshot -r Tap@…` does **not**
+  cover it: recursion follows the dataset tree, not the directory tree. So there is no
+  snapshot of it to stage, and no way to capture it consistently with the rest. It fell
+  out of the name filter and vanished — not staged, not in `skipped`, no error, backup
+  green. Stock has the same blind spot, but stock also refuses the nested config
+  outright; this patch is what relaxes that guard, so the hole is this patch's to close.
+  It now refuses, and names the offending datasets.
+
+## v0.6.1 — 2026-07-13
+### Fixed
+
+- **A reboot mid-backup orphaned the entire snapshot tree, permanently.** The sidecar
+  is the record of which snapshots a run pinned — and it lives in `/run`, which is
+  **tmpfs**. A reboot (or a crash) between taking the recursive snapshot and cleaning
+  it up destroyed that record, leaving one snapshot per descendant dataset — **250+ on
+  a real pool** — with nothing left pointing at them. Nothing would ever have found
+  them again.
+
+  `gc_stale_snapshots()` is the backstop: it identifies leftovers **by name**, so it
+  works when the record is gone. It runs at the start of every backup, after the
+  sidecar reclaim — the recorded path stays authoritative, and the collector only ever
+  mops up what the record lost.
+
+  Because it deletes data on a *name match* — a weaker claim than a recorded fact — the
+  selection is a **pure function** with the harshest tests in the suite. A snapshot is
+  collected only if **all** of these hold:
+
+  | | |
+  | --- | --- |
+  | name is exactly `<dataset>@<task>-<YYYYMMDDHHMMSS>` | so `cloud_backup-5` never matches `cloud_backup-50`, an `auto-*` periodic snapshot, or anything a human made |
+  | it is not the current run's | parent *and* children are excluded |
+  | **nothing is mounted from it** | an in-flight run pins its own snapshots — this, not the age guard, is what protects a concurrent backup |
+  | it is **over an hour old** | covers the seconds-long window where a live run has snapshotted but not yet mounted |
+
+  Verified against the real pool: of **4,728** snapshots — including **2,341** periodic
+  ones — it selects exactly the orphans of the task being run, and nothing else.
+
 ## v0.6.0 — 2026-07-13
 ### Added
 

NOTICE

@@ -0,0 +1,4 @@
+truenas-truecloud-patch
+
+Parts of this project were written with AI assistance (Claude); all of it is
+reviewed and tested before release.

README.md

@@ -1,5 +1,7 @@
 # truenas-truecloud-patch
 
+[![CI](https://git.arch.fyi/flan/truenas-truecloud-patch/actions/workflows/ci.yml/badge.svg)](https://git.arch.fyi/flan/truenas-truecloud-patch/actions)
+
 Extends TrueNAS SCALE's **TrueCloud Backup** to:
 
 - back up to **Backblaze B2 and any S3-compatible provider**, not just Storj;
@@ -59,9 +61,11 @@ If something is wrong, the reason is in `apply.log` — start at
 | --- | --- | --- | --- |
 | 24.10.2.4 | ok | ok | — |
 | 25.04.2.6 | ok | ok | — |
-| 25.10.4 | ok | ok | nested + providers; 252-snapshot recursive backup of /mnt/Tap, 18m |
-| 26.0.0-BETA.3 _(unreleased)_ | ok | **BROKEN** | — |
-| master _(unreleased)_ | **BROKEN** | **BROKEN** | — |
+| 25.10.6 | ok | ok | — |
+| 24.10.2.5 _(unreleased)_ | ok | ok | — |
+| 25.10.7 _(unreleased)_ | ok | ok | — |
+| 26.0.0-BETA.3 _(unreleased)_ | ok | ok | — |
+| master _(27-dev)_ | **BROKEN** | **BROKEN** | — |
 
 | verdict | meaning |
 | --- | --- |
@@ -72,17 +76,44 @@ If something is wrong, the reason is in `apply.log` — start at
 "ok" means *the patch's assumptions hold*, checked automatically against iX's
 source. It does not mean a human ran a backup on it — that is the
 **Hardware-verified** column, which is filled in by hand and only by doing it.
+
+**`master` is not the next release.** iX branches each major off to its own
+`release/` line and master rolls straight on to the one after — so master is
+`27-dev` while 26 is still in beta. A **BROKEN** master means iX has changed
+something that will reach users *a major release from now*, not in the version you
+are about to install. Read the numbered rows for that.
+
+A row like `25.10.5 _(unreleased)_` is the next maintenance release: branched by iX,
+not tagged yet, and the very next thing a 25.10.4 box gets. It is checked precisely
+because it is the one unshipped ref that reaches real users without warning.
 <!-- END COMPAT MATRIX -->
 
 The table is **regenerated daily by CI** against iXsystems' actual middleware source
 — it is not a claim somebody typed once and forgot.
 
-**TrueNAS 26: nested snapshots are not supported yet, and upgrading will not break
-you.** 26 rewrites `cloud_backup` and deletes the ZFS methods this module calls. On
-26 `apply.sh` finds that the assumptions no longer hold and **does not apply the
-module**: TrueNAS is left stock, B2/S3 keeps working, nested datasets are simply not
-covered, and the reason is named in `apply.log`. A broken backup is worse than a
-missing feature. Details: [How it works](docs/how-it-works.md#truenas-26).
+It is also **static analysis**: it proves the patch's assumptions still hold, which is
+a weaker claim than "a backup ran and a restore came back". For what has actually been
+run — which tasks, on which hardware, and the md5 of the file that came back — see
+[docs/verification.md](docs/verification.md).
+
+**TrueNAS 26 is supported** as of v0.7.0, and was verified on a real
+**26.0.0-BETA.1** install: a 274-snapshot recursive backup of a 292-dataset pool, and a
+byte-identical restore of a four-level-deep child dataset. The *Hardware-verified*
+column tracks the newest beta iX has tagged (currently BETA.3), so it does not carry
+that mark — a build nobody has actually run a backup on does not get credit for one.
+
+26 rewrites `cloud_backup` from async to synchronous and deletes the private ZFS
+methods this module used to call, so getting there took real work: the patch now
+injects the wrapper flavour that matches the installed middleware, reads dataset and
+snapshot lists **from ZFS rather than middleware** (whose queries hide TrueNAS's own
+datasets — 84 of 270 on a real pool, including live app data), and owns the snapshot
+sweep even when it stages nothing (26 decides `recursive` by a rule this patch does not
+share, and would otherwise orphan one snapshot per zvol on every run).
+
+**And if a future TrueNAS breaks it, you get a missing feature, not a broken backup.**
+`apply.sh` re-checks the patch's assumptions at every boot and **refuses to apply a
+module whose assumptions no longer hold** — TrueNAS is left stock, and the reason is
+named in `apply.log`. Details: [How it works](docs/how-it-works.md#truenas-26).
 
 ---
 
@@ -207,5 +238,7 @@ and restores the original UI bundle from backup.
 - Filing a TrueNAS bug? **Remove the patch first** and reproduce on a stock system.
 - Provided as-is, no warranty. See LICENSE.
 
-Parts of this project were written with AI assistance (Claude); all of it is reviewed
-and tested before release. Bugs are mine.
+## Support
+
+If truenas-truecloud-patch is useful to you, consider supporting development via
+[GitHub Sponsors](https://github.com/sponsors/sudolulo) or [Ko-fi](https://ko-fi.com/sudolulo).

docs/how-it-works.md

@@ -187,23 +187,71 @@ backup-breaking**, and none of them is visible from the `cloud_backup` files:
 | `get_dataset_recursive()` **deleted** from `plugins/cloud/snapshot.py` | `NameError` — the injected block called it out of the host module's namespace |
 | `plugins/zfs_/dataset.py` and `zfs_/snapshot.py` **deleted** | `zfs.dataset.query`, `zfs.snapshot.query` and `zfs.snapshot.delete` all vanish. 26 uses `filesystem.statfs` and `zfs.resource.*` |
 
-The first two are fixed: the patch reads which flavour of `cloud_backup` your box
-declares and injects the wrapper that matches (one implementation of the real logic,
-two thin wrappers), and it carries its own copy of the deleted helper.
+All three are fixed as of **v0.7.0**, and 26 is supported.
 
-The third is **not** fixed, and is why 26 reports BROKEN. Porting it means rewriting
-the module's ZFS calls onto 26's new API, and no single API spans 24.10 through 26 —
-so it needs a real 26 box to verify against, not a plausible-looking diff. Shipping a
-port nobody has run is exactly the failure this project exists to avoid.
+The first two were straightforward: the patch reads which flavour of `cloud_backup`
+your box declares and injects the wrapper that matches (one implementation of the real
+logic, two thin wrappers), and it carries its own copy of the deleted helper.
 
-It is also the row that would have hurt most. `zfs.snapshot.delete` is what sweeps the
-recursive snapshot; without it, **every run would orphan one snapshot per descendant
-dataset — 250 on a real pool — forever.** The compatibility check caught it only
-because it now asserts the middleware *methods the patch calls*, not just the symbols
-it wraps.
+The third was not, and it is the one that would have hurt most — `zfs.snapshot.delete`
+is what sweeps the recursive snapshot, and without it **every run would orphan one
+snapshot per descendant dataset (250 on a real pool), forever, while reporting
+success.**
 
-`master` (development after 26) reports BROKEN too: iXsystems are still reshaping
+The obvious port is to the public `pool.dataset.query` / `pool.snapshot.query`. Those
+methods exist, are documented, and are covered by iX's deprecation policy — and they
+are **not like-for-like replacements**. They apply a *visibility policy*: they hide the
+datasets TrueNAS considers its own (`ix-apps/*`, `.system/*`, `.ix-virt/*`). On a real
+pool that is **84 of 270 datasets, including live application data.** Staging from that
+view would have omitted every one of them from the backup — and the planner would never
+have seen them, so they would not have appeared in its "skipped" list either. A green
+backup, quietly missing data. The snapshot query lies the same way, so the sweep would
+have orphaned one snapshot per hidden dataset.
+
+No source analysis could have caught that. The methods are all present and correctly
+shaped. Only running it could, which is why it took a real 26 box.
+
+So the module now follows one rule:
+
+> **Read the truth from ZFS. Make changes through middleware.**
+
+Enumeration is `zfs list` — no policy can filter it, and it behaves identically on every
+release, which also means one code path instead of a version conditional. Mutation stays
+a middleware call, so TrueNAS's own bookkeeping stays consistent; an exact-name delete
+works fine even on a dataset the query hides. It is only enumeration that lies.
+
+The snapshot *delete* still needs a namespace, and no single one spans every release —
+24.10 and 25.04 have `zfs.snapshot`, 26 has only `pool.snapshot`, 25.10 has both. So it
+is resolved at runtime, by asking whether the namespace can actually delete. `tools/compat.py`
+asks the identical question against iX's source, and a test binds the two lists together,
+so what CI verifies and what runs cannot drift apart.
+
+### The one 26 changed that nothing warned about
+
+Stock decides whether to take a **recursive** snapshot by its own rule, and on 26 that
+rule stopped being ours:
+
+| | decides `recursive` by |
+| --- | --- |
+| stock ≤ 25.10 | `get_dataset_recursive()` — the same function this patch vendors |
+| **stock 26** | `filesystem.statfs`: `recursive = (path == the dataset's mountpoint)` |
+| this patch | `get_dataset_recursive()` — is a mounted *filesystem* child under the path? |
+
+Up to 25.10 those were the *same question*, so a snapshot the patch declined to stage
+provably had no children and stock's non-recursive delete was correct. On 26 they
+disagree: a dataset whose only descendants are **zvols** or **legacy-mountpoint**
+datasets gets a recursive snapshot, while the patch sees nothing to stage. Stock then
+destroys the parent only — and with no staging tree there was no sidecar, and the
+garbage collector only ever ran from the staging path. Nothing on the box would ever
+have found the children.
+
+It was reproduced on a 26 VM (one orphan per zvol, every run, backup green) and closed:
+**ownership of the sweep is no longer conditional on staging.**
+
+`master` (development after 26) **does** report BROKEN: iXsystems are still reshaping
 these functions there, renaming `middleware` → `context` and `cloud_backup` → `entry`
 and adding a required `credentials` parameter. That is a moving target and is
 deliberately not chased; the check keeps reporting it until it settles into a beta,
-which is when it becomes worth fixing.
+which is when it becomes worth fixing. Until then, a box running master would simply
+not get the modules — `apply.sh` refuses to apply a module whose assumptions no longer
+hold, and says why in `apply.log`.

docs/nested-snapshots.md

@@ -95,6 +95,21 @@ re-scan each time.
 
 ### Snapshot lifecycle
 
+> **Two mechanisms clean up, and the second exists because the first can be destroyed.**
+>
+> 1. **The sidecar** records exactly which snapshots a run pinned, and is removed only
+>    on a confirmed-clean sweep. Precise, and it survives a middlewared restart.
+> 2. **The garbage collector** finds leftovers by *name*, so it still works when the
+>    sidecar is gone — and it can be: **the sidecar lives in `/run`, which is tmpfs.** A
+>    reboot mid-backup takes it, and with it the only record of a 250-snapshot tree.
+>
+> The collector runs at the start of every backup, after the sidecar reclaim. It will
+> only touch a snapshot named `<dataset>@<task>-<timestamp>` that is not the current
+> run's, has **nothing mounted from it** (which is what protects a concurrently-running
+> backup), and is **over an hour old**. Periodic `auto-*` snapshots, other tasks'
+> snapshots, and anything you made by hand are structurally out of reach.
+
+
 > **A snapshot may survive a run, and that is expected.** ZFS **automounts**
 > `<dataset>/.zfs/snapshot/<snap>` the moment it is read, and holds it for
 > `zfs_expire_snapshot` seconds (**300** by default) after the last access. So

docs/releasing.md

@@ -25,7 +25,7 @@ are worth calling out, because nothing else would catch what they catch:
   newest CHANGELOG entry. `VERSION=` had silently drifted to three different
   values across the scripts before anything checked.
 
-The project is hosted on **Gitea** (`git.onetick.ninja/flan/truenas-truecloud-patch`)
+The project is hosted on **Gitea** (`git.arch.fyi/flan/truenas-truecloud-patch`)
 and mirrored to GitHub. Both run the same workflows — Gitea reads
 `.github/workflows/` too — so a change is checked twice, on two independent runners.
 

docs/verification.md

@@ -0,0 +1,77 @@
+# What has actually been run
+
+The support matrix in the README is **static analysis**: it proves the patch's
+assumptions about middlewared still hold. That is a strictly weaker claim than "a
+backup ran and a restore came back". This file is the stronger claim, and it is
+maintained by hand, because the only way to fill it in is to do it.
+
+If you are deciding whether to trust this with your backups, read this file, not the
+matrix.
+
+---
+
+## v0.7.0 — TrueNAS 25.10.4 (production hardware)
+
+Six live TrueCloud tasks, all `snapshot = true`, backing up to Backblaze B2. Three were
+exercised end to end, chosen to cover the three shapes the code handles differently:
+
+| Task | Path | Shape | Result |
+| --- | --- | --- | --- |
+| 5 | `/mnt/Tap` | 191 nested datasets staged, 282-snapshot recursive tree | SUCCESS |
+| 7 | `/mnt/Tank/backups` | 215 filesystems **+ 2 zvols** | SUCCESS |
+| 9 | `/mnt/Tank/flan` | **no** nested filesystem children | SUCCESS |
+
+After every run: **0 orphaned snapshots, 0 leaked bind mounts, 0 stale sidecars.**
+
+**The restore.** `apps/vaultwarden/data/config.json` — a file inside a *child* dataset,
+which is exactly what stock TrueNAS cannot capture — was restored from B2 and compared
+against the live file:
+
+    live      f809df6ba231986b1ba824044228a03a   1808 bytes
+    restored  f809df6ba231986b1ba824044228a03a   1808 bytes
+    => byte-identical
+
+**The collector earned its keep on real data.** The pool was already carrying an orphan:
+`Tap/apps/prometheus@cloud_backup-5-20260713202355`, left behind by an earlier run when
+ZFS's automount held the snapshot busy past all four retries. The first v0.7.0 run found
+it by name, reclaimed it, and the pool's snapshot count went 2148 → 2147. That is the
+garbage collector doing the job it was written for, against a leak that was already
+there and that nothing else would ever have found.
+
+**Boot path.** `apply.sh` is registered as a PREINIT `initshutdownscript`; it was
+re-run against the live middleware and left exactly one `TRUECLOUD_PATCH` marker in
+each patched module (a second copy stacked into a live middlewared module would break
+the box at boot). It correctly detected the box as **async** (`cloud_backup is async
+(TrueNAS <= 25.10)`) and injected the matching wrappers.
+
+**Upgrade path.** `update.sh` was used to move the box from the release candidate to
+the stable tag, in detached HEAD at `v0.7.0`, which is how a user's box actually
+upgrades.
+
+## v0.7.0 — TrueNAS 26.0.0-BETA.1 (VM)
+
+A throwaway VM whose pool reproduces the production pool's *shape* — 292 datasets, 26
+`legacy` mountpoints, nesting five deep — because every bug found on the real box came
+from the shape of the pool, not the bytes in it. MinIO was not used; `rclone serve s3`
+(already on the box) provided the S3 target, so no real B2 credential ever entered the
+VM.
+
+* 274-snapshot recursive backup of the 292-dataset pool. 0 orphans, 0 leaked mounts.
+* Restored `ix-apps/app_mounts/vaultwarden/pgData` — **four levels deep, and a dataset
+  that middleware's own `pool.dataset.query` hides from itself** — byte-identical.
+* The zvol-orphan case was **reproduced with the fix disabled** (one orphan per zvol,
+  every run, backup green), then **closed with it enabled**. See the CHANGELOG entry
+  for why TrueNAS 26 decides `recursive` by a different rule than this patch decides
+  `nested`.
+
+## What is NOT covered
+
+* **24.10 and 25.04** are `ok` in the matrix — the assumptions hold, checked against
+  iX's source — but nobody has run a backup on them. The matrix says so.
+* **master** is BROKEN, and correctly reports so: iX renamed the leading parameters of
+  `get_restic_config` and `restic_backup`. It is not a shipped release; the daily
+  compatibility bot files it, and `apply.sh` would refuse to apply the modules on a box
+  running it.
+* A **reboot** of the production box has not been done on v0.7.0. `apply.sh` was
+  re-executed by hand against the live middleware, which exercises the same code path,
+  but the PREINIT ordering itself has only been proven on earlier versions.

install.sh

@@ -18,7 +18,7 @@
 
 set -euo pipefail
 
-VERSION="0.6.0"
+VERSION="0.7.0"
 
 # The directory containing install.sh is the permanent install location.
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"

patch/alert_source.py

@@ -47,8 +47,8 @@ _VERSION_RE = re.compile(r'^VERSION="([^"]+)"', re.M)
 #: owner/repo out of any of:
 #:   git@github.com:sudolulo/repo.git
 #:   https://github.com/sudolulo/repo.git
-#:   ssh://git@git.onetick.ninja:55214/flan/repo.git
-#:   https://git.onetick.ninja/flan/repo.git
+#:   ssh://git@git.arch.fyi:55214/flan/repo.git
+#:   https://git.arch.fyi/flan/repo.git
 #: The SSH port is deliberately not captured: it is not the web port.
 _REMOTE_RE = re.compile(
     r"^(?:\w+://)?(?:[^@/]+@)?([^:/]+)(?::\d+)?[:/]([^/]+)/([^/]+?)(?:\.git)?/?$"

patch/apply.sh

@@ -32,7 +32,7 @@
 # Derive PATCH_DIR from this script's location (parent of the patch/ directory).
 PATCH_DIR="$(cd "$(dirname "$0")/.." && pwd)"
 LOG="$PATCH_DIR/apply.log"
-VERSION="0.6.0"
+VERSION="0.7.0"
 
 # Rotate log at 512 KB to avoid unbounded growth on a system volume.
 # Keep two prior generations (.1 and .2) so the last three boots are always available.
@@ -535,8 +535,8 @@ if _tc_nested is not None:
         # snapshot=true, not just ours. Two consequences, and the second is worse:
         #
         #   * everything below is a NEW failure mode for tasks that worked before we
-        #     were installed. A `zfs.dataset.query` that errors would break a
-        #     CloudSync job we have no business touching.
+        #     were installed. A `zfs list` that errors would break a CloudSync job
+        #     we have no business touching.
         #   * if a CloudSync task ever were staged, nothing would ever tear it down:
         #     the teardown is wired into cloud_backup's restic_backup finally, and
         #     CRUD_BLOCK deliberately leaves CloudSync's nesting guard intact. The
@@ -556,18 +556,28 @@ if _tc_nested is not None:
             # our staging plan would not -- silently omitting it from the backup.
             # Read afterwards, an unsnapshotted dataset instead trips the isdir()
             # check in plan_staging and fails the run loudly. Loud beats silent.
-            datasets = middleware.call_sync(
-                "zfs.dataset.query", [["type", "=", "FILESYSTEM"]]
-            )
+            # query_filesystems() reads ZFS directly. It deliberately does NOT use
+            # pool.dataset.query: that applies a visibility policy and hides
+            # TrueNAS-internal datasets (ix-apps/*, .system/*, .ix-virt/*) -- 84 of
+            # 270 on a real pool, including live app data. Staging from the filtered
+            # view omits them silently, which is the one thing this must never do.
+            datasets = _tc_nested.query_filesystems(middleware)
             # OUR copy of get_dataset_recursive, not the host module's: TrueNAS 26
             # deleted that helper (create_snapshot uses filesystem.statfs now), so
             # calling it out of the module namespace is a NameError there.
             dataset, nested = _tc_nested.get_dataset_recursive(datasets, path)
 
             if not nested:
-                # No children: stock behaviour, untouched. Stock's `finally` owns
-                # the snapshot from here (its non-recursive delete is correct,
-                # because a non-nested snapshot has no children).
+                # Nothing to STAGE -- but we still own the SWEEP, and that is not a
+                # formality. Stock decides `recursive` by its own rule, and on 26 that
+                # rule is no longer ours: it snapshots recursively whenever the backup
+                # path IS the dataset's mountpoint (filesystem.statfs), while
+                # get_dataset_recursive() sees nothing to stage when the only
+                # descendants are ZVOLs or legacy/none-mountpoint datasets. Stock then
+                # deletes the PARENT ONLY. Without this, one snapshot per descendant is
+                # orphaned on every run, forever, with no sidecar and no GC to find it --
+                # and the backup still reports success.
+                _tc_nested.own_snapshot(middleware, name, snapshot, logger=_logger)
                 return snapshot, snap_path
 
             staging_root = _tc_nested.stage_nested(
@@ -581,7 +591,19 @@ if _tc_nested is not None:
             # stays None and its `finally` deletes NOTHING. Sweep the tree ourselves
             # or leak the parent plus one snapshot per descendant dataset (160+ here)
             # on every failed run.
-            _tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger)
+            #
+            # The sweep is itself wrapped: a cleanup that raises would REPLACE the
+            # original exception with its own, hiding why the backup actually failed.
+            # An error handler must not be able to lose the error.
+            try:
+                _tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger)
+            except Exception as _tc_sweep_err:
+                if _logger:
+                    _logger.error(
+                        "truecloud-patch: could not sweep %s after a staging failure "
+                        "(%r) -- it is orphaned and must be deleted by hand",
+                        snapshot, _tc_sweep_err,
+                    )
             raise
 
         return snapshot, staging_root

patch/create_task.py

@@ -52,7 +52,7 @@ import subprocess
 import sys
 import time
 
-__version__ = "0.6.0"
+__version__ = "0.7.0"
 
 _PATCH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 _STATUS_FILE = os.path.join(_PATCH_DIR, "hook_status.json")

recover.sh

@@ -17,7 +17,7 @@
 #   bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh
 #   systemctl restart middlewared
 
-VERSION="0.6.0"
+VERSION="0.7.0"
 
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 

tests/test_apply_blocks.py

@@ -140,19 +140,60 @@ class TestSnapshotLeak:
     every path that creates one must also sweep the whole tree.
     """
 
-    def test_staging_failure_deletes_the_snapshot_tree(self):
-        # On a staging failure, sync.py's `snapshot, local_path = await
-        # create_snapshot(...)` never completes, so its local `snapshot` stays
-        # None and its finally deletes nothing. We must sweep it ourselves.
-        block = extract_blocks()["SNAPSHOT_ASYNC"]
-        assert "except Exception:" in block
-        assert "delete_snapshot_tree" in block
-        assert "raise" in block
+    # The behaviour these once asserted as substrings -- the sweep, the re-raise, the
+    # teardown in the finally -- is now asserted STRUCTURALLY, against the parsed
+    # block: see TestTheStagingFailurePathReallyReRaises and
+    # TestTheSyncBlockAlwaysTearsDown. As substring checks they were satisfied by
+    # COMMENTS ("a cleanup that raises...", "cleanup_task gets logger=None"), so
+    # deleting the actual `raise` and the actual cleanup call both left the suite
+    # green -- reinstating a silently-empty backup and ~250 orphans per run.
 
-    def test_sync_block_cleans_up_on_every_path(self):
-        block = extract_blocks()["SYNC_ASYNC"]
-        assert "finally:" in block
-        assert "cleanup_task" in block
+    def test_the_snapshot_block_still_owns_the_snapshot_when_not_staging(self):
+        # The TrueNAS 26 zvol/legacy orphan: stock decides `recursive` by its own rule
+        # (path == mountpoint) and deletes only the parent, so we must record the
+        # snapshot even on the path where we stage nothing.
+        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
+            stage = functions(tree_of(name), "_tc_stage")[0]
+            assert calls_to(stage, "_tc_nested.own_snapshot"), (
+                f"{name} hands an unstaged snapshot back to stock, whose delete is "
+                f"non-recursive -- every zvol/legacy child is orphaned, every run"
+            )
+
+    def test_the_staging_plan_is_enumerated_from_ZFS(self):
+        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
+            stage = functions(tree_of(name), "_tc_stage")[0]
+            assert calls_to(stage, "_tc_nested.query_filesystems"), (
+                "the staging plan must come from query_filesystems() (which reads ZFS "
+                "unfiltered); middleware's query hides ix-apps/*, .system/*, .ix-virt/*"
+            )
+            assert not calls_to(stage, "middleware.call_sync"), (
+                "the block calls middleware directly again -- its dataset/snapshot "
+                "queries are FILTERED and silently omit 84 of 270 datasets"
+            )
+
+    def test_the_vendored_helper_is_used_not_the_host_module(self):
+        # TrueNAS 26 DELETED get_dataset_recursive from plugins/cloud/snapshot.py, so
+        # calling it out of the host module's namespace is a NameError there.
+        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
+            stage = functions(tree_of(name), "_tc_stage")[0]
+            assert calls_to(stage, "_tc_nested.get_dataset_recursive"), (
+                "must call OUR vendored copy: TrueNAS 26 deleted the host's"
+            )
+
+    def test_datasets_are_enumerated_AFTER_the_snapshot(self):
+        # A dataset created between the listing and the snapshot would be captured by
+        # the recursive snapshot but missing from the staging plan -- silently omitted.
+        # Read afterwards, it instead trips plan_staging's probe and fails loudly.
+        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
+            src = extract_blocks()[name]
+            code = "\n".join(
+                ln for ln in src.splitlines() if not ln.lstrip().startswith("#")
+            )
+            # _tc_stage receives `snapshot` as a parameter -- i.e. it is taken by the
+            # caller, before any of this runs. If the enumeration ever moves ahead of
+            # create_snapshot it can only do so by leaving _tc_stage.
+            assert "def _tc_stage(middleware, path, name, snapshot, snap_path)" in code
+            assert "query_filesystems" in code
 
 
 def test_crud_block_is_scoped_to_cloud_backup():
@@ -438,8 +479,275 @@ class TestOnlyOurOwnTasksAreTouched:
         # The point is to add NO new failure mode to a CloudSync task. If any
         # middleware call happened before the bail-out, we would already have broken
         # the thing we are trying not to touch.
+        #
+        # Checked against whichever interactions the block ACTUALLY contains, not a
+        # fixed list: the dataset query moved behind `_tc_nested.query_filesystems()`
+        # when it switched to the public pool.* API, and a hardcoded
+        # `middleware.call_sync(` simply stopped being found -- a test that silently
+        # stops testing is worse than no test.
         block = extract_blocks()[name]
         gate = block.index('if not name.startswith("cloud_backup"):')
-        for call in ("middleware.call_sync(", "_tc_nested.stage_nested(",
-                     "_tc_nested.delete_snapshot_tree("):
+
+        interactions = [
+            "middleware.call_sync(",
+            "_tc_nested.query_filesystems(",
+            "_tc_nested.stage_nested(",
+            "_tc_nested.delete_snapshot_tree(",
+        ]
+        present = [c for c in interactions if c in block]
+        assert present, "found no middleware interaction at all -- the test is vacuous"
+        for call in present:
             assert gate < block.index(call), f"{call} runs before the cloud_backup gate"
+
+
+# ── structural assertions ────────────────────────────────────────────────────
+#
+# `assert "raise" in block` was TRUE because a COMMENT in the block says "a cleanup
+# that raises would replace the original exception". `assert "cleanup_task" in block`
+# was TRUE because a comment says "cleanup_task gets logger=None". Deleting the actual
+# `raise`, and deleting the actual cleanup call from the `finally`, both left the suite
+# green -- while reinstating, respectively, a silently-empty backup and ~250 orphaned
+# snapshots per run.
+#
+# A test that a comment can satisfy is not a test. These parse the block and assert on
+# the CODE.
+
+def tree_of(name):
+    return ast.parse(textwrap.dedent(extract_blocks()[name]))
+
+
+def functions(tree, name):
+    return [
+        n for n in ast.walk(tree)
+        if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef) and n.name == name
+    ]
+
+
+def calls_to(node, dotted):
+    """Every Call in `node` whose callee renders as `dotted` (e.g. a.b.c)."""
+    out = []
+    for n in ast.walk(node):
+        if isinstance(n, ast.Call):
+            try:
+                if ast.unparse(n.func) == dotted:
+                    out.append(n)
+            except Exception:                                    # noqa: BLE001
+                pass
+    return out
+
+
+class TestTheStagingFailurePathReallyReRaises:
+    """If staging fails and we swallow it, restic backs up the UN-STAGED path.
+
+    That is the silently-empty backup this entire module exists to prevent: stock
+    points the tool at the parent's `.zfs/snapshot/`, where child datasets are
+    invisible. The exception MUST propagate.
+    """
+
+    @pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
+    def test_the_handler_sweeps_the_snapshot_and_re_raises(self, name):
+        stage = functions(tree_of(name), "_tc_stage")
+        assert stage, "_tc_stage is gone"
+
+        handlers = [
+            h for t in ast.walk(stage[0]) if isinstance(t, ast.Try)
+            for h in t.handlers
+        ]
+        assert handlers, "the staging failure handler is gone"
+
+        sweeps = any(calls_to(h, "_tc_nested.delete_snapshot_tree") for h in handlers)
+        assert sweeps, (
+            "a staging failure no longer sweeps the snapshot. sync.py's `snapshot` "
+            "local stays None, so ITS finally deletes nothing -- the whole tree leaks "
+            "on every failed run."
+        )
+
+        # A bare `raise` directly in the handler body -- not one nested inside the
+        # defensive try/except that wraps the sweep.
+        reraises = any(
+            any(isinstance(s, ast.Raise) and s.exc is None for s in h.body)
+            for h in handlers
+        )
+        assert reraises, (
+            "the staging failure is SWALLOWED. restic then runs against the un-staged "
+            "path and uploads a near-empty tree, reporting SUCCESS."
+        )
+
+
+class TestTheSyncBlockAlwaysTearsDown:
+    """The teardown is what unmounts the staging tree and sweeps the snapshot.
+
+    It must run on EVERY exit from restic_backup -- success, failure, or exception --
+    or the bind mounts pin the snapshot and the tree is orphaned.
+    """
+
+    @pytest.mark.parametrize("name", ["SYNC_ASYNC", "SYNC_SYNC"])
+    def test_cleanup_runs_in_a_finally(self, name):
+        fns = functions(tree_of(name), "restic_backup")
+        assert fns, "the restic_backup wrapper is gone"
+
+        tries = [t for t in ast.walk(fns[0]) if isinstance(t, ast.Try) and t.finalbody]
+        assert tries, "restic_backup no longer has a try/finally"
+
+        cleans = any(
+            "cleanup_task" in ast.unparse(stmt)
+            for t in tries for stmt in t.finalbody
+        )
+        assert cleans, (
+            "cleanup_task is not called in the finally. The staging tree is never torn "
+            "down, its bind mounts pin the snapshot, and ~250 snapshots leak per run."
+        )
+
+
+class TestTheBlockingWorkNeverRunsOnTheEventLoop:
+    """`zfs list` and `call_sync` are BLOCKING. On <=25.10 these blocks are async.
+
+    Running them directly on middlewared's event loop stalls the whole daemon.
+    """
+
+    @pytest.mark.parametrize("name,fn", [
+        ("SNAPSHOT_ASYNC", "create_snapshot"),
+        ("SYNC_ASYNC", "restic_backup"),
+    ])
+    def test_the_async_flavour_hops_to_a_thread(self, name, fn):
+        fns = functions(tree_of(name), fn)
+        assert fns and isinstance(fns[0], ast.AsyncFunctionDef)
+        assert calls_to(fns[0], "middleware.run_in_thread"), (
+            f"{name}.{fn} does the blocking work on the asyncio event loop"
+        )
+
+    @pytest.mark.parametrize("name,fn", [
+        ("SNAPSHOT_SYNC", "create_snapshot"),
+        ("SYNC_SYNC", "restic_backup"),
+    ])
+    def test_the_sync_flavour_does_not(self, name, fn):
+        # On 26 stock already runs this in the thread pool; hopping again would be
+        # wrong (and there is no event loop to protect).
+        fns = functions(tree_of(name), fn)
+        assert fns and isinstance(fns[0], ast.FunctionDef)
+        assert not calls_to(fns[0], "middleware.run_in_thread")
+
+
+def test_the_flavour_mapping_is_not_inverted():
+    # `_snapshot_block = SNAPSHOT_ASYNC if _flavour else SNAPSHOT_SYNC` -- inverting it
+    # injects an async wrapper on 26 (a coroutine gets unpacked as a tuple) or a sync
+    # one on 25.10 (the event loop blocks). Every nested backup breaks, both ways.
+    with open(APPLY_SH, encoding="utf-8") as fh:
+        code = " ".join(
+            ln for ln in fh.read().splitlines() if not ln.lstrip().startswith("#")
+        )
+    code = re.sub(r"\s+", " ", code)          # the assignments are space-aligned
+    for block in ("SNAPSHOT", "CRUD", "SYNC"):
+        assert f"{block}_ASYNC if _flavour else {block}_SYNC" in code, (
+            f"the {block} flavour mapping is missing or inverted: _flavour is True for "
+            f"an ASYNC middleware, so it must select {block}_ASYNC"
+        )
+
+
+# ── the compat preflight ─────────────────────────────────────────────────────
+#
+# This is the guard that stands between a broken middleware and a live NAS: at every
+# boot, apply.sh checks the patch's assumptions against the middlewared actually
+# installed, and REFUSES to apply a module whose assumptions no longer hold.
+#
+# It had no test. An audit turned it into a no-op eight different ways -- `verdict()`
+# always returning 'ok', the broken branch never firing, the kill switch never honoured
+# -- and the suite stayed green every time. The most consequential safety net in the
+# project was unguarded.
+
+def preflight_heredoc():
+    """The preflight's Python, lifted out of apply.sh and made runnable.
+
+    Extracted, not reimplemented: a reimplementation would happily pass while the
+    SHIPPED preflight stayed broken, which is exactly the failure being guarded.
+    """
+    with open(APPLY_SH, encoding="utf-8") as fh:
+        sh = fh.read()
+    # Line-based: the compat heredoc opens with `<<'PYEOF'` on the _tc_compat line and
+    # closes at the next bare PYEOF. (A regex that matched `<< 'PYEOF'` silently found
+    # the OTHER heredoc and ran a different script entirely.)
+    lines = sh.splitlines()
+    start = next(
+        i for i, ln in enumerate(lines)
+        if ln.startswith("_tc_compat=$(") and "<<'PYEOF'" in ln
+    )
+    end = next(i for i in range(start + 1, len(lines)) if lines[i].strip() == "PYEOF")
+    m = "\n".join(lines[start + 1:end])
+    assert m, "could not find the compat preflight heredoc in apply.sh"
+    return m
+
+
+def run_preflight(result, tmp_path):
+    """Run the SHIPPED preflight against a fake compat.check_tree result.
+
+    The heredoc does `import sys`, so a fake `sys` in the namespace is immediately
+    rebound to the real module -- drive the real one instead.
+    """
+    import contextlib
+    import io
+    import sys
+    import types
+
+    src = preflight_heredoc()
+    fake = types.ModuleType("compat")
+    fake.check_tree = lambda _mw: result
+
+    saved_mod = sys.modules.get("compat")
+    saved_argv = sys.argv
+    sys.modules["compat"] = fake
+    sys.argv = ["x", "/patch", "/mw", str(tmp_path / "compat.json")]
+
+    buf = io.StringIO()
+    try:
+        with contextlib.redirect_stdout(buf):
+            exec(compile(src, "apply.sh:preflight", "exec"), {"__name__": "__main__"})  # noqa: S102
+    except SystemExit:
+        pass
+    finally:
+        sys.argv = saved_argv
+        if saved_mod is not None:
+            sys.modules["compat"] = saved_mod
+        else:
+            sys.modules.pop("compat", None)
+    return buf.getvalue().splitlines()
+
+
+def _mod(ok=True, native=False, unknown=False, problems=()):
+    return {"ok": ok, "native": native, "unknown": unknown, "problems": list(problems)}
+
+
+class TestTheBootPreflightRefusesABrokenMiddleware:
+    def test_a_healthy_tree_is_ok(self, tmp_path):
+        out = run_preflight({"providers": _mod(), "nested": _mod()}, tmp_path)
+        assert out[:2] == ["ok", "ok"]
+
+    def test_a_broken_module_is_reported_broken(self, tmp_path):
+        out = run_preflight({
+            "providers": _mod(),
+            "nested": _mod(ok=False, problems=[
+                {"id": "x", "detail": "gone", "why": "orphans every run"},
+            ]),
+        }, tmp_path)
+        assert "broken" in out, (
+            "the preflight did not report a module whose assumptions FAILED. It would "
+            "be injected into a middleware it does not fit -- broken backups, "
+            "discovered at restore time."
+        )
+
+    def test_a_module_that_went_NATIVE_is_also_not_applied(self, tmp_path):
+        # 'native' answers "do we still need it?", 'ok' answers "is it safe to inject?".
+        # Applying a module TrueNAS now implements itself is not safe either.
+        out = run_preflight({
+            "providers": _mod(),
+            "nested": _mod(ok=False, native=True),
+        }, tmp_path)
+        assert "broken" in out
+
+    def test_an_UNKNOWN_verdict_is_not_reported_as_broken(self, tmp_path):
+        # A network error or an unreadable file is not iX deleting our symbols. Calling
+        # it broken would switch a working module off on a healthy box.
+        out = run_preflight({
+            "providers": _mod(unknown=True),
+            "nested": _mod(unknown=True),
+        }, tmp_path)
+        assert "broken" not in out

tests/test_compat.py

@@ -15,6 +15,7 @@ dangerous thing this file can say -- it means "TrueNAS does this now, retire the
 module" -- and it rests on nothing more than a substring match.
 """
 
+import json
 import os
 import sys
 
@@ -23,6 +24,7 @@ import pytest
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
 
 import compat  # noqa: E402
+import compat_publish  # noqa: E402
 from compat import (  # noqa: E402
     NESTED,
     PROVIDERS,
@@ -51,24 +53,44 @@ GOOD = {
         "async def restic_backup(middleware, job, cloud_backup, dry_run=False, "
         "rate_limit=None):\n    pass\n"
     ),
-    # The middlewared METHODS the injected code calls. TrueNAS 26 deleted both of
-    # these files, taking zfs.dataset.query / zfs.snapshot.query / zfs.snapshot.delete
-    # with them -- see TestMiddlewareMethodsWeCall.
-    "plugins/zfs_/dataset.py": (
-        "class ZFSDataset(CRUDService):\n"
+    # The middlewared METHODS the injected code calls. These now go through the
+    # PUBLIC pool.* API: TrueNAS 26 deleted plugins/zfs_/ outright, taking the whole
+    # private zfs.* service with it -- see TestMiddlewareMethodsWeCall.
+    #
+    # This default tree is a MODERN box (25.10/26): it has pool.snapshot and no
+    # zfs.snapshot. The older shape is built explicitly where it is tested.
+    # Not a plugin: a method on the middleware OBJECT. `snapshot_service()` resolves
+    # the snapshot namespace through it, so if it vanishes the module cannot sweep the
+    # snapshot it just took.
+    "utils/plugins.py": (
+        "class LoadPluginsMixin:\n"
+        "    def get_service(self, name):\n        pass\n"
+    ),
+    "plugins/pool_/dataset.py": (
+        "class PoolDatasetService(CRUDService):\n"
         "    class Config:\n"
-        "        namespace = 'zfs.dataset'\n"
+        "        namespace = 'pool.dataset'\n"
         "    def query(self, filters, options):\n        pass\n"
     ),
-    "plugins/zfs_/snapshot.py": (
-        "class ZFSSnapshot(CRUDService):\n"
+    "plugins/pool_/snapshot.py": (
+        "class PoolSnapshotService(CRUDService):\n"
         "    class Config:\n"
-        "        namespace = 'zfs.snapshot'\n"
+        "        namespace = 'pool.snapshot'\n"
         "    def query(self, filters, options):\n        pass\n"
         "    def delete(self, id_, options={}):\n        pass\n"
     ),
 }
 
+#: A 24.10/25.04 box: `pool.snapshot` does not exist yet and the snapshot CRUD
+#: service still answers to the (then-public) `zfs.snapshot`.
+ZFS_ERA_SNAPSHOT = (
+    "class ZFSSnapshot(CRUDService):\n"
+    "    class Config:\n"
+    "        namespace = 'zfs.snapshot'\n"
+    "    def query(self, filters, options):\n        pass\n"
+    "    def do_delete(self, id_, options={}):\n        pass\n"
+)
+
 
 def loader(files):
     def load(path):
@@ -284,9 +306,23 @@ class TestAsyncFlavour:
         broken["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429")
         assert compat.async_flavour(loader(broken)) is None
 
-    def test_the_real_truenas_versions(self):
-        # Pinning the actual fact this whole port exists for.
-        assert compat.async_flavour(loader(GOOD)) is True
+    def test_it_reads_STOCK_source_not_our_own_injected_block(self):
+        # This was a byte-identical copy of test_async_middleware_is_detected under a
+        # name that promised more. The fact worth pinning: apply.sh re-runs on an
+        # ALREADY-PATCHED overlay, so the probe must cut our block off first -- our own
+        # SNAPSHOT_SYNC wrapper is a plain `def create_snapshot`, and reading it would
+        # report a 25.10 box as synchronous and inject the wrong flavour.
+        patched = dict(GOOD)
+        patched["plugins/cloud/snapshot.py"] = (
+            GOOD["plugins/cloud/snapshot.py"]
+            + "\n# TRUECLOUD_PATCH\n"
+            + 'def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+        )
+        assert compat.async_flavour(loader(patched)) is True, (
+            "the flavour probe read our own injected block and concluded the box is "
+            "synchronous -- it would then inject a sync wrapper into an async "
+            "middleware, and every nested backup would break"
+        )
 
 
 class TestMiddlewareMethodsWeCall:
@@ -303,52 +339,86 @@ class TestMiddlewareMethodsWeCall:
     per descendant dataset (250 on a real pool) on every run, forever.
     """
 
-    ZFS_SNAPSHOT = (
-        "class ZFSSnapshot(CRUDService):\n"
-        "    class Config:\n"
-        "        namespace = 'zfs.snapshot'\n"
-        "    def query(self, filters, options):\n        pass\n"
-        "    def delete(self, id_, options={}):\n        pass\n"
-    )
-    ZFS_DATASET = (
-        "class ZFSDataset(CRUDService):\n"
-        "    class Config:\n"
-        "        namespace = 'zfs.dataset'\n"
-        "    def query(self, filters, options):\n        pass\n"
-    )
+    POOL_SNAPSHOT = GOOD["plugins/pool_/snapshot.py"]
 
     def _tree(self, **over):
         files = dict(GOOD)
-        files["plugins/zfs_/snapshot.py"] = self.ZFS_SNAPSHOT
-        files["plugins/zfs_/dataset.py"] = self.ZFS_DATASET
         files.update(over)
         return files
 
+    #: A 26 box: pool.snapshot only.
+    def _modern(self, **over):
+        return self._tree(**over)
+
+    #: A 24.10/25.04 box: zfs.snapshot only -- plugins/pool_/snapshot.py does not
+    #: exist yet.
+    def _zfs_era(self, **over):
+        return self._tree(**{
+            "plugins/pool_/snapshot.py": None,
+            "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
+            **over,
+        })
+
     def test_present_methods_are_ok(self):
-        r = check_files(self._tree())
+        r = check_files(self._modern())
         assert r[NESTED]["ok"], r[NESTED]["problems"]
 
-    def test_a_deleted_plugin_file_is_broken(self):
-        # Literally TrueNAS 26: plugins/zfs_/snapshot.py does not exist.
-        r = check_files(self._tree(**{"plugins/zfs_/snapshot.py": None}))
+    def test_the_OLD_zfs_era_snapshot_service_also_satisfies_the_call(self):
+        # 24.10 and 25.04 have no `pool.snapshot` at all -- the CRUD service is the
+        # then-public `zfs.snapshot`. Pinning only the modern spelling marked both of
+        # those releases BROKEN and would have switched nested snapshots OFF on boxes
+        # where they work perfectly. The runtime picks the same way; see
+        # pick_snapshot_service().
+        r = check_files(self._zfs_era())
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+    def test_it_is_broken_only_when_NEITHER_namespace_exists(self):
+        # The real failure: middleware drops the last spelling we know how to call.
+        r = check_files(self._tree(**{
+            "plugins/pool_/snapshot.py": None,
+            "plugins/zfs_/snapshot.py": None,
+        }))
         assert is_broken(r[NESTED])
         details = " ".join(p["detail"] for p in r[NESTED]["problems"])
-        assert "zfs.snapshot.delete" in details
+        assert "pool.snapshot.delete" in details
+        assert "zfs.snapshot.delete" in details, (
+            "the report must say BOTH spellings were tried, or whoever reads it will "
+            "think we simply never looked for the one their box has"
+        )
+
+    def test_we_do_NOT_depend_on_a_middleware_dataset_query_at_all(self):
+        # iX could delete plugins/pool_/dataset.py tomorrow and the patch would not
+        # care, because the staging plan is enumerated from ZFS, not from middleware.
+        #
+        # That is deliberate, and it was expensive to learn. `pool.dataset.query`
+        # exists and is correctly shaped -- and it LIES: it applies a visibility
+        # policy that hides ix-apps/*, .system/* and .ix-virt/* (84 of 270 datasets
+        # on the real pool, including live app data). No source check could ever
+        # have caught that; only running it could. So there is no assumption here
+        # left to break.
+        r = check_files(self._modern(**{"plugins/pool_/dataset.py": None}))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+        ids = {c.id for c in compat.MIDDLEWARE_CALLS}
+        assert not any("dataset" in i or "query" in i for i in ids), (
+            "a dataset/snapshot QUERY assumption crept back into the manifest -- "
+            "middleware's queries are filtered; enumerate from ZFS"
+        )
 
     def test_a_renamed_namespace_is_broken(self):
         r = check_files(self._tree(**{
-            "plugins/zfs_/snapshot.py": self.ZFS_SNAPSHOT.replace(
-                "'zfs.snapshot'", "'zfs.resource.snapshot'"),
+            "plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace(
+                "'pool.snapshot'", "'zfs.resource.snapshot'"),
+            "plugins/zfs_/snapshot.py": None,
         }))
         assert is_broken(r[NESTED])
 
     def test_the_CRUDService_do_prefix_is_accepted(self):
-        # 24.10 and 25.04 declare `do_delete`; 25.10 renamed it to `delete`. BOTH
-        # answer to zfs.snapshot.delete. Accepting only the literal name reported the
-        # two older releases as broken -- a false BROKEN that would have switched off
-        # nested snapshots on boxes where they work perfectly.
-        r = check_files(self._tree(**{
-            "plugins/zfs_/snapshot.py": self.ZFS_SNAPSHOT.replace(
+        # A CRUDService exposes `delete` from a method NAMED `do_delete`. Both
+        # spellings are live across the matrix. Accepting only the literal name
+        # reported working releases as broken.
+        r = check_files(self._modern(**{
+            "plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace(
                 "def delete(", "def do_delete("),
         }))
         assert r[NESTED]["ok"], r[NESTED]["problems"]
@@ -356,6 +426,383 @@ class TestMiddlewareMethodsWeCall:
     def test_the_snapshot_delete_reason_names_the_orphan_risk(self):
         # If this ever regresses, whoever reads the bug report must understand that
         # it is not a cosmetic failure.
-        r = check_files(self._tree(**{"plugins/zfs_/snapshot.py": None}))
+        r = check_files(self._tree(**{
+            "plugins/pool_/snapshot.py": None,
+            "plugins/zfs_/snapshot.py": None,
+        }))
         whys = " ".join(p["why"] for p in r[NESTED]["problems"])
         assert "orphan" in whys
+
+
+class TestTheMethodCheckIsNotJustANamespaceCheck:
+    """compat must verify the METHOD, not merely that the namespace still exists.
+
+    Deleting the method check entirely used to leave all 304 tests green -- so the
+    "namespace AND method" claim was unenforced and silently revertible. It is the
+    half of the predicate that catches iX gutting a method while keeping its service,
+    which they have already done to `pool.snapshot.do_update` on master.
+    """
+
+    def test_a_namespace_that_no_longer_defines_delete_is_broken(self):
+        gutted = (
+            "class PoolSnapshotService(CRUDService):\n"
+            "    class Config:\n"
+            "        namespace = 'pool.snapshot'\n"
+            "    def query(self, filters, options):\n        pass\n"
+            # do_delete is GONE -- the service is still registered and still a
+            # CRUDService, so it still INHERITS a callable `delete`.
+        )
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": gutted,
+            "plugins/zfs_/snapshot.py": None,      # no fallback either
+        }))
+        assert is_broken(r[NESTED]), (
+            "a namespace with no delete must be BROKEN. Checking only that the "
+            "namespace exists would apply the patch to a box that cannot sweep its "
+            "own snapshots."
+        )
+
+    def test_the_alternative_still_saves_it_when_only_the_primary_is_gutted(self):
+        gutted = (
+            "class PoolSnapshotService(CRUDService):\n"
+            "    class Config:\n"
+            "        namespace = 'pool.snapshot'\n"
+            "    def query(self, filters, options):\n        pass\n"
+        )
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": gutted,
+            "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
+        }))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+
+class TestUnreadableIsNeverOkAndNeverBroken:
+    """A rate limit is not a regression, and it is not a clean bill of health either.
+
+    compat runs ~30 unauthenticated GitHub requests per matrix; 429 is a real outcome.
+    It also runs at BOOT against the installed tree, where a read can fail with EACCES.
+
+      * treating unreadable as BROKEN repaints the README, files a bug report, and
+        makes apply.sh refuse the module on a box where it works.
+      * treating it as OK injects a module whose delete may be gone.
+
+    Both mutations used to pass the whole suite.
+    """
+
+    def test_both_spellings_unreadable_is_unknown_not_broken(self):
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": Unreadable("HTTP 429"),
+            "plugins/zfs_/snapshot.py": Unreadable("HTTP 429"),
+        }))
+        assert not is_broken(r[NESTED]), "a 429 is not iX deleting the snapshot service"
+        assert r[NESTED]["unknown"]
+
+    def test_an_unreadable_primary_with_a_healthy_alternative_is_ok(self):
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": Unreadable("HTTP 429"),
+            "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
+        }))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+        assert not r[NESTED]["unknown"], (
+            "one spelling answered the question; the other's 429 is irrelevant"
+        )
+
+    def test_a_missing_primary_with_an_unreadable_alternative_is_unknown(self):
+        # We cannot tell whether the box is broken. Saying either would be a guess.
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": None,
+            "plugins/zfs_/snapshot.py": Unreadable("HTTP 429"),
+        }))
+        assert not is_broken(r[NESTED])
+        assert r[NESTED]["unknown"]
+
+
+class TestGetServiceIsChecked:
+    """The runtime resolves the snapshot namespace through `middleware.get_service`.
+
+    It is not a plugin method, so the manifest had no way to express it and never
+    checked it. If it vanishes, `_can_delete` reports BOTH namespaces unusable and
+    every nested backup fails -- on a box the preflight had declared healthy.
+    """
+
+    def test_a_middleware_without_get_service_is_broken(self):
+        r = check_files(with_(**{"utils/plugins.py": None}))
+        assert is_broken(r[NESTED])
+        details = " ".join(p["detail"] for p in r[NESTED]["problems"])
+        assert "get_service" in details
+
+
+class TestATransientNetworkBlipDoesNotWakeAnybody:
+    """The fingerprint must digest what iX BROKE, not what GitHub failed to serve.
+
+    `unknown` problems (a 429 on one of ~30 unauthenticated fetches, an EACCES at boot)
+    used to be folded into an already-broken module's problem list, so one blip flipped
+    the fingerprint, `compat_publish` rewrote the issue body, and the next clean run
+    rewrote it back. Daily churn is what teaches people to ignore the bot -- which is
+    the whole thing this fingerprint exists to prevent.
+    """
+
+    def _rows(self, files):
+        return [{"ref": "master", "modules": check_files(files)}]
+
+    def test_an_unreadable_file_does_not_change_the_fingerprint_of_a_broken_ref(self):
+        # The blip must land in the SAME module that is broken. Put it in `providers`
+        # (which is healthy) and `fingerprint()` skips the whole module via
+        # `is_broken(m)` -- so the `state` filter under test never runs and the test
+        # passes no matter what the code does. `nested` is the broken one here, so the
+        # unreadable file goes in `nested` too.
+        broken = with_(**{
+            "plugins/cloud/snapshot.py":
+                "async def create_snapshot(name, path, middleware):\n    return 1, 2\n",
+        })
+        clean = compat.fingerprint(self._rows(broken))
+
+        blipped = dict(broken)
+        blipped["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429")   # nested
+        assert compat.fingerprint(self._rows(blipped)) == clean, (
+            "a rate-limited fetch changed the fingerprint, so the bot rewrites the "
+            "issue body and then rewrites it back tomorrow"
+        )
+
+    def test_a_REAL_new_finding_still_changes_it(self):
+        # ...and the anti-noise measure must not have made it deaf.
+        broken = with_(**{
+            "plugins/cloud/snapshot.py":
+                "async def create_snapshot(name, path, middleware):\n    return 1, 2\n",
+        })
+        worse = dict(broken)
+        worse["plugins/cloud_backup/restic.py"] = (
+            "class ResticConfig:\n    cmd: list\n\n"
+            "def get_restic_config(entry, credentials):\n    pass\n"
+        )
+        assert compat.fingerprint(self._rows(worse)) != compat.fingerprint(self._rows(broken))
+
+
+class TestTheBotFindsItsOwnIssueOnBOTHForges:
+    """`find_issue` decides "have I already filed this?" -- and it ran on two forges.
+
+    It used to skip pull requests with `"pull_request" not in i`. GitHub omits that key
+    on a plain issue; **Gitea sends it as `null`**. So on Gitea every issue looked like
+    a PR, the match list was always empty, and the bot took the "nothing filed yet"
+    branch on EVERY run: nine duplicate copies of the same report on the canonical
+    forge, four of them filed after the commit that was supposed to stop exactly this.
+
+    It is the same failure the spam fix was written to prevent, moved from comments to
+    issues -- and it survived because `find_issue` was the one function here with no
+    test. So the payload shapes are pinned, per forge, by hand.
+    """
+
+    TITLE = compat_publish.TITLE
+
+    def _find(self, monkeypatch, payload):
+        monkeypatch.setattr(compat_publish, "_call", lambda *a, **k: payload)
+        return compat_publish.find_issue("https://forge/api", "tok", self.TITLE)
+
+    def test_gitea_sends_pull_request_as_null_and_the_issue_is_still_found(self, monkeypatch):
+        found = self._find(monkeypatch, [
+            {"number": 7, "title": self.TITLE, "state": "open", "pull_request": None},
+            {"number": 1, "title": self.TITLE, "state": "open", "pull_request": None},
+        ])
+        assert found is not None, (
+            "find_issue missed a Gitea issue, so the bot files a NEW duplicate report "
+            "every run -- which is how nine of them piled up"
+        )
+        assert found["number"] == 1, "lowest-numbered wins"
+
+    def test_github_omits_the_key_entirely_and_the_issue_is_still_found(self, monkeypatch):
+        found = self._find(monkeypatch, [
+            {"number": 2, "title": self.TITLE, "state": "open"},
+        ])
+        assert found is not None and found["number"] == 2
+
+    def test_a_real_PR_with_the_same_title_is_still_skipped_on_both(self, monkeypatch):
+        # The reason the filter exists at all: both forges list PRs on /issues, and
+        # commenting on a PR instead of the bug report would be worse than useless.
+        assert self._find(monkeypatch, [
+            {"number": 3, "title": self.TITLE, "state": "open",           # Gitea PR
+             "pull_request": {"merged": False}},
+            {"number": 4, "title": self.TITLE, "state": "open",           # GitHub PR
+             "pull_request": {"url": "https://api.github.com/..."}},
+        ]) is None
+
+    def test_an_unrelated_issue_is_not_mistaken_for_the_report(self, monkeypatch):
+        assert self._find(monkeypatch, [
+            {"number": 1, "title": "TypeError when create B2 backup on Electric Eel",
+             "state": "closed", "pull_request": None},
+        ]) is None
+
+
+class TestTheNextMaintenanceReleaseIsChecked:
+    """`release/25.10.5` fell through every sieve, and it is the one that reaches users.
+
+    Shipped versions come from `TS-*` TAGS; unreleased ones come from `release/*`
+    BRANCHES that carry `-BETA`/`-RC`. A branched-but-untagged MAINTENANCE release is
+    neither: no tag, and its line (25.10) has already shipped, so the "prereleases of
+    a shipped line are history" filter threw it out. It was invisible.
+
+    That is backwards. `release/24.10-RC.2` is history -- nobody can install it. But
+    `release/25.10.5` is the FUTURE of a shipped line: it is what a 25.10.4 box gets
+    on its next update. A break there ships to real users before the daily check has
+    ever looked at it.
+    """
+
+    TAGS = ["TS-24.10.2.4", "TS-25.04.2.6", "TS-25.10.4"]
+    HEADS = [
+        "release/25.10.4.1",
+        "release/25.10.5",          # branched, untagged -- the next maintenance release
+        "release/24.10-RC.2",       # history: its line shipped long ago
+        "release/25.20.2.2",        # iX's typo branch: 25.20 is not a TrueNAS version
+        "release/26.0.0-BETA.3",
+        "master",
+    ]
+
+    def _refs(self, monkeypatch):
+        monkeypatch.setattr(
+            compat, "_ls_remote",
+            lambda remote, what: self.TAGS if what == "--tags" else self.HEADS)
+        return compat.discover_refs("origin")
+
+    def test_the_next_maintenance_release_is_checked(self, monkeypatch):
+        assert "release/25.10.5" in self._refs(monkeypatch), (
+            "the next thing a 25.10.4 box updates to is not checked, so a break in it "
+            "reaches users before the bot ever sees it"
+        )
+
+    def test_a_superseded_maintenance_branch_is_not(self, monkeypatch):
+        # 25.10.4.1 sorts OLDER than the newest tag TS-25.10.4? No -- it is NEWER, and
+        # both are on the 25.10 line, so only the newest branch on the line is taken.
+        refs = self._refs(monkeypatch)
+        assert "release/25.10.4.1" not in refs, "only the newest branch per line"
+
+    def test_the_typo_branch_stays_out(self, monkeypatch):
+        # 25.20 has no TS tag, so it is not a release line at all. A typo branch in the
+        # matrix reads as a real supported release we are silently broken on.
+        assert "release/25.20.2.2" not in self._refs(monkeypatch)
+
+    def test_a_prerelease_of_an_already_shipped_line_stays_out(self, monkeypatch):
+        assert "release/24.10-RC.2" not in self._refs(monkeypatch)
+
+    def test_an_untagged_branch_counts_as_UNRELEASED(self, monkeypatch):
+        # The exit code keys off this. Calling 25.10.5 "shipped" would fail the build
+        # as a live outage on a version nobody is running yet.
+        assert compat.is_unreleased("release/25.10.5")
+        assert compat.is_unreleased("master")
+        assert not compat.is_unreleased("TS-25.10.4")
+
+
+class TestMasterIsNotTheNextRelease:
+    """A red `master` row used to read as "the version you are about to install".
+
+    On 2026-07-14 master was 27-dev -- every recent commit targeted 27.0.0-BETA.1 --
+    while 26 was still in beta on its own branches. So `master BROKEN` meant "iX will
+    break us a major release from now", but the matrix said "master _(unreleased)_",
+    which any reader takes as the next thing out the door. For a table whose whole job
+    is helping somebody decide whether to trust this with their backups, that is a
+    false alarm in the worst possible place.
+    """
+
+    def _rows(self, refs):
+        return [{"ref": r, "unreleased": compat.is_unreleased(r), "modules": {}}
+                for r in refs]
+
+    def test_master_is_labelled_with_the_major_AFTER_the_newest_known_one(self):
+        rows = self._rows(["TS-25.10.4", "release/26.0.0-BETA.3", "master"])
+        assert compat.dev_label(rows) == "27-dev"
+
+    def test_it_rolls_over_on_its_own_when_the_next_beta_branches(self):
+        # Derived, not hardcoded: when release/27.0.0-BETA.1 appears, master is 28-dev.
+        rows = self._rows(["TS-26.0.0", "release/27.0.0-BETA.1", "master"])
+        assert compat.dev_label(rows) == "28-dev"
+
+    def test_the_rendered_matrix_says_dev_not_unreleased(self):
+        healthy = check_files(with_())
+        rows = [
+            {"ref": r, "unreleased": compat.is_unreleased(r), "modules": healthy}
+            for r in ("TS-25.10.4", "release/26.0.0-BETA.3", "master")
+        ]
+        md = compat.render_markdown(rows)
+        assert "master _(27-dev)_" in md
+        assert "master _(unreleased)_" not in md
+        # ...and the ordinary rows are untouched.
+        assert "| 25.10.4 |" in md
+        assert "| 26.0.0-BETA.3 _(unreleased)_ |" in md
+
+
+class TestTheBodyIsTruthAndCommentsAreTheChangelog:
+    """An unchanged fingerprint used to freeze the BODY, not just silence the comments.
+
+    Two different questions were sharing one answer. "Have the findings changed?" gates
+    COMMENTS -- they notify, and a daily "still broken, same as yesterday" is what
+    teaches people to ignore the one that finally matters. But "is the body still
+    true?" gates the BODY, and editing a body notifies nobody, so keeping it honest
+    costs nothing.
+
+    Conflated, an unchanged fingerprint meant the report could never be corrected --
+    and the fingerprint deliberately ignores everything that moves on its own, which
+    includes how a row is LABELLED. Relabelling master `27-dev` would have reached the
+    README and never the issue anybody opens.
+    """
+
+    ROWS = [{"ref": "master", "unreleased": True,
+             "modules": check_files(with_(**{
+                 "plugins/cloud_backup/restic.py":
+                     "class ResticConfig:\n    cmd: list\n\n"
+                     "def get_restic_config(entry, credentials):\n    pass\n",
+             }))}]
+
+    def _run(self, monkeypatch, tmp_path, existing_body):
+        calls = []
+
+        def fake(url, token, method="GET", data=None):
+            calls.append((method, url, data))
+            if url.endswith("/issues?state=all&per_page=100&limit=100"):
+                return [{"number": 1, "title": compat_publish.TITLE,
+                         "state": "open", "body": existing_body,
+                         "pull_request": None}]
+            return {"number": 1}
+
+        monkeypatch.setattr(compat_publish, "_call", fake)
+        matrix = tmp_path / "m.json"
+        matrix.write_text(json.dumps(self.ROWS))
+        compat_publish.main([
+            "prog", "--api", "https://forge/api", "--token", "t",
+            "--matrix", str(matrix)])
+        return calls
+
+    def _writes(self, calls):
+        patched = [c for c in calls if c[0] == "PATCH"]
+        commented = [c for c in calls if c[0] == "POST" and c[1].endswith("/comments")]
+        return patched, commented
+
+    def test_identical_body_and_findings_touches_nothing(self, monkeypatch, tmp_path):
+        body = compat.render_issue(self.ROWS)
+        patched, commented = self._writes(self._run(monkeypatch, tmp_path, body))
+        assert not patched and not commented, "a quiet run must be completely silent"
+
+    def test_a_relabel_refreshes_the_body_but_says_NOTHING(self, monkeypatch, tmp_path):
+        # Same findings (same fingerprint), different rendering -- the exact shape of
+        # the master -> 27-dev relabel.
+        stale = compat.render_issue(self.ROWS).replace("27-dev", "unreleased")
+        assert compat.extract_fingerprint(stale) == compat.fingerprint(self.ROWS)
+
+        patched, commented = self._writes(self._run(monkeypatch, tmp_path, stale))
+        assert patched, "the body was left stale, so the issue keeps telling lies"
+        assert "27-dev" in patched[0][2]["body"]
+        assert not commented, (
+            "a rendering change is not news -- commenting on it is how the bot gets "
+            "muted before the next real finding"
+        )
+
+    def test_a_REAL_findings_change_still_comments(self, monkeypatch, tmp_path):
+        # ...and the fix must not have made it mute.
+        stale = compat.render_issue(self.ROWS).replace(
+            compat.fingerprint(self.ROWS), "0" * 16)
+        patched, commented = self._writes(self._run(monkeypatch, tmp_path, stale))
+        assert patched and commented, "a genuine change must still notify"
+
+    def test_a_body_differing_only_by_CRLF_is_not_rewritten(self, monkeypatch, tmp_path):
+        # Forges round-trip line endings. Without normalising, every run would rewrite
+        # the body -- silent, but it churns updated_at and looks freshly touched daily.
+        body = compat.render_issue(self.ROWS).replace("\n", "\r\n")
+        patched, _ = self._writes(self._run(monkeypatch, tmp_path, body))
+        assert not patched

tests/test_release_notes.py

@@ -9,6 +9,7 @@ across install.sh / uninstall.sh / recover.sh / apply.sh and nothing noticed.
 """
 
 import os
+import re
 import sys
 
 import pytest
@@ -233,3 +234,60 @@ class TestCandidateNotesResolveToTheBaseVersion:
     def test_a_genuinely_missing_section_still_raises(self):
         with pytest.raises(KeyError):
             extract_notes(self.CHANGELOG, "v9.9.9-rc1")
+
+
+class TestTheChangelogIsStructurallySound:
+    """The release body IS this file, so a mangled section ships to every user.
+
+    It has been mangled once: an edit matched the literal `## Unreleased` inside a
+    backticked phrase in a prose bullet and spliced a whole new section into the middle
+    of it, splitting the sentence in half.
+    """
+
+    def changelog(self):
+        with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh:
+            return fh.read()
+
+    def test_no_version_section_is_empty(self):
+        text = self.changelog()
+        for v in changelog_versions(text):
+            assert extract_notes(text, v).strip(), f"v{v} has an empty section"
+
+    def test_versions_are_in_descending_order(self):
+        from release_notes import version_tuple
+        versions = changelog_versions(self.changelog())
+        assert versions == sorted(versions, key=version_tuple, reverse=True), (
+            "CHANGELOG versions are out of order — a section was spliced in wrong"
+        )
+
+    def test_headings_are_at_the_start_of_a_line_and_not_inside_prose(self):
+        # A `### Fixed` that ends up indented under a bullet is a section nobody sees.
+        for i, line in enumerate(self.changelog().splitlines(), 1):
+            if line.lstrip().startswith(("## ", "### ")) and line != line.lstrip():
+                raise AssertionError(
+                    f"line {i}: heading is indented, so it is inside a list item "
+                    f"rather than being a section: {line!r}"
+                )
+
+    def test_every_bullet_that_opens_a_bold_phrase_closes_it(self):
+        # The splice cut `- **A stable release ... under \`## Unreleased` in half,
+        # leaving an unterminated ** and a dangling sentence.
+        #
+        # A bullet is the `- ` line plus everything up to the next top-level bullet or
+        # heading -- bold phrases routinely wrap across lines, so a per-line check
+        # would flag every long bullet in the file.
+        text = self.changelog()
+        bullets = re.split(r"^(?=- |#{2,3} )", text, flags=re.M)
+        bad = []
+        for b in bullets:
+            if not b.startswith("- "):
+                continue
+            # Code spans are not markup: `*args, **kwargs` is a literal, not a bold
+            # phrase, and counting its ** would flag a perfectly well-formed bullet.
+            prose = re.sub(r"`[^`]*`", "", b)
+            if prose.count("**") % 2:
+                bad.append(b.splitlines()[0][:70])
+        assert not bad, (
+            "unbalanced ** in a bullet — a section was probably spliced into the "
+            "middle of it:\n  " + "\n  ".join(bad)
+        )

tests/test_workflows.py

@@ -10,7 +10,8 @@ import re
 
 import pytest
 
-WORKFLOWS = os.path.join(os.path.dirname(__file__), "..", ".github", "workflows")
+ROOT = os.path.join(os.path.dirname(__file__), "..")
+WORKFLOWS = os.path.join(ROOT, ".github", "workflows")
 
 
 def workflow_files():
@@ -84,11 +85,77 @@ class TestBothForges:
         assert "if: ${{ contains(github.server_url, 'github.com') }}" in src
         assert "if: ${{ !contains(github.server_url, 'github.com') }}" in src
 
-    def test_compat_files_an_issue_on_each_forge(self):
+    def test_compat_files_its_report_through_ONE_implementation(self):
+        # It used to be two near-identical shell steps, one per forge. Two copies of
+        # "find the issue, decide whether to comment, post it" is two chances to drift,
+        # and the Gitea one duplicated an issue for real.
         with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
             src = fh.read()
-        assert "file a bug report (GitHub)" in src
-        assert "file a bug report (Gitea)" in src
+        assert "tools/compat_publish.py" in src
+        assert "file a bug report (GitHub)" not in src
+        assert "file a bug report (Gitea)" not in src
+
+
+class TestTheBotDoesNotSpam:
+    """It left 11 identical 3,000-character comments on one issue in a single day.
+
+    A bot that repeats itself daily gets muted — and then the next REAL finding is
+    scrolled past, which defeats the entire reason for building it.
+    """
+
+    def publisher(self):
+        with open(os.path.join(ROOT, "tools", "compat_publish.py"), encoding="utf-8") as fh:
+            return fh.read()
+
+    def test_it_compares_a_fingerprint_before_saying_anything(self):
+        src = self.publisher()
+        assert "extract_fingerprint" in src
+        assert "staying quiet" in src
+
+    def test_the_body_is_edited_in_place_not_appended_to(self):
+        src = self.publisher()
+        assert '"PATCH"' in src, "the issue body must be updated, not commented onto"
+
+    def test_it_closes_the_issue_when_everything_is_fixed(self):
+        src = self.publisher()
+        assert '"state": "closed"' in src
+
+    def test_the_matrix_refresh_opens_a_PR_rather_than_pushing_to_main(self):
+        # An unattended push to main from CI is exactly what the release barrier exists
+        # to prevent: a bot that can move main can move it somewhere nobody looked.
+        #
+        # Checked against CODE, not comments — the step's own commentary explains what
+        # it replaced, and that mention must not read as the thing itself.
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        code = "\n".join(
+            ln for ln in src.splitlines() if not ln.lstrip().startswith("#")
+        )
+        assert "/pulls" in code, "the matrix refresh must open a PR"
+        assert "HEAD:main" not in code, "CI still pushes straight to main"
+
+    def test_the_matrix_PR_targets_the_CANONICAL_forge_not_the_mirror(self):
+        # GitHub is a one-way mirror: a PR merged there would be silently clobbered by
+        # the next `fleet-repos mirror` push from Gitea. A bot opening PRs against a
+        # mirror is a bot doing nothing, slowly.
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        i = src.index("refresh the README matrix")
+        step = src[i:i + 400]
+        assert "!contains(github.server_url, 'github.com')" in step, (
+            "the matrix PR must be opened on Gitea (canonical), not GitHub (mirror)"
+        )
+
+    def test_the_workflow_has_the_permissions_its_steps_actually_need(self):
+        # It shipped with `contents: read` while the step pushed a branch and opened a
+        # PR — it would have died with a 403 on the first scheduled run, and I would
+        # have had a bot that silently never worked.
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        perms = src[src.index("permissions:"):src.index("jobs:")]
+        assert "contents: write" in perms, "pushing a branch needs contents: write"
+        assert "pull-requests: write" in perms, "opening a PR needs pull-requests: write"
+        assert "issues: write" in perms
 
 
 class TestCompatCannotSilentlyPass:

tools/compat.py

@@ -43,6 +43,7 @@ from __future__ import annotations
 
 import argparse
 import ast
+import hashlib
 import json
 import os
 import sys
@@ -132,9 +133,43 @@ ASSUMPTIONS = [
         params=["middleware", "job", "cloud_backup"],
         why="SYNC_BLOCK wraps it to tear down bind mounts in a finally",
     ),
+    Assumption(
+        # Not a plugin method -- a method on the middleware OBJECT itself, which the
+        # manifest had no way to express and therefore never checked.
+        #
+        # The nested module calls `middleware.get_service(<ns>)` to decide whether to
+        # sweep snapshots through `pool.snapshot` or `zfs.snapshot` (see
+        # SNAPSHOT_SERVICES). If it ever disappears, `_can_delete()` catches the
+        # AttributeError, reports BOTH namespaces unusable, and every nested backup
+        # fails -- loudly, but only at RUN time, on a box the preflight had already
+        # declared healthy. Checking it costs one file read.
+        "get-service", NESTED, "utils/plugins.py",
+        "LoadPluginsMixin.get_service", kind="method",
+        params=["self", "name"],
+        why="snapshot_service() resolves the snapshot namespace through it; without "
+            "it the module cannot sweep the snapshot it just took",
+    ),
 ]
 
 
+def accepted_spellings(name):
+    """The method names that satisfy a call to `<namespace>.<name>`.
+
+    A CRUDService exposes `create`/`update`/`delete` from methods NAMED
+    `do_create`/`do_update`/`do_delete`. Both are live across the matrix: 24.10 and
+    25.04 declare `do_delete`, 25.10 renamed it to `delete`, and all of them answer
+    to `<ns>.delete`. Accepting only the literal name reported working releases as
+    BROKEN and would have switched nested snapshots off on boxes where they work.
+    """
+    return (name, f"do_{name}")
+
+
+#: The spellings that satisfy `<ns>.delete`. A test binds this to the runtime's
+#: `truecloud_nested.DELETE_METHODS`, so the checker and the patch cannot come to
+#: disagree about what "can delete" means on the same box.
+DELETE_NAMES = accepted_spellings("delete")
+
+
 class MiddlewareCall:
     """A middlewared METHOD the injected code calls at runtime.
 
@@ -161,86 +196,166 @@ class MiddlewareCall:
     the whole design: declining is always the cheaper mistake.
     """
 
-    def __init__(self, ident, module, method, path, why=""):
+    def __init__(self, ident, module, method, path, why="", also=()):
         self.id = ident
         self.module = module
-        self.method = method              # "zfs.snapshot.delete"
+        self.method = method              # "pool.snapshot.delete"
         self.path = path                  # plugin file that declares it
         self.why = why
+        #: Equally acceptable spellings of the SAME call, as (method, path) pairs.
+        #:
+        #: No single snapshot namespace spans every supported release. 24.10 and
+        #: 25.04 expose the CRUD service as the public `zfs.snapshot`; 25.10
+        #: promoted it to `pool.snapshot` and demoted `zfs.snapshot` to private;
+        #: 26 deleted `plugins/zfs_/` entirely. Pinning either one alone marks
+        #: half the matrix BROKEN and declines to apply on versions that work
+        #: perfectly well.
+        #:
+        #: The call is satisfied if ANY option is present. The runtime picks the
+        #: same way -- see `pick_snapshot_service()` in the nested module -- so
+        #: what this checks and what the patch does cannot drift apart.
+        self.also = tuple(also)
 
     @property
-    def namespace(self):
-        return self.method.rsplit(".", 1)[0]
+    def options(self):
+        """Every (method, path) that would satisfy this call, best first."""
+        return ((self.method, self.path), *self.also)
 
-    @property
-    def name(self):
-        return self.method.rsplit(".", 1)[1]
+    @staticmethod
+    def namespace_of(method):
+        return method.rsplit(".", 1)[0]
+
+    @staticmethod
+    def name_of(method):
+        return method.rsplit(".", 1)[1]
 
 
 #: Every middlewared method the nested module calls at runtime.
+#: The middleware methods the nested module CALLS.
+#:
+#: These used to be the PRIVATE `zfs.*` service (`zfs.dataset.query`,
+#: `zfs.snapshot.delete`, `zfs.snapshot.query`). TrueNAS 26 deleted
+#: `plugins/zfs_/` outright and every one of them vanished -- silently, because a
+#: private service carries no stability contract and nothing warned us. The patch
+#: would have applied cleanly and then failed on the first backup.
+#:
+#: The replacements are the PUBLIC `pool.*` API, and switching to it is not merely
+#: a TrueNAS 26 fix -- it is the correct call on every version:
+#:
+#:   * It is public, documented, and covered by iX's deprecation policy, so it
+#:     cannot be deleted from under us the way `zfs.*` just was.
+#:   * The same methods, in the same files, exist on 24.10 through 26. One code
+#:     path, no version conditionals.
+#:   * Both spellings take `recursive`, so ONE call sweeps the whole tree instead
+#:     of ~250 individual deletes, any of which could be missed.
 MIDDLEWARE_CALLS = [
     MiddlewareCall(
-        "call-zfs-dataset-query", NESTED, "zfs.dataset.query",
-        "plugins/zfs_/dataset.py",
-        why="SNAPSHOT_BLOCK enumerates FILESYSTEM datasets to build the staging plan",
-    ),
-    MiddlewareCall(
-        "call-zfs-snapshot-delete", NESTED, "zfs.snapshot.delete",
-        "plugins/zfs_/snapshot.py",
+        "call-snapshot-delete", NESTED, "pool.snapshot.delete",
+        "plugins/pool_/snapshot.py",
+        also=[("zfs.snapshot.delete", "plugins/zfs_/snapshot.py")],
         why="delete_snapshot_tree() sweeps the recursive snapshot. Without it every "
             "run orphans one snapshot per descendant dataset (250 on a real pool)",
     ),
-    MiddlewareCall(
-        "call-zfs-snapshot-query", NESTED, "zfs.snapshot.query",
-        "plugins/zfs_/snapshot.py",
-        why="delete_snapshot_tree()'s fallback sweep enumerates the tree by name",
-    ),
 ]
 
+# There is deliberately NO entry here for a dataset or snapshot QUERY.
+#
+# The patch used to call `zfs.dataset.query` / `zfs.snapshot.query` (private, and
+# deleted in TrueNAS 26). The obvious port was to the public `pool.dataset.query` /
+# `pool.snapshot.query` -- and that port was WRONG in a way no source check could
+# ever have caught, because the methods are all present and correctly shaped.
+#
+# They are simply filtered. On a real box they return 205 of 274 datasets and 205
+# of 274 snapshots, hiding `ix-apps/*`, `.system/*` and `.ix-virt/*` -- 84 of 270
+# on the production pool, including live application data. Staging from that view
+# silently omits them; sweeping from it orphans one snapshot per hidden dataset,
+# forever.
+#
+# So the module enumerates from ZFS itself and there is no middleware assumption
+# left to check. That is the point: the fewer things we assume about middleware,
+# the less there is for iX to break. Only the MUTATION is still a middleware call,
+# and that is the one entry above.
+
+
+def check_call(c: MiddlewareCall, src: str | None,
+               method: str | None = None, path: str | None = None,
+               ) -> tuple[str, str | None]:
+    """Is `method` still registered by middlewared?
+
+    `method`/`path` name WHICH spelling of the call is being tried -- a call may
+    have several equally acceptable ones (see MiddlewareCall.also). They default
+    to the preferred spelling.
+    """
+    method = method or c.method
+    path = path or c.path
+    namespace = MiddlewareCall.namespace_of(method)
+    name = MiddlewareCall.name_of(method)
 
-def check_call(c: MiddlewareCall, src: str | None) -> tuple[str, str | None]:
-    """Is `c.method` still registered by middlewared?"""
     if src is None:
         return "broken", (
-            f"{c.path} no longer exists, so `{c.method}` is gone"
+            f"{path} no longer exists, so `{method}` is gone"
         )
 
     try:
         tree = ast.parse(_stock(src))
     except SyntaxError as e:
-        return "unknown", f"{c.path} does not parse: {e}"
+        return "unknown", f"{path} does not parse: {e}"
 
-    # namespace = 'zfs.snapshot' on some Service class in this file...
-    namespaces = {
-        n.value.value
-        for n in ast.walk(tree)
-        if isinstance(n, ast.Assign)
-        and isinstance(n.value, ast.Constant)
-        and isinstance(n.value.value, str)
-        and any(isinstance(t, ast.Name) and t.id == "namespace" for t in n.targets)
-    }
-    if c.namespace not in namespaces:
-        return "broken", (
-            f"{c.path} no longer declares namespace {c.namespace!r} "
-            f"(found: {sorted(namespaces) or 'none'}), so `{c.method}` is gone"
-        )
-
-    # ...and it defines the method.
+    # Find the CLASS that declares this namespace, and look for the method THERE.
+    #
+    # Not anywhere in the file. `ast.walk` over the whole module made *any* function
+    # called `delete` satisfy the check -- one on an unrelated class, or even a nested
+    # local function inside `do_query`. That is a FALSE OK, and it breaks the one
+    # invariant this checker and the runtime share: `_defines_delete()` looks in
+    # `vars(klass)` for a PLUGIN class on the service's MRO. If iX gutted
+    # `PoolSnapshotService.do_delete` while some other class in the same file still had
+    # a `delete`, compat would say ok, apply.sh would patch, and the runtime would then
+    # correctly refuse `pool.snapshot`, fall through to a `zfs.snapshot` that does not
+    # exist on 26, and fail every nested backup on a box the preflight called healthy.
+    #
+    # Same question on both sides: does the class that OWNS this namespace define the
+    # method?
     #
     # A CRUDService exposes `create`/`update`/`delete` from methods NAMED
-    # `do_create`/`do_update`/`do_delete`. Both spellings are live right now:
-    # 24.10 and 25.04 declare `do_delete`, 25.10 renamed it to `delete`, and all
-    # three answer to `zfs.snapshot.delete`. Accepting only the literal name reported
-    # the two older releases as broken -- a false BROKEN that would have switched off
-    # nested snapshots on boxes where they work.
-    defined = {
-        n.name for n in ast.walk(tree)
-        if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef)
-    }
-    if c.name not in defined and f"do_{c.name}" not in defined:
-        return "broken", f"{c.path} no longer defines `{c.method}`"
+    # `do_create`/`do_update`/`do_delete`. Both spellings are live: 24.10 and 25.04
+    # declare `do_delete`, 25.10 renamed it to `delete`, and all answer to
+    # `<ns>.delete`. Accepting only the literal name reported working releases as
+    # broken.
+    owners = []
+    all_namespaces = set()
+    for cls in (n for n in ast.walk(tree) if isinstance(n, ast.ClassDef)):
+        declared = {
+            n.value.value
+            for n in ast.walk(cls)
+            if isinstance(n, ast.Assign)
+            and isinstance(n.value, ast.Constant)
+            and isinstance(n.value.value, str)
+            and any(isinstance(t, ast.Name) and t.id == "namespace" for t in n.targets)
+        }
+        all_namespaces |= declared
+        if namespace in declared:
+            owners.append(cls)
 
-    return "ok", None
+    if not owners:
+        return "broken", (
+            f"{path} no longer declares namespace {namespace!r} "
+            f"(found: {sorted(all_namespaces) or 'none'}), so `{method}` is gone"
+        )
+
+    wanted = accepted_spellings(name)
+    for cls in owners:
+        # Direct members of the class, not its nested scopes: a `def delete` inside
+        # another method is a local function, not a service method.
+        if any(
+            isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef) and n.name in wanted
+            for n in cls.body
+        ):
+            return "ok", None
+
+    return "broken", (
+        f"{path} still declares namespace {namespace!r}, but its class no longer "
+        f"defines `{'` or `'.join(wanted)}` -- so `{method}` is gone"
+    )
 
 
 #: Things that mean iX has done the job themselves and the module should RETIRE,
@@ -403,7 +518,10 @@ def check_source(a: Assumption, src: str | None) -> tuple[str, str | None]:
     breaks a box that was working.
     """
     if src is None:
-        return "broken", f"{a.path} does not exist"
+        # Name the SYMBOL, not just the file. Whoever reads the bug report needs to
+        # know what the patch can no longer reach, and "utils/plugins.py does not
+        # exist" does not tell them that `get_service` is gone.
+        return "broken", f"{a.path} does not exist, so `{a.symbol}` is gone"
 
     try:
         tree = ast.parse(src)
@@ -538,6 +656,7 @@ def check(loader, modules=None) -> dict:
             out[a.module]["unknown"] = True
             out[a.module]["problems"].append({
                 "id": a.id, "detail": f"could not read {a.path}: {e}", "why": a.why,
+                "state": "unknown",
             })
             continue
 
@@ -545,37 +664,57 @@ def check(loader, modules=None) -> dict:
         if status == "broken":
             out[a.module]["ok"] = False
             out[a.module]["problems"].append({
-                "id": a.id, "detail": detail, "why": a.why,
+                "id": a.id, "detail": detail, "why": a.why, "state": "broken",
             })
         elif status == "unknown":
             out[a.module]["unknown"] = True
             out[a.module]["problems"].append({
-                "id": a.id, "detail": detail, "why": a.why,
+                "id": a.id, "detail": detail, "why": a.why, "state": "unknown",
             })
 
     # The methods the injected code CALLS, not just the symbols it wraps.
+    #
+    # A call may have several equally acceptable spellings, because no single
+    # snapshot namespace spans every supported release (24.10 has `zfs.snapshot`,
+    # 26 has only `pool.snapshot`). It is satisfied if ANY of them is present --
+    # exactly as the runtime resolves it -- and BROKEN only when they all vanish.
     for c in MIDDLEWARE_CALLS:
         if c.module not in out:
             continue
-        try:
-            text = src(c.path)
-        except Unreadable as e:
-            out[c.module]["unknown"] = True
-            out[c.module]["problems"].append({
-                "id": c.id, "detail": f"could not read {c.path}: {e}", "why": c.why,
-            })
+
+        satisfied, unknown, details = False, False, []
+        for method, path in c.options:
+            try:
+                text = src(path)
+            except Unreadable as e:
+                unknown = True
+                details.append(f"could not read {path}: {e}")
+                continue
+
+            status, detail = check_call(c, text, method, path)
+            if status == "ok":
+                satisfied = True
+                break
+            if status == "unknown":
+                unknown = True
+            details.append(detail)
+
+        if satisfied:
             continue
 
-        status, detail = check_call(c, text)
-        if status == "broken":
-            out[c.module]["ok"] = False
-            out[c.module]["problems"].append({
-                "id": c.id, "detail": detail, "why": c.why,
-            })
-        elif status == "unknown":
+        # Every spelling failed. If we could not READ one of them we do not know
+        # that it is broken -- a rate-limited fetch is not a regression.
+        if unknown:
             out[c.module]["unknown"] = True
             out[c.module]["problems"].append({
-                "id": c.id, "detail": detail, "why": c.why,
+                "id": c.id, "detail": "; ".join(details), "why": c.why,
+                "state": "unknown",
+            })
+        else:
+            out[c.module]["ok"] = False
+            out[c.module]["problems"].append({
+                "id": c.id, "detail": "; ".join(details), "why": c.why,
+                "state": "broken",
             })
 
     for module, (path, phrase, native_when_present) in NATIVE_PROBES.items():
@@ -732,6 +871,24 @@ def discover_refs(remote: str = REPO) -> list[str]:
       * UNRELEASED comes from the BRANCHES, because that is where a beta appears
         first: `release/26.0.0-BETA.3` had no tag yet while it was the newest beta.
         Catching breakage here, before it ships, is the whole point of this file.
+
+    THE NEXT MAINTENANCE RELEASE IS ALSO A BRANCH, and it used to fall through both
+    sieves. `release/25.10.5` is branched but not yet tagged, and 25.10 has already
+    shipped -- so it is not in `shipped` (no tag) and it was excluded from `upcoming`
+    (its line is in `shipped_lines`). It was invisible. That is the one ref a 25.10.4
+    user is actually about to be upgraded onto, so a break there reaches people
+    BEFORE the daily check ever looks at it -- the exact hole this file exists to
+    close, on the only line anybody is running.
+
+    The rule that separates it from the two things we must NOT report:
+
+      * `release/24.10-RC.2` -- a prerelease of an already-shipped line. It is
+        history, not a warning; it sorts OLDER than TS-24.10.2.4, so it is dropped.
+      * `release/25.20.2.2` -- iX's typo branch. 25.20 never shipped, so it has no
+        TS tag, so it is not a line at all and is dropped.
+
+    ...which is: a plain `release/X.Y.Z` branch counts only if its line HAS shipped
+    and it sorts NEWER than that line's newest tag. Both exclusions fall out of it.
     """
     tags = _ls_remote(remote, "--tags")
     heads = _ls_remote(remote, "--heads")
@@ -739,26 +896,45 @@ def discover_refs(remote: str = REPO) -> list[str]:
     shipped = _newest_per_line([
         t for t in tags if t.startswith("TS-") and "-BETA" not in t and "-RC" not in t
     ])
+    newest_shipped = {_version_of(t)[0][:2]: _version_of(t) for t in shipped}
+
+    release_heads = [h for h in heads if h.startswith("release/")]
 
     # A prerelease of a line that has ALREADY shipped is history, not a warning:
     # release/24.10-RC.2 still exists, and the nested module does not apply to it,
     # but 24.10 shipped long ago and TS-24.10.2.4 is fine. Reporting it would be a
     # standing red row in the matrix for a version nobody can install.
-    shipped_lines = {_version_of(t)[0][:2] for t in shipped}
     upcoming = [
         h for h in _newest_per_line([
-            h for h in heads
-            if h.startswith("release/") and ("-BETA" in h or "-RC" in h)
+            h for h in release_heads if "-BETA" in h or "-RC" in h
         ])
-        if _version_of(h)[0][:2] not in shipped_lines
+        if _version_of(h)[0][:2] not in newest_shipped
     ]
 
-    return [*shipped, *upcoming, "master"]
+    # The next maintenance release of a line that HAS shipped: branched, untagged,
+    # and the very next thing those users get. See the docstring.
+    pending = []
+    for h in _newest_per_line([
+        h for h in release_heads if "-BETA" not in h and "-RC" not in h
+    ]):
+        v = _version_of(h)
+        tagged = newest_shipped.get(v[0][:2])
+        if tagged and v > tagged:
+            pending.append(h)
+
+    return [*shipped, *pending, *upcoming, "master"]
 
 
 def is_unreleased(ref: str) -> bool:
-    """master and any BETA/RC. Breakage here is early warning, not an outage."""
-    return ref == "master" or "-BETA" in ref or "-RC" in ref
+    """Anything iX has not TAGGED. Breakage here is early warning, not an outage.
+
+    Keyed on where the ref came from, not on its name: `discover_refs` takes shipped
+    releases from `TS-*` TAGS and everything else from BRANCHES, so a `release/*` ref
+    is by construction something iX has not released yet. Testing for `-BETA`/`-RC`
+    instead would call `release/25.10.5` SHIPPED, and a break there would fail the
+    build as a live outage -- on a version nobody is running yet.
+    """
+    return ref == "master" or ref.startswith("release/")
 
 
 def matrix(refs=None, remote: str = REPO) -> list[dict]:
@@ -802,7 +978,16 @@ def is_broken(r: dict) -> bool:
 #: is static analysis of iX's source, which proves the patch's assumptions hold --
 #: a strictly weaker claim than "a restore worked". Add a row only after doing it.
 HARDWARE_VERIFIED = {
-    "25.10.4": "nested + providers; 252-snapshot recursive backup of /mnt/Tap, 18m",
+    "25.10.4": (
+        "v0.7.0: 3 live tasks — 191-dataset nested backup of /mnt/Tap, a "
+        "215-filesystem/2-zvol backup of /mnt/Tank/backups, and a non-nested one; "
+        "0 orphans, 0 leaked mounts, byte-identical restore; the collector also "
+        "reclaimed a real orphan the pool had been carrying"
+    ),
+    "26.0.0-BETA.1": (
+        "v0.7.0: 274-snapshot recursive backup of a 292-dataset pool; restored a "
+        "4-deep child dataset byte-identical; zvol-orphan case reproduced then closed"
+    ),
 }
 
 _LEGEND = """
@@ -815,6 +1000,16 @@ _LEGEND = """
 "ok" means *the patch's assumptions hold*, checked automatically against iX's
 source. It does not mean a human ran a backup on it — that is the
 **Hardware-verified** column, which is filled in by hand and only by doing it.
+
+**`master` is not the next release.** iX branches each major off to its own
+`release/` line and master rolls straight on to the one after — so master is
+`27-dev` while 26 is still in beta. A **BROKEN** master means iX has changed
+something that will reach users *a major release from now*, not in the version you
+are about to install. Read the numbered rows for that.
+
+A row like `25.10.5 _(unreleased)_` is the next maintenance release: branched by iX,
+not tagged yet, and the very next thing a 25.10.4 box gets. It is checked precisely
+because it is the one unshipped ref that reaches real users without warning.
 """
 
 
@@ -862,17 +1057,42 @@ def update_readme(rows: list[dict], path: str = README) -> bool:
     return True
 
 
+def dev_label(rows: list[dict]) -> str:
+    """What `master` is a development line FOR -- e.g. "27-dev".
+
+    `master` is NOT the next release, and labelling it "master _(unreleased)_" said
+    it was. On 2026-07-14 every recent commit on master targeted 27.0.0-BETA.1 while
+    26 was still in beta on its own `release/26.0.0-BETA.*` branches: master had
+    already rolled over to the major AFTER the one that has not shipped yet. So a red
+    `master` row read as "the version you are about to install is broken" when the
+    breakage was a year out, on a line nobody can even download. That is a false alarm
+    aimed squarely at the person deciding whether to trust this with their backups.
+
+    Derived, not hardcoded: the newest major we can see anywhere, plus one. iX branches
+    `release/N.0.0-BETA.1` off master and master immediately becomes N+1 -- so when 27
+    betas appear, this says 28-dev on its own.
+    """
+    majors = [
+        v[0][0] for v in (_version_of(r["ref"]) for r in rows if r["ref"] != "master")
+        if v
+    ]
+    return f"{max(majors) + 1}-dev" if majors else "unreleased"
+
+
 def render_markdown(rows: list[dict]) -> str:
     """The matrix, for the README."""
     out = [
         "| TrueNAS | B2/S3 providers | Nested snapshots | Hardware-verified |",
         "| --- | --- | --- | --- |",
     ]
+    dev = dev_label(rows)
     for row in rows:
         m = row["modules"]
         ref = row["ref"]
         label = ref.removeprefix("TS-").removeprefix("release/")
-        if row["unreleased"]:
+        if ref == "master":
+            label = f"master _({dev})_"
+        elif row["unreleased"]:
             label = f"{label} _(unreleased)_"
 
         cells = []
@@ -892,6 +1112,96 @@ def render_markdown(rows: list[dict]) -> str:
     return "\n".join(out) + "\n" + _LEGEND
 
 
+FINGERPRINT = "<!-- compat-fingerprint:"
+
+
+def fingerprint(rows: list[dict]) -> str:
+    """A stable digest of WHAT IS BROKEN, and nothing else.
+
+    The bug report must be updated when the findings change and stay silent when they
+    do not. Without this the workflow commented on every run -- it left **11 identical
+    3,000-character comments** on one issue in a single day, which is not a warning
+    system, it is a mute button with extra steps.
+
+    Deliberately excludes anything that moves on its own: the matrix's `ok` rows, the
+    hardware-verified column, and the exact TrueNAS point-release (`TS-25.10.4` ->
+    `TS-25.10.5` is not news). Only the broken (ref, module, problem-id) triples count.
+    """
+    findings = sorted(
+        (r["ref"], mod, p["id"])
+        for r in rows
+        for mod, m in r["modules"].items()
+        if is_broken(m)
+        for p in m["problems"]
+        # `unknown` problems are things we could not READ (a 429, an EACCES), not
+        # things iX changed. On a ref that is broken for some other reason they would
+        # otherwise join the digest, so one transient network blip rewrites the issue
+        # body and the next clean run rewrites it back. That is the daily-noise
+        # failure this fingerprint exists to prevent, wearing a different hat.
+        if p.get("state", "broken") == "broken"
+    )
+    return hashlib.sha256(repr(findings).encode()).hexdigest()[:16]
+
+
+def extract_fingerprint(body: str) -> str | None:
+    """The fingerprint a previous run left in the issue body, if any."""
+    if not body:
+        return None
+    i = body.find(FINGERPRINT)
+    if i == -1:
+        return None
+    return body[i + len(FINGERPRINT):].split("-->", 1)[0].strip() or None
+
+
+def render_issue(rows: list[dict]) -> str:
+    """The bug report body: what is broken, why it matters, and nothing else up front.
+
+    Short by design. The full matrix and the healthy versions go in a fold -- somebody
+    opening this wants to know what broke and whether it can hurt them, not to re-read
+    a table they can see in the README.
+    """
+    broken = [r for r in rows if any(is_broken(m) for m in r["modules"].values())]
+
+    out = [
+        "`tools/compat.py` checks what this patch assumes about middlewared against "
+        "iXsystems' actual source, every day. Those assumptions no longer hold on the "
+        "versions below.",
+        "",
+        "**This does not break anyone today.** `apply.sh` re-checks on every boot and "
+        "**declines to apply** a module whose assumptions fail, so TrueNAS is left "
+        "stock rather than half-patched. The cost is the module's feature, not a "
+        "broken backup.",
+        "",
+    ]
+
+    for r in broken:
+        out.append(f"### `{r['ref']}`")
+        out.append("")
+        for mod, m in sorted(r["modules"].items()):
+            if not is_broken(m):
+                continue
+            out.append(f"**{mod}**")
+            out.append("")
+            for p in m["problems"]:
+                out.append(f"- {p['detail']}")
+                out.append(f"  <br><sub>{p['why']}</sub>")
+            out.append("")
+
+    out += [
+        "<details><summary>Full support matrix</summary>",
+        "",
+        render_markdown(rows),
+        "</details>",
+        "",
+        "_Filed and kept up to date by "
+        "[`compat.yml`](.github/workflows/compat.yml). It edits this body when the "
+        "findings change, and stays quiet when they do not._",
+        "",
+        f"{FINGERPRINT} {fingerprint(rows)} -->",
+    ]
+    return "\n".join(out)
+
+
 def render_matrix(rows: list[dict]) -> str:
     """A support table.
 
@@ -900,7 +1210,13 @@ def render_matrix(rows: list[dict]) -> str:
     hardware-verified column lives in COMPATIBILITY.md and is maintained by hand,
     because nothing else can honestly fill it in.
     """
-    w = max((len(r["ref"]) for r in rows), default=10)
+    dev = dev_label(rows)
+    # Same relabel as the README: a red `master` is a warning about the major AFTER
+    # next, and "master" alone reads as "the release you are about to install".
+    names = {r["ref"]: (f"master ({dev})" if r["ref"] == "master" else r["ref"])
+             for r in rows}
+
+    w = max((len(n) for n in names.values()), default=10)
     lines = [
         f"{'TrueNAS'.ljust(w)}  {'providers':<10}  {'nested':<10}",
         f"{'-' * w}  {'-' * 10}  {'-' * 10}",
@@ -908,7 +1224,7 @@ def render_matrix(rows: list[dict]) -> str:
     for row in rows:
         m = row["modules"]
         lines.append(
-            f"{row['ref'].ljust(w)}  "
+            f"{names[row['ref']].ljust(w)}  "
             f"{_verdict(m[PROVIDERS]):<10}  {_verdict(m[NESTED]):<10}"
         )
     return "\n".join(lines)

tools/compat_publish.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""Keep ONE bug report in sync with what compat.py currently finds.
+
+WHY THIS IS NOT JUST "POST A COMMENT"
+-------------------------------------
+The first version commented on every run that found a break. In one day it left
+**11 identical 3,000-character comments** on the same issue. That is not a warning
+system; it is a mute button with extra steps. The next real finding would have been
+scrolled past, which defeats the entire point of building it.
+
+So:
+
+  * **The issue body is the current truth.** It is edited in place, never appended to.
+  * **Comments are a changelog of CHANGES.** A run whose findings are identical to the
+    last one says nothing at all -- no comment, no edit, no notification.
+  * A fingerprint of the findings (broken ref/module/problem triples only) is embedded
+    in the body. It deliberately ignores things that move on their own -- healthy rows,
+    the hardware-verified column, TrueNAS point releases -- so `TS-25.10.4` becoming
+    `TS-25.10.5` is not news, and does not wake anybody up.
+
+  * When everything is fixed, the issue is **closed** with a comment saying so.
+
+Works against GitHub and Gitea, which differ only in the auth header and the issue
+list URL. One implementation, so the two cannot drift.
+
+    python3 tools/compat_publish.py --api <url> --token <tok> --matrix /tmp/matrix.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.error
+import urllib.request
+
+sys.path.insert(0, __file__.rsplit("/", 1)[0])
+
+from compat import (  # noqa: E402
+    extract_fingerprint,
+    fingerprint,
+    is_broken,
+    render_issue,
+)
+
+TITLE = "TrueNAS compatibility: the patch's assumptions no longer hold"
+
+
+def _call(url, token, method="GET", data=None):
+    req = urllib.request.Request(
+        url, method=method,
+        headers={
+            # Gitea wants `token <t>`; GitHub accepts `Bearer <t>`. GitHub also
+            # accepts `token <t>`, so one header serves both.
+            "Authorization": f"token {token}",
+            "Content-Type": "application/json",
+            "Accept": "application/vnd.github+json",
+        },
+        data=json.dumps(data).encode() if data else None,
+    )
+    with urllib.request.urlopen(req) as r:  # noqa: S310
+        return json.load(r) if r.length != 0 else {}
+
+
+def _same(a, b):
+    """Is the issue body already what we would write?
+
+    Compared after normalising line endings and trailing space: forges are free to
+    round-trip `\r\n`, and a body that only "differs" by that would be rewritten on
+    every single run -- a silent edit, but a pointless one that churns `updated_at`
+    and makes the issue look freshly touched every morning.
+    """
+    def norm(s):
+        return "\n".join(line.rstrip() for line in (s or "").replace("\r\n", "\n").split("\n")).strip()
+    return norm(a) == norm(b)
+
+
+def find_issue(api, token, title):
+    """The LOWEST-numbered issue with this title, open or closed.
+
+    Lowest, not "whichever the API returns first": two issues with the same title
+    existed once (an earlier version put the ref list in the title, so the identity
+    changed whenever that set changed), and an order-dependent pick would alternate
+    between them -- reopening one while commenting on the other.
+
+    Both forges list PRs alongside issues, but they SAY SO DIFFERENTLY: GitHub omits
+    the `pull_request` key on a plain issue, Gitea sends it as `null`. Testing for the
+    KEY therefore discards every Gitea issue as if it were a PR -- so this returned
+    None on every Gitea run, and the bot filed a brand-new duplicate report each time
+    instead of editing the one it already had. Test the VALUE; it is the only form
+    that is true on both.
+    """
+    issues = _call(f"{api}/issues?state=all&per_page=100&limit=100", token)
+    mine = [
+        i for i in issues
+        if i.get("title") == title and not i.get("pull_request")
+    ]
+    return min(mine, key=lambda i: i["number"]) if mine else None
+
+
+def main(argv):
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("--api", required=True, help="…/repos/<owner>/<repo>")
+    ap.add_argument("--token", required=True)
+    ap.add_argument("--matrix", required=True, help="compat.py --matrix --json output")
+    args = ap.parse_args(argv[1:])
+
+    with open(args.matrix, encoding="utf-8") as fh:
+        rows = json.load(fh)
+
+    broken = [r for r in rows if any(is_broken(m) for m in r["modules"].values())]
+    issue = find_issue(args.api, args.token, TITLE)
+
+    # ── everything is healthy ────────────────────────────────────────────────
+    if not broken:
+        if issue and issue["state"] == "open":
+            _call(f"{args.api}/issues/{issue['number']}/comments", args.token, "POST",
+                  {"body": "All of the patch's assumptions hold again on every "
+                           "checked TrueNAS version. Closing."})
+            _call(f"{args.api}/issues/{issue['number']}", args.token, "PATCH",
+                  {"state": "closed"})
+            print(f"closed #{issue['number']} — nothing is broken any more")
+        else:
+            print("nothing broken; no open report to close")
+        return 0
+
+    body = render_issue(rows)
+    want = fingerprint(rows)
+
+    # ── nothing to file yet ──────────────────────────────────────────────────
+    if issue is None:
+        made = _call(f"{args.api}/issues", args.token, "POST",
+                     {"title": TITLE, "body": body})
+        print(f"filed #{made['number']}")
+        return 0
+
+    have = extract_fingerprint(issue.get("body") or "")
+    n = issue["number"]
+
+    # ── two different questions, and they were being answered with one answer ────
+    #
+    #   * IS THE BODY STILL TRUE?     -> if not, rewrite it. Editing an issue body
+    #     notifies NOBODY on either forge, so keeping it honest is free.
+    #   * HAVE THE FINDINGS CHANGED?  -> only then comment. Comments DO notify, and a
+    #     daily "still broken, same as yesterday" is what teaches everyone to ignore
+    #     the one that finally matters.
+    #
+    # Conflating them meant an unchanged FINGERPRINT froze the BODY. The fingerprint
+    # deliberately ignores everything that moves on its own -- healthy rows, the
+    # hardware-verified column, point releases, how a row is LABELLED -- so none of
+    # that could ever reach the report. Relabelling master `27-dev` (it is not the
+    # next release; a red row there was reading as "the version you are about to
+    # install is broken") would have shipped to the README and never to the issue
+    # anybody actually opens.
+    body_is_current = _same(issue.get("body"), body)
+
+    if have == want and issue["state"] == "open" and body_is_current:
+        print(f"#{n} is already current ({want}) — staying quiet")
+        return 0
+
+    _call(f"{args.api}/issues/{n}", args.token, "PATCH", {"body": body, "state": "open"})
+
+    if have != want:
+        refs = ", ".join(f"`{r['ref']}`" for r in broken)
+        note = (
+            "The findings changed — the report above has been updated.\n\n"
+            f"Currently broken on: {refs}."
+            if have else
+            "This report is now kept up to date automatically: the body above always "
+            "reflects the current findings, and a comment is only added when they "
+            "change."
+        )
+        _call(f"{args.api}/issues/{n}/comments", args.token, "POST", {"body": note})
+        print(f"updated #{n}: {have} -> {want}")
+    elif issue["state"] != "open":
+        print(f"reopened #{n}")
+    else:
+        # Same findings, new rendering. Silent by design: nothing has changed that
+        # anybody needs waking up for, but the report should not be telling lies.
+        print(f"#{n}: findings unchanged ({want}); body refreshed silently")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

uninstall.sh

@@ -3,7 +3,7 @@
 
 set -euo pipefail
 
-VERSION="0.6.0"
+VERSION="0.7.0"
 
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 _HOOK_COMMENT='TrueCloud provider patch (S3/B2)'

update.sh

@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-VERSION="0.6.0"
+VERSION="0.7.0"
 
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 _PREV_FILE="$PATCH_DIR/.update_previous"

uv.lock

@@ -0,0 +1,3 @@
+version = 1
+revision = 3
+requires-python = ">=3.12"