docs: refresh the TrueNAS compatibility matrix

Reviewed-on: #12

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: sudolulo
+ko_fi: sudolulo

.github/workflows/ci.yml

@@ -43,20 +43,21 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: install dev deps
-        run: python -m pip install --upgrade pip pytest ruff
+      # uv-managed interpreters instead of actions/setup-python: the prebuilt-CPython
+      # download path setup-python relies on does not work on the self-hosted Gitea
+      # runner (all three matrix jobs failed at setup there while passing on GitHub);
+      # uv works identically on both.
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
 
       - name: ruff
-        run: ruff check patch tests tools
+        # Pinned: an unpinned ruff means any upstream release can turn main red
+        # with no code change.
+        run: uvx ruff@0.16.1 check patch tests tools
 
       - name: pytest
-        run: pytest tests -v
+        run: uvx --python ${{ matrix.python }} pytest tests -v
 
       - name: verify injected middleware blocks compile
         # Belt-and-braces: the *_BLOCK strings are appended into live middlewared
         # modules. A syntax error there would break the box at boot.
-        run: pytest tests/test_apply_blocks.py -v
+        run: uvx --python ${{ matrix.python }} pytest tests/test_apply_blocks.py -v

.github/workflows/compat.yml

@@ -0,0 +1,241 @@
+name: TrueNAS compatibility
+
+# Find out that iX broke us BEFORE their release ships, not after a user's backup
+# fails.
+#
+# This patch appends code to middlewared's internal modules. There is no stability
+# contract: TrueNAS 26 rewrote the whole cloud_backup path from async to sync, and
+# every block the nested module injects is an `async def` wrapping an `await`ed
+# original. Nobody would have found out until a restore did not work.
+#
+# tools/compat.py records what the patch assumes and checks it against iX's actual
+# source at every release line -- including master and the current BETA/RC, which is
+# where a break shows up first. When an UNRELEASED line breaks, this opens a bug
+# report so there is time to fix it before that version reaches anyone.
+#
+# Runs on both forges: Gitea (canonical) and GitHub (mirror). Only the "file an
+# issue" call differs.
+
+on:
+  schedule:
+    - cron: "17 6 * * *"     # daily, off the hour: everyone crons on the hour
+  workflow_dispatch:
+  push:
+    paths:
+      # The manifest itself changed -- re-check immediately rather than waiting a day.
+      - "tools/compat.py"
+      # ...and so did the thing that PUBLISHES the finding. This was missing, and it
+      # showed: the commit teaching the bot to refresh a stale report body touched only
+      # compat_publish.py, so no run fired, and the report stayed stale until the next
+      # scheduled one. A fix nobody runs is a fix nobody has.
+      - "tools/compat_publish.py"
+      - ".github/workflows/compat.yml"
+
+permissions:
+  # write, because the matrix refresh pushes a branch and opens a PR. It does NOT get
+  # to move `main` -- that is the whole reason it is a PR. See the refresh step below.
+  contents: write
+  pull-requests: write
+  issues: write
+
+jobs:
+  compat:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      # ONE pass over the network. Every other step renders from this JSON -- calling
+      # compat.py three times would re-fetch every file from every release line three
+      # times, and could even disagree with itself if iX pushed mid-run.
+      #
+      # The exit code is CAPTURED, not allowed to abort the step: a nonzero exit means
+      # "a shipped release is broken", which is a result to report, not a reason to
+      # die before reporting it. (Actions runs `bash -e`, so `cmd > out` followed by
+      # `echo $?` never reaches the echo.) It becomes a job failure at the end, after
+      # the bug report has been filed.
+      - name: check every TrueNAS release line
+        id: check
+        run: |
+          rc=0
+          python3 tools/compat.py --matrix --json > /tmp/matrix.json || rc=$?
+          echo "shipped_broken=$rc" >> "$GITHUB_OUTPUT"
+
+      # The report body is written to a FILE, and never becomes a step output.
+      #
+      # An earlier version did `echo "${{ steps.report.outputs.body }}"`, which
+      # splices the text into the shell script itself -- and the report is full of
+      # backticks, so bash ran `create-snapshot`, `def` and `async` as commands. It
+      # is also an injection vector: the report is built from iX's source, so
+      # anything that lands in middleware would execute on the runner.
+      #
+      # The rule that avoids the whole class: never interpolate ${{ }} into a `run:`
+      # body. Files for data, `env:` for scalars (the runner sets those, rather than
+      # pasting them into the script).
+      - name: build the report
+        id: report
+        run: |
+          python3 - <<'PY' >> "$GITHUB_OUTPUT"
+          import json, sys
+
+          sys.path.insert(0, "tools")
+          import compat
+
+          with open("/tmp/matrix.json") as fh:
+              rows = json.load(fh)
+
+          with open("/tmp/matrix.md", "w") as fh:
+              fh.write(compat.render_markdown(rows))
+
+          broken = [
+              r for r in rows
+              if any(compat.is_broken(m) for m in r["modules"].values())
+          ]
+          native = [
+              (r["ref"], mod)
+              for r in rows
+              for mod, m in sorted(r["modules"].items())
+              if m["native"] and not compat.is_broken(m)
+          ]
+
+          lines = [
+              "`tools/compat.py` found that the patch's assumptions about "
+              "middlewared no longer hold.",
+              "",
+              compat.render_markdown(rows),
+              "",
+          ]
+          for r in broken:
+              lines.append(f"### {r['ref']}")
+              lines.append("")
+              for mod, m in sorted(r["modules"].items()):
+                  if not compat.is_broken(m):
+                      continue
+                  lines.append(f"**{mod}** — the patch will not apply:")
+                  lines.append("")
+                  for p in m["problems"]:
+                      lines.append(f"- `{p['id']}`: {p['detail']}")
+                      lines.append(f"  - why it matters: {p['why']}")
+                  lines.append("")
+          for ref, mod in native:
+              lines.append(
+                  f"- `{ref}`: **{mod}** appears to be NATIVE now — retire the "
+                  f"module rather than fixing it."
+              )
+          lines += ["", "_Filed automatically by `.github/workflows/compat.yml`._"]
+
+          with open("/tmp/issue.md", "w") as fh:
+              fh.write("\n".join(lines))
+
+          # Scalars only. The body stays in the file.
+          print(f"broken={'1' if broken else '0'}")
+          print(f"refs={','.join(r['ref'] for r in broken)}")
+          PY
+
+      - name: matrix
+        run: cat /tmp/matrix.md
+
+      # Keep the README's table true — as a PULL REQUEST, on the CANONICAL forge.
+      #
+      # Two things this gets right that the obvious version gets wrong:
+      #
+      #  1. It is a PR, not a push to main. This used to `git push origin HEAD:main`
+      #     from CI. An unattended write to main is exactly what the release barrier
+      #     exists to prevent — a bot that can move main can move it somewhere nobody
+      #     looked. Nothing lands by itself.
+      #
+      #  2. It runs on GITEA, not GitHub. GitHub is a one-way MIRROR: a PR merged there
+      #     would be silently clobbered by the next `fleet-repos mirror` push from Gitea.
+      #     A bot opening PRs against a mirror is a bot doing nothing, slowly.
+      #
+      # A stale support matrix is not a stale doc — it is a false promise to somebody
+      # deciding whether to trust this with their backups. So it is refreshed daily; it
+      # just asks first.
+      - name: refresh the README matrix (PR on the canonical forge)
+        if: ${{ github.event_name == 'schedule' && !contains(github.server_url, 'github.com') }}
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+        run: |
+          python3 - <<'PY'
+          import json, sys
+          sys.path.insert(0, "tools")
+          import compat
+          with open("/tmp/matrix.json") as fh:
+              rows = json.load(fh)
+          print("changed" if compat.update_readme(rows) else "unchanged")
+          PY
+
+          git diff --quiet -- README.md && { echo "matrix unchanged — nothing to propose"; exit 0; }
+
+          git config user.name  "truecloud-patch bot"
+          git config user.email "bot@onetick.ninja"
+
+          BRANCH=bot/compat-matrix
+          git checkout -B "$BRANCH"
+          git add README.md
+          git commit -m "docs: refresh the TrueNAS compatibility matrix"
+          git push -f origin "$BRANCH"
+
+          # ONE long-lived PR, force-pushed in place — not a new one every morning.
+          # (A daily PR is the same mistake as a daily comment, wearing a hat.)
+          BRANCH="$BRANCH" python3 - <<'PY'
+          import json, os, urllib.error, urllib.request
+
+          api, token, branch = os.environ["API"], os.environ["TOKEN"], os.environ["BRANCH"]
+          h = {"Authorization": f"token {token}", "Content-Type": "application/json"}
+
+          def call(url, method="GET", data=None):
+              r = urllib.request.Request(
+                  url, method=method, headers=h,
+                  data=json.dumps(data).encode() if data else None)
+              with urllib.request.urlopen(r) as resp:  # noqa: S310
+                  return json.load(resp) if resp.length != 0 else {}
+
+          existing = [
+              p for p in call(f"{api}/pulls?state=open")
+              if p["head"]["ref"] == branch
+          ]
+          if existing:
+              print(f"PR #{existing[0]['number']} already open; the force-push updated it")
+          else:
+              pr = call(f"{api}/pulls", "POST", {
+                  "head": branch, "base": "main",
+                  "title": "docs: refresh the TrueNAS compatibility matrix",
+                  "body": (
+                      "The daily compatibility check found that the support matrix in "
+                      "the README no longer matches iXsystems' actual middleware.\n\n"
+                      "This only touches the block between the `COMPAT MATRIX` markers. "
+                      "It is regenerated by `tools/compat.py --matrix --update-readme` "
+                      "and force-pushed, so it always reflects the latest run."
+                  ),
+              })
+              print(f"opened PR #{pr['number']}")
+          PY
+
+      # ONE bug report, kept in sync. It is edited in place when the findings change and
+      # says NOTHING when they do not.
+      #
+      # The first version commented on every run and left 11 identical 3,000-character
+      # comments on one issue in a single day. A bot that repeats itself daily gets
+      # muted, and then the next real finding is scrolled past — which defeats the whole
+      # reason for building it.
+      #
+      # Runs on whichever forge it lands on; compat_publish.py handles both, so the two
+      # cannot drift.
+      - name: file / update / close the bug report
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
+          API: ${{ contains(github.server_url, 'github.com') && 'https://api.github.com' || format('{0}/api/v1', github.server_url) }}/repos/${{ github.repository }}
+        run: python3 tools/compat_publish.py --api "$API" --token "$TOKEN" --matrix /tmp/matrix.json
+
+      # A broken SHIPPED release is an outage: users are on it right now. Fails LAST, so
+      # the report is filed before the job goes red.
+      - name: fail if a shipped release is broken
+        if: ${{ steps.check.outputs.shipped_broken != '0' }}
+        run: |
+          echo "::error::The patch is broken on a SHIPPED TrueNAS release."
+          exit 1

.github/workflows/release.yml

@@ -30,15 +30,30 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
+      # Via `env:`, never spliced into the script. `inputs.tag` is attacker-chosen on
+      # a workflow_dispatch, and a ${{ }} in a `run:` body is pasted into the shell
+      # TEXT -- a tag of `$(...)` would simply execute. env: is safe: the runner sets
+      # the variable instead of rewriting the script.
       - name: Resolve tag
         id: tag
+        env:
+          EVENT: ${{ github.event_name }}
+          INPUT_TAG: ${{ inputs.tag }}
         run: |
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            echo "tag=${{ inputs.tag }}" >> "$GITHUB_OUTPUT"
+          if [ "$EVENT" = "workflow_dispatch" ]; then
+            tag="$INPUT_TAG"
           else
-            echo "tag=${GITHUB_REF#refs/tags/}" >> "$GITHUB_OUTPUT"
+            tag="${GITHUB_REF#refs/tags/}"
           fi
 
+          # Whatever it came from, it has to look like a tag we cut.
+          case "$tag" in
+            v[0-9]*.[0-9]*.[0-9]*) ;;
+            *) echo "::error::refusing to release a tag that is not vX.Y.Z[-rcN]: $tag"; exit 1 ;;
+          esac
+
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+
       - uses: actions/checkout@v4
         with:
           ref: ${{ steps.tag.outputs.tag }}
@@ -70,23 +85,62 @@ jobs:
 
       # Catches the failure mode this repo actually had: VERSION= drifted to
       # three different values across the scripts, and nothing noticed.
-      - name: version matches tag and CHANGELOG has a section
-        run: python3 tools/release_notes.py check "${{ steps.tag.outputs.tag }}"
+      - name: "gate: version matches tag, CHANGELOG complete, nothing stranded"
+        env:
+          TAG: ${{ steps.tag.outputs.tag }}
+        run: python3 tools/release_notes.py check "$TAG"
+
+      # THE BARRIER. A stable release must have been a release candidate on this
+      # exact commit. Candidates are invisible to users (update.sh and the alert
+      # source both take the newest plain vX.Y.Z), so debugging happens across
+      # rc1/rc2/rc3 at no cost to anyone -- instead of across v0.5.0/v0.5.1/v0.5.2,
+      # which alerts every installed box every time.
+      #
+      # Same code release.sh runs locally, so this should never be the first place
+      # you find out. It is here because this is the only place that cannot be
+      # bypassed: it holds the token that publishes.
+      - name: "gate: this commit was a release candidate"
+        env:
+          TAG: ${{ steps.tag.outputs.tag }}
+        run: python3 tools/release_gate.py "$TAG" -C .
+
+      # There is deliberately NO "did the candidate's CI run pass?" gate here.
+      #
+      # It would have to query the forge's run history, which is the one thing that
+      # differs between GitHub and Gitea -- and it adds nothing: the steps above
+      # re-run ruff, pytest and the shell checks against the TAGGED COMMIT, and
+      # release_gate.py has already proved a candidate points at that same commit.
+      # If the code passes now, it passed then; they are the same code.
+      #
+      # What a candidate really buys is the thing no CI can check: that a human
+      # installed it on a real box and exercised it. The barrier makes room for
+      # that; it cannot verify it.
 
       - name: extract release notes from CHANGELOG
+        env:
+          TAG: ${{ steps.tag.outputs.tag }}
         run: |
-          python3 tools/release_notes.py notes "${{ steps.tag.outputs.tag }}" > /tmp/notes.md
+          python3 tools/release_notes.py notes "$TAG" > /tmp/notes.md
           echo "--- release body ---"
           cat /tmp/notes.md
 
-      - name: create or update the release
+      # This repo is canonically hosted on Gitea (git.arch.fyi) and mirrored to
+      # GitHub, and BOTH run this workflow -- Gitea reads .github/workflows too. So
+      # the publish step has to work on whichever forge it lands on. Everything
+      # above is forge-agnostic; only the "create a release" API differs.
+
+      - name: publish the release (GitHub)
+        if: ${{ contains(github.server_url, 'github.com') }}
         env:
           GH_TOKEN: ${{ github.token }}
           TAG: ${{ steps.tag.outputs.tag }}
         run: |
-          # Pre-1.0 and any -rc/-beta suffix ship as prereleases, not "Latest".
+          # Lowercased: release_gate/release_notes match the suffix case-INsensitively
+          # (`is_prerelease` uses re.I), so a `v0.6.0-RC1` skipped the barrier as a
+          # candidate and then landed here as a case-sensitive MISS -- published as the
+          # forge's "Latest release" on a commit that was never a candidate.
           prerelease=""
-          case "$TAG" in
+          case "$(printf '%s' "$TAG" | tr '[:upper:]' '[:lower:]')" in
             *-rc*|*-beta*|*-alpha*) prerelease="--prerelease" ;;
           esac
 
@@ -95,8 +149,60 @@ jobs:
             gh release edit "$TAG" --notes-file /tmp/notes.md
           else
             # shellcheck disable=SC2086
-            gh release create "$TAG" \
-              --title "$TAG" \
-              --notes-file /tmp/notes.md \
-              $prerelease
+            gh release create "$TAG" --title "$TAG" --notes-file /tmp/notes.md $prerelease
           fi
+
+      - name: publish the release (Gitea)
+        if: ${{ !contains(github.server_url, 'github.com') }}
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
+          TAG: ${{ steps.tag.outputs.tag }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+        run: |
+          prerelease=false
+          case "$(printf '%s' "$TAG" | tr '[:upper:]' '[:lower:]')" in
+            *-rc*|*-beta*|*-alpha*) prerelease=true ;;
+          esac
+
+          # python3, not jq. The changelog is full of quotes, backticks and newlines,
+          # so the body must be properly JSON-encoded -- but `jq` is not guaranteed on
+          # a self-hosted Gitea runner, and a publish step that dies on a missing tool
+          # leaves a tag with no release behind it. python3 is guaranteed: setup-python
+          # ran above.
+          python3 - "$TAG" "$API" "$TOKEN" "$prerelease" <<'PY'
+          import json, sys, urllib.error, urllib.request
+
+          tag, api, token, prerelease = sys.argv[1:5]
+          with open("/tmp/notes.md", encoding="utf-8") as fh:
+              body = fh.read()
+
+          payload = {
+              "tag_name": tag, "name": tag, "body": body,
+              "prerelease": prerelease == "true",
+          }
+          headers = {
+              "Authorization": f"token {token}",
+              "Content-Type": "application/json",
+          }
+
+          def call(url, method, data=None):
+              req = urllib.request.Request(
+                  url, method=method, headers=headers,
+                  data=json.dumps(data).encode() if data else None)
+              with urllib.request.urlopen(req) as r:  # noqa: S310
+                  return r.status, json.load(r) if r.length != 0 else {}
+
+          try:
+              _, existing = call(f"{api}/releases/tags/{tag}", "GET")
+          except urllib.error.HTTPError as e:
+              if e.code != 404:
+                  raise
+              existing = None
+
+          if existing:
+              status, _ = call(f"{api}/releases/{existing['id']}", "PATCH", payload)
+              print(f"updated release {tag} -> {status}")
+          else:
+              status, _ = call(f"{api}/releases", "POST", payload)
+              print(f"created release {tag} (prerelease={payload['prerelease']}) -> {status}")
+          PY

.gitignore

@@ -5,6 +5,9 @@
 /hook_status.json
 /disabled
 /nested_snapshots_enabled
+# Written by apply.sh when a module's assumptions no longer fit the installed
+# middlewared (see tools/compat.py). Evidence for the alert; not source.
+/incompatible.json
 
 # Python
 __pycache__/
@@ -13,3 +16,4 @@ __pycache__/
 .ruff_cache/
 .venv/
 venv/
+/update_alerts_disabled

CHANGELOG.md

@@ -1,5 +1,641 @@
 # Changelog
 
+Work lands under **Unreleased** and stays there until a release promotes it. That
+is deliberate: see [Releasing](docs/releasing.md). Twelve releases were cut on
+2026-07-13, several of them fixing the release before — and with the update alert
+live, every one of those interrupts every user. An alert people learn to ignore is
+worse than no alert, because one day it carries a security fix.
+
+## Unreleased
+### Changed
+
+- **CI's python matrix is green on the self-hosted Gitea runner again.** The real
+  failure was that the Gitea runner image executes jobs as root, and the two
+  unreadable-sidecar tests build their scenario with `chmod(0)` — which cannot make
+  a file unreadable for root (`CAP_DAC_OVERRIDE`). Those two tests now skip as root
+  with that reason; GitHub's non-root runner still exercises them. The matrix also
+  moved to uv-managed interpreters (one toolchain across both runners) and ruff is
+  pinned to 0.16.1 so an upstream ruff release can't turn `main` red without a code
+  change.
+- **README badges point at the public GitHub mirror** (workflow status and
+  releases) instead of the private forge. The release badge had also been reading
+  the stale Gitea v0.6.1 release instead of the current v0.7.0 on GitHub.
+
+- **`master` is now labelled `27-dev`, because it is not the next release.** iX
+  branches each major onto its own `release/` line and master rolls straight on to the
+  one after — on 2026-07-14 every recent commit on master targeted `27.0.0-BETA.1`
+  while 26 was still in beta. So a **BROKEN** master row, rendered as
+  "master _(unreleased)_", read as *"the version you are about to install is broken"*
+  when the breakage was a major release away on a line nobody can download. In a table
+  whose entire job is helping somebody decide whether to trust this with their backups,
+  that is a false alarm in the worst possible place. The label is derived from the
+  newest major in the matrix plus one, so it rolls over to `28-dev` by itself once 27
+  branches.
+
+  For the record, the breakage is `NAS-141498` (2026-06-24), "Convert cloud_backup
+  plugin to the typesafe pattern": it re-signatures `restic_backup` and
+  `get_restic_config`, splitting `entry`/`credentials` out of the `cloud_backup` dict.
+  It is deliberately not being chased while the 27 line is still churning.
+
+### Fixed
+
+- **The next maintenance release was never checked, and it is the one that reaches
+  users.** Shipped versions were discovered from `TS-*` tags and unreleased ones from
+  `release/*` branches carrying `-BETA`/`-RC`. A branched-but-untagged *maintenance*
+  release is neither: `release/25.10.5` has no tag, and its line has already shipped,
+  so the "a prerelease of a shipped line is history" filter discarded it. It was
+  invisible — and it is precisely what a 25.10.4 box gets on its next update. A break
+  there would have reached real users before the daily check ever looked at it, on the
+  only line anybody is actually running.
+
+  A plain `release/X.Y.Z` branch is now checked when its line **has** shipped and it
+  sorts **newer** than that line's newest tag. Both things that must stay out fall out
+  of the same rule: `release/24.10-RC.2` sorts older than `TS-24.10.2.4` (history, not
+  a warning), and iX's typo branch `release/25.20.2.2` is on a line that has no tag at
+  all, so it is not a release line. This immediately surfaced two refs that had never
+  been checked — `release/25.10.5` and `release/24.10.2.5` — both of which pass.
+
+  `is_unreleased()` now keys off where a ref came from (branch = not yet shipped)
+  rather than looking for `-BETA`/`-RC` in its name. Otherwise `release/25.10.5` would
+  count as shipped and a break in it would fail the build as a live outage — on a
+  version nobody is running yet.
+
+- **An unchanged fingerprint froze the bug report's body, not just its comments.** Two
+  questions were sharing one answer. *Have the findings changed?* gates **comments** —
+  they notify, and a daily "still broken, same as yesterday" is what teaches everyone
+  to ignore the one that finally matters. *Is the body still true?* gates the **body** —
+  and editing an issue body notifies nobody on either forge, so keeping it honest is
+  free. Conflated, the report could never be corrected while the findings held steady,
+  and the fingerprint deliberately ignores everything that moves on its own — healthy
+  rows, the hardware-verified column, point releases, and how a row is labelled. The
+  `master` → `27-dev` relabel above would have reached the README and never the issue
+  anybody actually opens. The body is now rewritten whenever it is out of date (after
+  normalising line endings, so a forge round-tripping `\r\n` does not cause a rewrite
+  every run) and comments remain strictly a changelog of real changes.
+
+- **A change to the publisher did not re-run the check.** `compat.yml`'s `push:` paths
+  listed `tools/compat.py` but not `tools/compat_publish.py` — so the very commit that
+  taught the bot to refresh a stale report body triggered no run, and the report stayed
+  stale until the next scheduled one. A fix nobody runs is a fix nobody has.
+
+- **The compatibility bot filed a new duplicate bug report on every Gitea run.**
+  `find_issue()` skipped pull requests by testing for the *presence* of the
+  `pull_request` key. GitHub omits that key on a plain issue; Gitea sends it as
+  `null`. So on Gitea every issue was discarded as a PR, the lookup always came back
+  empty, and the bot took the "nothing filed yet" branch and opened a fresh report
+  each run — **nine copies on the canonical forge**, four of them filed *after* the
+  commit that was meant to stop precisely this. The mirror was fine, which is why it
+  went unnoticed: GitHub's payload shape is the one the filter was written against.
+
+  It is the same failure the anti-spam fix was written to prevent, moved from
+  comments to issues, and it survived because `find_issue` was the only function in
+  `compat_publish.py` with no test. It now has one, per forge, and the daily cron —
+  which had not yet run once — no longer accumulates a report a day.
+
+  The issue list is also requested with **both** paging parameters (`per_page` for
+  GitHub, `limit` for Gitea). Each forge ignores the other's, and Gitea's default page
+  is 30, so the lookup would have started missing the report again once the pile it
+  was creating grew past one page.
+
+## v0.7.0 — 2026-07-14
+### Added
+
+- **TrueNAS 26 support, verified on a real TrueNAS 26 install.** 26 deletes
+  `plugins/zfs_/` outright, taking the private `zfs.dataset.query`,
+  `zfs.snapshot.query` and `zfs.snapshot.delete` with it. Every one of those was on
+  the nested module's critical path, so nested snapshots were **BROKEN** on 26 and
+  `apply.sh` correctly refused to apply the module there.
+
+  Snapshot **deletion** now resolves its namespace at runtime — `pool.snapshot` on
+  25.10 and 26, `zfs.snapshot` on 24.10 and 25.04, because no single namespace spans
+  every supported release. `tools/compat.py` checks the same list the runtime uses,
+  so what CI verifies and what runs cannot drift apart.
+
+  Hardware-verified on TrueNAS 26.0.0-BETA.1: a 274-snapshot recursive backup of a
+  292-dataset pool, then a **byte-identical restore of a four-level-deep child
+  dataset**.
+
+### Fixed
+
+- **Enumeration no longer trusts middleware's dataset and snapshot queries — they
+  are filtered.** This is the important one, and it is the bug that a test VM caught
+  and no amount of source analysis ever could have.
+
+  The obvious port of the deleted private `zfs.dataset.query` was the public
+  `pool.dataset.query`. It exists, it is documented, it is covered by iX's
+  deprecation policy — and it is **not a like-for-like replacement**. It applies a
+  *visibility policy*: it hides the datasets TrueNAS considers its own — `ix-apps/*`,
+  `.system/*`, `.ix-virt/*`. On a real pool that is **84 of 270 datasets**, and
+  `ix-apps` holds **live application data**.
+
+  Staging from that view would have silently omitted every one of them. Worse,
+  `plan_staging()` would never have seen them, so they would not have appeared in its
+  `skipped` list either — no warning, no failure, just a green backup quietly missing
+  data. That is precisely the failure this module exists to prevent. The snapshot
+  query lies the same way (205 of 274), so the sweep would have orphaned one snapshot
+  per hidden dataset, on every run, forever.
+
+  The module now **reads the truth from ZFS and makes changes through middleware**:
+  enumeration is `zfs list`, which no policy can filter and which behaves identically
+  on every release; mutation stays a middleware call, so TrueNAS's own bookkeeping
+  stays consistent. A failing `zfs list` raises rather than returning an empty list —
+  "no datasets" and "the command broke" must never look the same.
+
+  **No shipped release is affected.** v0.6.1 and earlier call the *private*
+  `zfs.dataset.query`, which returns all 270 datasets. The bug existed only in the
+  unreleased TrueNAS 26 port.
+
+- **The patch now owns the snapshot sweep even when it does not stage anything.**
+  Stock decides whether to take a *recursive* snapshot by its own rule, and on
+  TrueNAS 26 that rule stopped being ours.
+
+  Up to 25.10, stock's `create_snapshot` called `get_dataset_recursive()` — the same
+  function this module vendors — so "stock went recursive" and "we have something to
+  stage" were the *same question*, and stock's non-recursive delete was correct for
+  everything the patch declined to stage. TrueNAS 26 uses `filesystem.statfs`:
+  `recursive = (path == the dataset's mountpoint)`. The two rules now disagree for a
+  dataset whose only descendants are **ZVOLs** or **legacy/none-mountpoint** datasets
+  — stock snapshots it recursively, while the patch sees nothing to stage.
+
+  The patch then handed the snapshot back to stock, which destroys the parent only.
+  With no staging tree there was no sidecar, and the garbage collector only ever ran
+  from the staging path — so nothing on the box would ever have found the children.
+  Reproduced on the test VM: one orphaned snapshot per zvol, on every run, forever,
+  with the backup reporting success. Ownership of the sweep is no longer conditional
+  on staging.
+
+- **The runtime resolved a *namespace*; the checker verified a *method*.** Those are
+  different questions, and the gap is a false "ok". `get_service()` only proves a
+  namespace is registered — it says nothing about whether `delete` still exists on it.
+  So if iX guts the method while keeping the service (they have already done exactly
+  that to `pool.snapshot.do_update` on master), `tools/compat.py` would fall through
+  to `zfs.snapshot`, report the box healthy, and let the patch apply — while the
+  runtime picked `pool.snapshot` and failed *every* delete, orphaning the whole tree.
+  Both sides now ask the same question, and a test binds the two lists together.
+
+- `query_filesystems()` **dropped malformed `zfs list` rows silently** — the last
+  remaining silent-omission path, and a direct contradiction of this module's cardinal
+  rule. It raises now. A missing `zfs` binary raised `FileNotFoundError` rather than
+  `ZfsError`; also fixed.
+
+- The snapshot retry loop **discarded the delete error** and reported every survivor
+  as "(still busy?)" — naming the one cause that is benign and self-healing, and
+  hiding the ones that are permanent. It keeps and reports the real error.
+
+- The staging-failure handler could **lose the original exception** if its own cleanup
+  sweep raised. An error handler must not be able to lose the error.
+
+- **A snapshot delete that returns cleanly is not proof that anything was deleted.**
+  The recursive sweep's fast path took the call's word for it and returned "no
+  survivors" — so `cleanup_task` read that as a clean sweep and removed the sidecar,
+  the only record the tree ever existed. Roughly 250 snapshots would have been orphaned
+  on every run, with nothing left able to find them, and the backup reporting success.
+
+  This is not a hypothetical about a well-behaved API: iX has already gutted
+  `pool.snapshot.do_update` on master into a no-op whose body is commented out and
+  which returns `None`. A source check still sees the `def`; a runtime check still sees
+  a callable method. Only asking ZFS can tell. The sweep now confirms against ZFS, and
+  where it *cannot* confirm it keeps owning the tree rather than claiming success — a
+  false survivor self-heals on the next run, a lost record never does.
+
+- `_write_sidecar` **swallowed `OSError`**. The sidecar is the only thing that survives
+  a middlewared restart; failing to write it is not fatal, but it must never be
+  invisible. `_read_sidecar` had the mirror bug — it conflated "there is no sidecar"
+  with "I could not read the sidecar", and `cleanup_task` then took the empty branch
+  and **unlinked the only record** of a tree it had failed to read.
+
+- **A dataset from another tree, mounted inside the backup path, was omitted
+  silently.** The staging plan scopes by dataset *name*, which is correct — a dataset
+  with no mountpoint cannot be scoped by path at all. But ZFS lets any dataset mount
+  anywhere, so one from an unrelated tree can sit inside the path:
+
+      Tank/photos   mountpoint=/mnt/Tap/apps/photos
+
+  It holds data inside the backed-up path, and `zfs snapshot -r Tap@…` does **not**
+  cover it: recursion follows the dataset tree, not the directory tree. So there is no
+  snapshot of it to stage, and no way to capture it consistently with the rest. It fell
+  out of the name filter and vanished — not staged, not in `skipped`, no error, backup
+  green. Stock has the same blind spot, but stock also refuses the nested config
+  outright; this patch is what relaxes that guard, so the hole is this patch's to close.
+  It now refuses, and names the offending datasets.
+
+## v0.6.1 — 2026-07-13
+### Fixed
+
+- **A reboot mid-backup orphaned the entire snapshot tree, permanently.** The sidecar
+  is the record of which snapshots a run pinned — and it lives in `/run`, which is
+  **tmpfs**. A reboot (or a crash) between taking the recursive snapshot and cleaning
+  it up destroyed that record, leaving one snapshot per descendant dataset — **250+ on
+  a real pool** — with nothing left pointing at them. Nothing would ever have found
+  them again.
+
+  `gc_stale_snapshots()` is the backstop: it identifies leftovers **by name**, so it
+  works when the record is gone. It runs at the start of every backup, after the
+  sidecar reclaim — the recorded path stays authoritative, and the collector only ever
+  mops up what the record lost.
+
+  Because it deletes data on a *name match* — a weaker claim than a recorded fact — the
+  selection is a **pure function** with the harshest tests in the suite. A snapshot is
+  collected only if **all** of these hold:
+
+  | | |
+  | --- | --- |
+  | name is exactly `<dataset>@<task>-<YYYYMMDDHHMMSS>` | so `cloud_backup-5` never matches `cloud_backup-50`, an `auto-*` periodic snapshot, or anything a human made |
+  | it is not the current run's | parent *and* children are excluded |
+  | **nothing is mounted from it** | an in-flight run pins its own snapshots — this, not the age guard, is what protects a concurrent backup |
+  | it is **over an hour old** | covers the seconds-long window where a live run has snapshotted but not yet mounted |
+
+  Verified against the real pool: of **4,728** snapshots — including **2,341** periodic
+  ones — it selects exactly the orphans of the task being run, and nothing else.
+
+## v0.6.0 — 2026-07-13
+### Added
+
+- **`release.sh` — a two-stage release process, and a barrier that enforces it.**
+  A stable `vX.Y.Z` tag is now only publishable if a `vX.Y.Z-rcN` tag points at the
+  **same commit**, and the release job re-runs the entire suite against that tagged
+  commit before publishing. Candidates are invisible to users — `update.sh` and the
+  update alert both take the newest plain `vX.Y.Z` tag — so debugging happens across
+  rc1, rc2, rc3 at nobody's expense, instead of across v0.5.0, v0.5.1, v0.5.2 at
+  everybody's.
+
+      bash release.sh 0.6.0 --rc        # candidate. Invisible to users.
+      bash release.sh 0.6.0 --promote   # stable. Refused unless an rc passed HERE.
+
+  The rule is enforced in `tools/release_gate.py`, which `release.sh` runs locally
+  (so you fail in 200 ms) and `.github/workflows/release.yml` runs again where it
+  cannot be bypassed (so failing locally is not optional). "The candidate passed,
+  then I pushed one more little fix" is refused by name — that is precisely how
+  v0.5.1 happened.
+
+- **TrueNAS compatibility is now checked, not hoped for.**
+  [`tools/compat.py`](tools/compat.py) is a written-down record of everything each
+  module assumes about middlewared, checked in two places:
+
+  - **CI, daily** — against iXsystems' source at every release line *including
+    `master` and the current BETA/RC*. When an unreleased TrueNAS breaks the patch
+    it files a bug report automatically, so there is time to fix it before that
+    version reaches anyone. It also refreshes the README's support matrix, so the
+    table cannot quietly become a false promise.
+  - **`apply.sh`, at every boot** — against the middleware actually installed on the
+    box. **A module whose assumptions no longer hold is not applied.** Stock TrueNAS
+    without a feature beats TrueNAS with a broken backup.
+
+  It immediately found two real breaks: TrueNAS 26 (below), and a nested-snapshot bug
+  that had been shipping for two releases (below).
+
+- **The compatibility check now covers the middlewared methods the patch _calls_,**
+  not only the symbols it wraps — and that gap was hiding a catastrophe.
+
+  TrueNAS 26 **deletes `plugins/zfs_/dataset.py` and `plugins/zfs_/snapshot.py`
+  outright**, taking `zfs.dataset.query`, `zfs.snapshot.query` and
+  `zfs.snapshot.delete` with them (26 uses `filesystem.statfs` and `zfs.resource.*`).
+  Nothing about the five `cloud_backup` files reveals that, so every other check went
+  green. The patch would have applied perfectly and then **failed on the first
+  backup** — or, far worse, snapshotted successfully and failed to *delete*,
+  orphaning one snapshot per descendant dataset (**250 on a real pool**) on every
+  single run, forever.
+
+  This is now an assumption class of its own, so a method disappearing is a BROKEN
+  verdict rather than a silent time bomb.
+
+- **Groundwork for TrueNAS 26** (async→sync and the deleted helper — see below).
+  **26 is still reported BROKEN and the nested module will not apply there**, because
+  the ZFS API rewrite above is not yet ported. Porting it needs a real 26 box to
+  verify against, and shipping a port nobody has run is exactly the failure this
+  project exists to avoid. On 26, TrueNAS is left stock: B2/S3 keeps working, nested
+  datasets are simply not covered.
+
+### Fixed
+
+- **A few snapshots leaked on every nested run, forever.** Found on real hardware, in
+  the one place it could be: a 256-snapshot backup of `/mnt/Tap` swept 253 cleanly and
+  left **3 behind** with `dataset is busy`.
+
+  The cause is ZFS's own automount. Reading anything under
+  `<dataset>/.zfs/snapshot/<snap>/` makes ZFS **automount that snapshot**, and it stays
+  mounted for `zfs_expire_snapshot` seconds (**300** by default) after the last access.
+  `teardown()` unmounts *our* bind mounts — but not the automount underneath — so
+  `zfs destroy` refuses for exactly the datasets restic read most recently. Then
+  `cleanup_task()` removed the sidecar anyway, destroying the only record that those
+  snapshots existed. Nothing would ever have reclaimed them.
+
+  Three changes, and the third is the one that makes it safe rather than merely
+  unlikely:
+  - `release_snapdirs()` unmounts ZFS's own `.zfs/snapshot` automounts (deepest first)
+    before deleting, so the snapshots are not busy in the first place.
+  - `delete_snapshot_tree()` **retries** the transient busy, and **returns the
+    snapshots it could not delete** instead of swallowing them.
+  - **The sidecar is now removed only on a confirmed-clean sweep** — including on the
+    staging-failure path, which used to remove it *before* the caller swept. The
+    asymmetry is deliberate: a sidecar left behind when the tree is already gone costs
+    one no-op delete on the next run, while a sidecar removed while the tree still
+    exists is unrecoverable. Survivors are reclaimed by the next run.
+
+  **Expect the occasional straggler, and expect it to clean itself up.** On a
+  256-snapshot tree this reliably sweeps ~255 immediately and may leave **one**: it is
+  whatever restic read last, so its 300-second window has barely opened. That one is
+  logged, its sidecar is kept, and the next run reclaims it before doing anything else.
+  The leak is bounded at a single cycle rather than growing without limit — which is
+  the property that actually matters. Blocking a backup job for five minutes to chase
+  the last snapshot would be a worse trade, so it is not made.
+
+- **Installing the patch permanently blocked updating it.** `install.sh` does
+  `chmod +x update.sh`, and git recorded `update.sh` as `100644` — so the chmod was a
+  *tracked modification*, and `update.sh` refuses to run over a dirty tree. Install
+  once and you could never update again; the error even told you to run
+  `git checkout -- .`, which just undoes the exec bit so the next install can re-dirty
+  it. A real box sat on an old version for exactly this reason.
+
+  Fixed on both sides: the scripts `install.sh` chmods are now executable in git (so
+  the chmod is a no-op), and `update.sh`'s dirty check now looks at **content**, not
+  file mode — `git diff --numstat` reports `0 0` for a mode-only change. A test
+  asserts every script in `install.sh`'s chmod loop is already `100755` in git.
+
+- **Nested snapshots were broken on TrueNAS 24.10 and 25.04, and had been all
+  along.** `SYNC_BLOCK`'s wrapper spelled out the stock signature and forwarded five
+  arguments — but those releases declare `restic_backup(middleware, job,
+  cloud_backup, dry_run)`; `rate_limit` only arrived in 25.10. Every nested backup on
+  24.10/25.04 raised `TypeError: restic_backup() takes 4 positional arguments but 5
+  were given`. The wrapper now takes `*args, **kwargs` and forwards whatever it is
+  handed, so a trailing parameter appearing or disappearing is a non-event.
+
+  Found by the new compatibility check, not by a user — which is the whole argument
+  for having it. The check it replaced only asked whether the parameter *names* still
+  appeared somewhere in the signature, so it happily passed a call that could never
+  work.
+
+- **The nested module is now one synchronous implementation behind two thin
+  wrappers.** TrueNAS 26 rewrites `cloud_backup` from async to **synchronous** and
+  separately **deletes `get_dataset_recursive()`**, which an injected block called out
+  of the host module's namespace. Either alone is a broken backup found at restore
+  time: an `async def` wrapper hands `sync.py` a coroutine where it unpacks a tuple,
+  and the vanished helper is a straight `NameError`.
+
+  The module now talks to middlewared through `call_sync`, and `apply.sh` reads which
+  flavour the installed middleware declares and injects the matching wrapper —
+  TrueNAS ≤ 25.10 reaches it via `await middleware.run_in_thread(...)`; a synchronous
+  TrueNAS, already in a worker thread, calls it directly. The logic that owns the
+  snapshots, the bind mounts and the failure modes exists **once**; an async twin
+  would mean every future fix had to land twice, and the one that got missed would be
+  the one that eats a backup. A middleware whose three wrapped functions **disagree**
+  about async-ness is refused outright rather than guessed at, and
+  `get_dataset_recursive` is carried as our own copy — removing the dependency on both
+  versions instead of asserting it.
+
+- **The patch no longer reaches into CloudSync tasks it has no business touching.**
+  `create_snapshot` is module-global in `plugins/cloud/snapshot.py` and is imported by
+  **`cloud_sync.py` as well as `cloud_backup/sync.py`** — so the wrapper sat in the
+  path of every rclone/Storj **CloudSync** task with `snapshot=true`, and issued a
+  `zfs.dataset.query` before deciding it had nothing to do. That added a brand-new
+  failure mode to jobs that worked fine before this patch was installed, and worse: a
+  CloudSync task that ever *did* get staged would **never be torn down**, because the
+  teardown is wired into `cloud_backup`'s `restic_backup` and `CRUD_BLOCK`
+  deliberately leaves CloudSync's nesting guard intact — the bind mounts would pin the
+  ZFS snapshot forever. The staging path now bails out immediately unless the snapshot
+  is named `cloud_backup-*`, before any middleware call.
+
+- **Teardown warnings are no longer silently swallowed on TrueNAS ≤ 25.10.** The async
+  wrapper's `finally` dropped the `logger=` kwarg that the sync one passes, so a
+  cleanup that failed to unmount a bind mount *or* to delete a snapshot tree logged
+  **nothing at all** — on the only platform anyone actually runs. `run_in_thread`
+  forwards `**kwargs` via `functools.partial`; it was a regression, not a limitation.
+
+- **`do_delete` is recognised as `delete`.** TrueNAS 24.10 and 25.04 declare
+  `do_delete` (the `CRUDService` convention); 25.10 renamed it to `delete`. Both
+  answer to `zfs.snapshot.delete`. Accepting only the literal name reported both older
+  releases as BROKEN — a false verdict that would have switched nested snapshots off
+  on boxes where they work perfectly.
+
+- **An incompatible TrueNAS no longer sets the permanent kill switch.** `apply.sh`
+  reused a "nothing left to do" exit that touches `disabled`, which suppresses
+  patching on every future boot and is cleared only by `install.sh` — never by
+  `update.sh`. On TrueNAS 26 (providers-compatible, nested opt-out by default) that
+  branch would have fired, and the very release that fixed 26 could not have
+  re-enabled itself: the user would run `bash update.sh`, exactly as the update alert
+  tells them to, and the patch would stay dead with their B2 backups off.
+  Incompatibility now means "apply nothing this boot, try again next boot".
+  Retirement and incompatibility are opposite situations and no longer share an exit.
+
+- **The compatibility check itself could be fooled**, in ways that each had teeth: a
+  reordered, keyword-only, or newly-required parameter now reads as broken (the patch
+  calls these positionally); a **re-exported or conditionally-defined** symbol reads
+  as *unknown* rather than broken, so an innocent upstream refactor cannot make a
+  working module decline to apply; an **unreadable** source (rate limit, DNS, timeout)
+  is *unknown* rather than "iXsystems deleted this file", so a network blip cannot
+  file a bug report, fail CI, and repaint the published support matrix; and `native`
+  no longer masks `BROKEN`, which used to render a TrueNAS that both reworded the
+  nesting guard *and* reshaped the functions as good news.
+
+- **`compat.py --tree` no longer reads the patch's own code as native support.**
+  `B2_BLOCK` writes `B2RcloneRemote.restic = True` into `b2.py` — exactly the string
+  the providers native-probe looks for — so the one command the docs recommend for
+  checking a live box said "retire the providers module" on every *patched* machine.
+  It now reads only the part of the file iXsystems wrote.
+
+- **`release.sh --promote` could never succeed.** It refused to run if the stable tag
+  existed, and the gate refused if it did not — mutually exclusive, so the only way to
+  cut a stable release was to hand-tag and bypass every gate this work exists to
+  enforce. The gate now resolves the tag's commit if it exists and `HEAD` otherwise.
+  The tests hid it by always tagging first.
+
+### Changed
+
+- **The minimum supported TrueNAS is stated, and enforced: 24.10.** TrueCloud Backup
+  does not exist before it — `plugins/cloud_backup/` is simply absent — so the patch
+  had nothing to attach to and would have done nothing at all, silently, while the
+  user believed their backups were configured. `install.sh` now reads
+  `system.version` and refuses, naming the reason. A version it cannot *parse* is a
+  warning, not a refusal: declining to install over a string we failed to read would
+  be a worse failure than the one being prevented.
+
+- **A stable release may not leave work stranded under `## Unreleased`.** Either it
+  is finished and belongs in the release, or the release is premature. Candidates
+  are exempt: an rc may legitimately have work queued behind it.
+
+- **`release.sh` refuses to run on an installed box.** The whole repo is cloned onto
+  every box, so this file is there too; `update.sh` pins the checkout to a tag in
+  detached HEAD, and `release.sh` now recognises that and says so, rather than
+  emitting a confusing branch error.
+
+- **Gitea (`git.onetick.ninja/flan/truenas-truecloud-patch`) is now canonical**, with
+  GitHub as a mirror. Both forges run the same workflows and publish the same
+  releases. The update alert now **derives the changelog URL from the `origin`
+  remote** instead of hard-coding GitHub — which matters more than it sounds: when
+  the changelog cannot be read, the alert deliberately fires *anyway* rather than risk
+  hiding a security fix, so a stale URL would not have disabled the alert, it would
+  have made it nag on every release, including documentation-only ones.
+
+### Security
+
+- **Workflow expressions are no longer interpolated into shell.**
+  `echo "${{ steps.report.outputs.body }}"` pasted the compatibility report into the
+  script text, and the report is full of backticks — bash ran `create-snapshot`,
+  `def` and `async` as commands. Since that report is built from iXsystems' source,
+  anything landing in their tree would have executed on the runner. `inputs.tag` on
+  `workflow_dispatch` had the same shape, and that one is attacker-chosen. Data now
+  moves through files and scalars through `env:`; a test enforces it across every
+  workflow.
+
+### Internal
+
+- Static-analysis annotations in `patch/alert_source.py` (`# noqa` placement). No
+  runtime change.
+
+## v0.5.1 — 2026-07-13
+
+### Fixed
+
+- **The update alert could have broken middlewared at startup.** middlewared's
+  `alert.load()` imports every file in `alert/source/` with **no try/except**, and
+  it runs during setup — so a module that raises on import takes middlewared down
+  with it. `apply.sh` now **compiles the substituted alert source and refuses to
+  write it** if it does not parse. An uninstalled alert is a missing convenience;
+  a broken one is a broken box.
+
+- **`@PATCH_DIR@` is substituted with `repr()`**, so a repository path containing
+  a quote or a backslash produces a valid Python literal instead of a syntax error
+  in the installed module.
+
+- **The alert source no longer mutates `sys.path`.** It loaded
+  `tools/release_notes.py` via `sys.path.insert(0, …)`, which shadows the stdlib
+  for that interpreter — and `ThreadedAlertSource` runs in middlewared's thread
+  pool, so mutating `sys.path` is a race. It now loads the module by file path with
+  `importlib`.
+
+### Notes
+
+Timing, for the record: `process_alerts` is `@periodic(60)` and
+`alert_source_last_run` is in-memory, so the check runs **within 60 seconds of any
+middlewared restart** (which this patch performs at every boot) and otherwise
+**within 24 hours** of a release.
+
+## v0.5.0 — 2026-07-13
+
+### Added
+
+- **A TrueNAS alert when an update is available** — the bell in the UI, not a log
+  line nobody reads. On by default, checked once a day.
+  `install.sh --no-update-alerts` turns it off.
+
+  **It does not nag.** A release whose CHANGELOG contains only a `### Docs`
+  section changed no code and raises nothing. Anything else raises INFO; a
+  `### Security` section raises WARNING. The CHANGELOG's own section headings are
+  the signal, and a security fix anywhere in the range escalates the whole span —
+  so a docs-only release sitting on top of a security fix still reports as
+  security, rather than hiding it.
+
+  **Why an AlertSource and not `midclt`:** TrueNAS cannot raise an alert from the
+  CLI. `midclt` exposes only `alert.dismiss`, `alert.list`, `alert.list_categories`,
+  `alert.list_policies` and `alert.restore` — alert *creation* is internal to
+  middlewared, and none of its ~60 one-shot classes is generic enough to reuse. So
+  registering an `AlertSource` is the only way, and it is also the least invasive
+  thing this patch does: it **adds one file and modifies none**, where the
+  providers and nested modules both append code to stock middleware files. It is
+  the native mechanism, and TrueNAS polls it itself — no cron, no systemd timer.
+
+  - Fail-safe: every error path returns `None`; it cannot take middlewared down.
+  - Read-only: `git ls-remote` plus an HTTPS fetch of the CHANGELOG. It never
+    writes to `.git`, so it cannot leave root-owned objects behind the way a
+    `git fetch` from middlewared (running as root) would.
+  - Removed by `uninstall.sh`.
+  - It only *tells* you; it never updates anything.
+
+## v0.4.2 — 2026-07-13
+
+### Docs
+
+- **The Updating section never said how to *get* `update.sh`.** It ships inside the
+  patch, so a clone older than v0.4.0 doesn't have it — the docs told you to run a
+  script you didn't have. There is now an explicit bootstrap step (`git pull &&
+  bash install.sh`, once), including the fix for the *"insufficient permission for
+  adding an object to repository database"* failure that past `sudo git pull`s
+  cause.
+
+- **`After a TrueNAS update` rewritten.** It didn't explain that the patch
+  re-applies itself at every boot (so you never reinstall), and it didn't say what
+  each failure actually costs you. "Fail-safe" means *the box stays up* — not that
+  your backups keep running. A `[FAIL] providers` is a **broken backup**, and the
+  docs now say so rather than implying everything degrades gracefully.
+
+- Added a repo map. `patch/mw_patch.py` and `tools/release_notes.py` were
+  documented nowhere.
+
+- `Development` told you to run `ruff check patch tests`, which misses `tools/`.
+
+- Every command and file path in the README is now verified to exist and run.
+
+## v0.4.1 — 2026-07-13
+
+### Fixed
+
+- **`update.sh` would have picked a release candidate as "the newest release".**
+  Git's version sort ranks `v0.5.0-rc1` *above* `v0.5.0` (verified), and the
+  release workflow deliberately supports rc/beta tags — so an RC would have been
+  installed as though it were the latest stable. Tag selection is now filtered to
+  plain `vX.Y.Z`.
+
+- **`update.sh` would have died mid-update on an untracked file.** The dirty-tree
+  guard uses `--untracked-files=no`, so an untracked file that the *target* tracks
+  slipped past it — and `git checkout` then aborts. Under `set -e` the script died
+  with a raw git error, *after* recording the rollback point. This is exactly what
+  blocked a pull on a real box (a hand-copied `patch/wait_restart.sh`). It now
+  detects the collision up front and names the files. Gitignored files are
+  correctly *not* treated as blockers — git overwrites those silently.
+
+  Special case: if `update.sh` *itself* is the blocker, you hand-copied it in to
+  bootstrap — and "delete `update.sh`, then re-run `update.sh`" is impossible. It
+  now says so and prints the git commands that bootstrap it properly.
+
+- **`--rollback` skipped that check entirely**, so it would have hit the identical
+  failure. The check is now a shared function used by both paths, and rollback also
+  validates that the recorded revision still exists (history can be rewritten).
+
+- `install.sh`'s `chmod` aborted under `set -e` if any listed file was missing. The
+  file set changes between versions, so `update.sh --rollback` to an older revision
+  must not be killed by a filename this version happens to know about.
+
+- `--to` with no value was silently ignored and fell back to the default target.
+
+## v0.4.0 — 2026-07-13
+
+### Added
+
+- **`update.sh`** — fetch a newer release and apply it, preserving your
+  nested-snapshot opt-in setting.
+
+  ```bash
+  bash update.sh              # to the newest release, with a confirmation
+  bash update.sh --check      # show what would happen; change nothing
+  bash update.sh --rollback   # undo the last update
+  ```
+
+  **Run it by hand. Never from cron or a systemd timer.** This patch injects
+  Python into middlewared and re-applies itself at every boot, so an unattended
+  pull would let any bad upstream commit reach your box with no human in the loop
+  and take effect on the next reboot. v0.0.4 shipped exactly such a bug and took
+  every app on the box down. The manual step *is* the safety gate.
+
+  Design:
+
+  - **Defaults to the newest release tag, not `main`.** `main` can be mid-refactor;
+    a tag is the tested artifact. `--main` exists but says so loudly.
+  - Tags are ordered by **version**, not by date — date order silently downgrades
+    the box the first time a hotfix is tagged out of band (a v0.3.6 released after
+    v0.4.0 would sort as "newest").
+  - **Refuses to run over a dirty working tree** rather than merging across
+    hand-edited or scp'd files.
+  - Shows the commits you don't have and the target's release notes (read from the
+    *target's* CHANGELOG, via `tools/release_notes.py` — not a second copy of the
+    extractor), then asks before doing anything.
+  - **Records the previous revision before moving**, so `--rollback` works even if
+    `install.sh` dies halfway.
+  - Repairs `.git` ownership, which past `sudo git pull`s leave root-owned and
+    which then breaks every later non-root git command.
+
+- `update.sh` is covered by the version-drift check, so it cannot quietly go stale
+  the way `create_task.py.__version__` did.
+
 ## v0.3.5 — 2026-07-13
 
 ### Changed

NOTICE

@@ -0,0 +1,4 @@
+truenas-truecloud-patch
+
+Parts of this project were written with AI assistance (Claude); all of it is
+reviewed and tested before release.

README.md

@@ -1,406 +1,194 @@
 # truenas-truecloud-patch
 
-Extends TrueNAS SCALE's **TrueCloud Backup** feature to:
+[![CI](https://git.arch.fyi/flan/truenas-truecloud-patch/actions/workflows/ci.yml/badge.svg)](https://git.arch.fyi/flan/truenas-truecloud-patch/actions)
 
-- work with S3-compatible providers and native Backblaze B2, instead of Storj only;
-- take **consistent snapshots of datasets that have child datasets** — which is
-  every box running Apps (see [Nested-dataset snapshots](#nested-dataset-snapshots)).
+Extends TrueNAS SCALE's **TrueCloud Backup** to:
 
----
+- back up to **Backblaze B2 and any S3-compatible provider**, not just Storj;
+- snapshot **datasets that have child datasets** — which is every box running Apps.
 
-## Why this exists
+**Requires TrueNAS SCALE 24.10 or newer.** TrueCloud Backup does not exist before
+that, and `install.sh` will refuse.
 
-In 2026, Storj raised the price of their TrueNAS-integrated storage tier from
-**$5/month to $50/month** — a 10× increase. For many home lab and small-office
-users, the TrueCloud Backup feature became unaffordable overnight.
-
-TrueCloud Backup is the only native TrueNAS mechanism that provides:
-- Integrated ZFS snapshot support before each backup
-- Restic-based incremental deduplication
-- Scheduled tasks with progress and log tracking in the UI
-- Dataset lock integration
-
-Running restic manually is possible but loses all of the above.
-This patch restores access to the TrueCloud Backup feature for users who
-need a provider other than Storj, with storage they already pay for or that
-costs a fraction of the new Storj price.
-
----
-
-## Before you install
-
-This project is unofficial and not affiliated with iXsystems. A few things worth
-knowing:
-
-- It targets **internal middleware APIs** with no stability contract, so a
-  TrueNAS update can break it. Every patch is fail-safe: if it can't apply,
-  middlewared starts normally and the reason is logged to `apply.log`. Check the
-  log after an update.
-- If you file a TrueNAS bug report, **remove the patch first** and reproduce on a
-  stock system.
-- **Test your restores.** True of any backup, but it matters more here — see
-  [Verifying it works](#verifying-it-works).
-- Provided as-is, no warranty. See LICENSE.
-
-The patch is two independent modules — **providers** (B2/S3) and **nested**
-(snapshots on nested datasets) — and each retires on its own once TrueNAS ships
-that capability natively. See [Native support](#if-truenas-adds-native-support).
-
-## Development
-
-Parts of this project were written with AI assistance (Claude). All of it is
-reviewed and tested before release; the test suite and CI exist in large part to
-make that review meaningful. Bugs are mine.
-
-```bash
-pip install pytest ruff
-ruff check patch tests
-pytest tests
-```
-
-CI runs shellcheck, `bash -n`, ruff, and pytest on Python 3.11–3.13. The tests
-include a pass that `compile()`s the `*_BLOCK` strings in `patch/apply.sh` —
-those are Python source appended into live `middlewared` modules, so a syntax
-error there would break the box at boot.
-
----
-
-## What is actually patched
-
-**Nothing in TrueNAS's persistent database or configuration is modified**
-(other than the boot-hook entry itself). On every boot, `patch/apply.sh` runs
-as a PREINIT script. It mounts a writable
-[overlayfs](https://docs.kernel.org/filesystems/overlayfs.html) over the
-relevant directories in `/usr/` (upper layer in `/run` tmpfs), then patches
-`b2.py` and `restic.py` inside that overlay. The overlay is volatile — it
-exists only for the current boot — but the PREINIT script recreates it
-automatically on every subsequent boot. Nothing in `/usr/` is written to
-directly.
-
-PREINIT scripts are executed *by* middlewared, which by then has already
-imported the stock modules — so after patching, `apply.sh` schedules a single
-detached middlewared restart (transient systemd unit `truecloud-mw-restart`
-running `patch/wait_restart.sh`) that loads the patched modules once boot has
-*actually* settled: the script waits for the systemd boot job queue to drain
-and for the docker/apps state machine to reach a terminal state before
-restarting. Expect one middlewared restart shortly after every boot; the UI
-and API are briefly unavailable while it happens, and running services are
-not affected.
-
-| Module | What changes | Technique |
-|---|---|---|
-| **providers** | `B2RcloneRemote` gains `get_restic_config()` — skipped automatically if TrueNAS already provides one on the class. `restic.py` URL builder is fixed: strips the stray leading slash and converts the slash separator to a colon (`b2:bucket:path`), which is the format restic 0.16.x expects. URL wrapper is a no-op if the URL is already correctly formed. | File patch applied inside the overlayfs upper layer |
-| **providers** (UI) | The Angular bundle's `filterByProviders` binding is widened from `["STORJ_IX"]` to `["STORJ_IX","S3","B2"]` | In-place text replacement in the compiled JS chunk; original is backed up before patching |
-| **nested** (opt-in) | `_truecloud_nested.py` is installed into `plugins/cloud/`, and `plugins/cloud/{snapshot,crud}.py` + `plugins/cloud_backup/sync.py` are patched so `snapshot = true` works on a dataset that has child datasets. See [below](#nested-dataset-snapshots). | New module + file patches inside the overlayfs upper layer |
-
-All changes are **fail-safe**: if a patch cannot be applied (e.g. TrueNAS
-restructured the relevant code), middlewared starts normally, the affected module
-is simply inactive, and the reason is logged to `apply.log` in your repo root. The
-two modules are independent — one failing or going native does not disable the
-other.
-
-## Nested-dataset snapshots
-
-**Opt-in, off by default.** It changes how backups read their source data, so it
-is never enabled implicitly:
-
-```bash
-bash install.sh --enable-nested-snapshots
-bash install.sh --disable-nested-snapshots
-```
-
-With neither flag `install.sh` leaves the setting alone, so `git pull && bash
-install.sh` won't flip it. The providers module is unaffected either way.
-
-Validated end to end on a live 252-dataset pool: an unattended scheduled backup
-of `/mnt/Tap` built a 173-mount staging tree, completed in **18m14s**, and left
-**zero** orphaned snapshots and **zero** stale mounts behind. The same backup
-previously stalled at 74% for over 12 hours reading live files.
-
-Still: verify your own first run actually contains child-dataset data before you
-rely on it — see [Verifying it works](#verifying-it-works). That advice is not
-boilerplate; it is the specific thing this feature exists to make true.
-
-TrueCloud Backup's **Take Snapshot** option makes restic read from a frozen ZFS
-snapshot instead of live files. Without it the backup reads data *while apps are
-writing to it* — databases get captured mid-write, and an app that rewrites its
-files continuously can stall a backup indefinitely as restic chases a moving
-target.
-
-Stock TrueNAS refuses to enable it on most real-world paths:
-
-```
-[EINVAL] cloud_backup_update.snapshot:
-  This option is only available for datasets that have no further nesting
-```
-
-That rules out **any pool running Apps** — every app is its own dataset, usually
-with `config`/`pgdata` children of its own. On a typical box that is 100+ nested
-datasets, so the feature is effectively unusable exactly where it matters most.
-
-### Why stock refuses
-
-The guard is **correct**. `plugins/cloud/snapshot.py` already takes a *recursive*
-ZFS snapshot — but it then points restic at the **parent** dataset's
-`.zfs/snapshot/<snap>/` directory, and ZFS does not expose child datasets
-through a parent's snapshot directory:
-
-```
-/mnt/Tap/.zfs/snapshot/<snap>/apps/                 ->  0 entries  (children invisible)
-/mnt/Tap/apps/lidarr/config/.zfs/snapshot/<snap>/   ->  the real data
-```
-
-So if you just remove the validation, restic walks a near-empty tree, reports
-SUCCESS, and uploads almost nothing — a green backup job protecting no data. iX
-gate the config rather than ship a backup that lies about succeeding.
-
-That is worth spelling out, because deleting those four lines in
-`plugins/cloud/crud.py` is the obvious "fix" and it is the wrong one. The guard
-is load-bearing: it has to be *replaced* with a working traversal, not removed.
-
-### What this patch does instead
-
-After the (already recursive) snapshot is taken, every descendant dataset's own
-`.zfs/snapshot/<snap>` is bind-mounted into a **staging tree** that mirrors the
-original layout, and restic is pointed at the staging root — a complete,
-consistent, point-in-time view of the whole subtree. Only then is the guard
-relaxed.
-
-Safety properties, in order of importance:
-
-- **Staging failure is loud.** If any descendant cannot be staged, the backup
-  *fails*. A silently-incomplete backup is precisely what the stock guard exists
-  to prevent, and it would be worse than not having the feature at all.
-- **Post-mount verification** asserts every planned target really is a mountpoint
-  and the staging root is non-empty — so this cannot regress into the empty
-  backup it exists to fix.
-- **The guard is relaxed last.** `apply.sh` installs the traversal, patches
-  `snapshot.py`, then `sync.py`, and only then `crud.py`. Any partial failure
-  leaves the guard intact and the option merely unavailable — never
-  "guard removed, traversal missing".
-- Datasets that cannot contribute to a file tree (`mountpoint=none|legacy`,
-  unmounted, locked/encrypted) are skipped and **reported to the log** — never
-  dropped silently.
-- Scoped to **cloud_backup only**. Cloud Sync (rclone) shares the same
-  validation mixin but has no staging teardown wired in, so its guard is
-  deliberately left in place.
-
-Side benefit: the staging root is a **stable path per task**, so restic can find
-its parent snapshot between runs. Stock's `.zfs/snapshot/<name>-<timestamp>/`
-path changes every run, defeating restic's parent detection and forcing a full
-re-scan each time.
-
-### Snapshot lifecycle
-
-`zfs.snapshot.delete` defaults to **`recursive=False`**, and stock
-`restic_backup()` calls it with no options. Stock is safe only because its
-validation means a *recursive* snapshot never actually happens in the field.
-Enabling nested datasets makes them real: on a 250-dataset pool,
-`zfs snapshot -r` creates **250 snapshots**, and stock's delete removes only the
-parent — orphaning **249 on every successful run** (measured, not theorised).
-
-So the patch owns the whole lifecycle:
-
-- **Sweeps the parent and every child**, and is idempotent against stock's
-  `finally` winning the race once the mounts are released.
-- **Records the snapshot in a sidecar file before mounting anything**, so a
-  middlewared restart mid-backup cannot orphan the tree (this patch *schedules*
-  a restart at boot, so that is not hypothetical).
-- **Reclaims the tree left by a crashed run** instead of overwriting the record.
-- **Deletes the tree when staging fails** — sync.py's own `finally` deletes
-  *nothing* in that case, because its `snapshot` local never gets assigned.
-- **Enumerates datasets *after* the snapshot, never before.** A list read
-  beforehand can miss a dataset created in the gap, which the recursive snapshot
-  *would* capture but the staging plan would not — a silent omission.
-
-**Expected log noise:** stock's delete fails with `EBUSY` while the staging
-mounts pin the snapshot. You will see one benign `Error deleting snapshot ...`
-warning per run; the patch then unmounts and deletes the tree for real.
-
-### Verifying it works
-
-This feature exists because a backup can report SUCCESS while containing
-nothing, so check the contents rather than the exit status:
-
-```bash
-# 1. Does the restic snapshot actually contain child-dataset data?
-#    Pick a path that lives in a CHILD dataset (e.g. an app's config).
-midclt call cloud_backup.list_snapshots <task_id> | head
-
-# 2. List a child-dataset path inside the newest restic snapshot.
-#    If this is empty, the staging tree did not work and you are backing up NOTHING.
-midclt call cloud_backup.list_snapshot_directory <task_id> "<snapshot_id>" "/apps/lidarr/config"
-```
-
-You should see the app's real files (`lidarr.db`, `config.xml`, …). An empty
-listing means the child datasets were not staged; disable the feature and open an
-issue.
-
-```bash
-# 3. No snapshots may be left behind after a run.
-zfs list -t snapshot -r <pool> | grep -c cloud_backup-   # expect 0 between runs
-
-# 4. No staging mounts may be left behind.
-mount | grep truecloud-nested                            # expect no output
-```
-
-### Troubleshooting
-
-| Symptom | Cause |
-|---|---|
-| `This option is only available for datasets that have no further nesting` | Feature not enabled. Run `install.sh --enable-nested-snapshots`, then restart middlewared. |
-| Backup fails: `dataset '…' has no snapshot '…'; refusing to back up an incomplete tree` | Working as designed — a descendant dataset was not covered by the snapshot. The backup is refused rather than silently omitting that data. |
-| Backup fails: `snapshot '…' cannot be read (Permission denied)` | The snapshot exists but is unreadable. Middleware runs as root, so this indicates a real permissions problem, not a missing snapshot. |
-| `cloud_backup-*` snapshots accumulating | The sweep is not running. Check `apply.log` for the nested patch applying, and confirm `sync.py` carries the `TRUECLOUD_PATCH` block. |
-| Web UI blank after a patch | A bad pattern unbalanced the bundle. `apply.sh` now refuses to write in that case, but if you hit it on an older version: restore `chunk-*.js.pre-truecloud-patch` over the live chunk, then re-run `install.sh`. (`MARKER` makes an already-patched file skip, so the patch cannot heal a corrupted bundle by itself.) |
-| Stale mounts under `/run/truecloud-nested` | A crashed run. The next backup tears them down. To clear them now: `python3 patch/truecloud_nested.py cleanup` (also run by `uninstall.sh` and `recover.sh`). It names any ZFS snapshot an interrupted run left pinned. |
-
-## Supported providers after patching
-
-| Provider | Credential type in TrueNAS |
-|---|---|
-| Backblaze B2 (native B2 API) | `B2` |
-| AWS S3, Wasabi, Cloudflare R2, MinIO, and any S3-compatible endpoint | `S3` |
-| Storj (unchanged) | `STORJ_IX` |
-
-## How persistence works
-
-Two different things must survive two different events:
-
-| Event | What would be lost | What makes it survive |
-|---|---|---|
-| **Reboot** | The overlay holding the patched files lives in `/run` (tmpfs) and vanishes | The PREINIT hook re-runs `apply.sh` on every boot and schedules one middlewared restart to load the result |
-| **TrueNAS update** | `/usr/` is replaced entirely; custom files in `/etc/` are wiped with the new boot environment | This repo lives on your **data pool**, and the hook registration lives in the **TrueNAS config database** — both survive updates. The first boot after an update is just a normal boot |
-
-### What happens on every boot
-
-1. **middlewared starts** with the stock (unpatched) modules. This is
-   unavoidable: PREINIT scripts are executed *by* middlewared
-   (`ix-preinit.service` → `midclt call initshutdownscript.execute_init_tasks`),
-   so nothing registered there can run before it.
-2. **Pools import** (`ix-zfs.service`), making `/mnt/<pool>` — and this
-   repository — available.
-3. **`apply.sh` runs** (`ix-preinit.service`): mounts the writable overlay
-   (upper layer in `/run`), patches `b2.py` and `restic.py` on disk inside it,
-   patches the UI bundle, and writes `apply.log` and `hook_status.json`.
-4. **A deferred restart is scheduled.** The middlewared that is running
-   imported the stock modules in step 1 and never re-imports, so the on-disk
-   patch alone is not enough. `apply.sh` detects it was invoked by middlewared
-   and creates a transient systemd unit (`truecloud-mw-restart`, via
-   `systemd-run --no-block`) running `patch/wait_restart.sh` — detached so it
-   cannot disrupt the remainder of the boot sequence.
-5. **Once boot has settled, middlewared restarts once** and imports the
-   patched modules from the overlay. `wait_restart.sh` holds the restart until
-   the systemd boot job queue has drained (so in-flight `ix-*` units like
-   `ix-reporting` finish first) *and* middlewared's docker/apps startup has
-   reached a terminal state — plain unit ordering cannot see either, and
-   restarting middlewared while they run kills apps and dashboard reporting
-   for the whole boot. S3/B2 backup support is then active until the next
-   reboot, when the cycle repeats.
-
-What you will observe: one middlewared restart shortly after every boot (a
-brief web UI/API blip; running services are unaffected). Between steps 3
-and 5 there is a short window — typically well under a minute — where the UI
-already shows S3/B2 (the JS bundle is read from disk per request) but the
-backend is still stock. A backup job that fires inside that window fails once
-with `NotImplementedError` and succeeds on its next run; see
-[Troubleshooting](#troubleshooting) if it persists beyond boot.
-
-Manual runs of `bash patch/apply.sh` never trigger the restart — that only
-happens in boot context. `install.sh` and `recover.sh` perform their own
-explicit restarts instead, which is why a manual re-apply must be followed by
-`systemctl restart middlewared`.
+> Storj raised the price of their TrueNAS-integrated tier from **$5/month to
+> $50/month** in 2026. TrueCloud Backup is the only native TrueNAS feature that gives
+> you pre-backup ZFS snapshots, restic dedup, scheduled tasks with UI progress, and
+> dataset-lock integration. Running restic by hand loses all of it. This gets the
+> feature back with storage you already pay for.
 
 ---
 
 ## Install
 
-Clone the repository to a **persistent ZFS pool** so it survives OS updates,
-then run `install.sh` from there:
+Clone it onto a **pool** (not the boot device — that is wiped on TrueNAS upgrades),
+then run `install.sh` as root:
 
 ```bash
-# Replace /mnt/tank with your pool name
 git clone https://github.com/sudolulo/truenas-truecloud-patch.git \
-    /mnt/tank/truenas-truecloud-patch
+    /mnt/tank/truenas-truecloud-patch          # replace `tank` with your pool
 cd /mnt/tank/truenas-truecloud-patch
-bash install.sh
+sudo bash install.sh
 ```
 
-The directory you clone into becomes the **permanent install location**. The
-PREINIT boot hook is registered with the exact path you chose, and TrueNAS will
-call that path on every boot.
+That registers a PREINIT boot hook, patches middleware in a volatile overlay, and
+restarts middlewared. **It survives TrueNAS updates** — the patch is re-applied at
+every boot, never written to the system dataset.
 
-> **Do not delete or move the repository after install.**
-> If you need to relocate it, run `bash uninstall.sh` first, move the directory,
-> then run `bash install.sh` again from the new location. Deleting the repo
-> without uninstalling leaves a dangling PREINIT hook in the TrueNAS database —
-> if that happens, see [Emergency recovery](#emergency-recovery) below.
-
-Refresh your browser. S3 and B2 credentials now appear in the
-**Data Protection → TrueCloud Backup → Add** credential dropdown.
-
-## Updating
-
-To update to a new version of the patch:
+Nested-dataset snapshots are **opt-in**:
 
 ```bash
-cd /mnt/tank/truenas-truecloud-patch
-
-# If install.sh was previously run as root, the .git directory may be owned
-# by root. Fix it first, or just pull as root:
-sudo git pull          # easiest option
-# — or —
-sudo chown -R $(whoami) .git && git pull
-
-bash install.sh
+sudo bash install.sh --enable-nested-snapshots
 ```
 
-`install.sh` clears any stale kill switch, re-applies the updated patches,
-and restarts middlewared. Run `python3 patch/create_task.py verify` afterwards
-to confirm the patches loaded successfully.
+Then create a TrueCloud Backup task in the UI with a B2 or S3 credential, or from
+[the CLI](docs/cli.md).
 
-Check [CHANGELOG.md](CHANGELOG.md) to see what changed between versions.
+**Check it worked:**
+
+```bash
+sudo python3 patch/create_task.py verify
+```
+
+If something is wrong, the reason is in `apply.log` — start at
+[Recovery](docs/recovery.md).
 
 ---
 
-## Creating a task via CLI
+## TrueNAS compatibility
 
-If the UI still shows only Storj after refreshing (e.g. the JS bundle pattern
-changed in a new TrueNAS version), create tasks directly. Run this **on the
-TrueNAS host** — it talks to the local middleware via `midclt`, so it needs no
-host address or API key:
+<!-- BEGIN COMPAT MATRIX (generated by tools/compat.py --matrix --markdown) -->
+| TrueNAS | B2/S3 providers | Nested snapshots | Hardware-verified |
+| --- | --- | --- | --- |
+| 24.10.2.4 | ok | ok | — |
+| 25.04.2.6 | ok | ok | — |
+| 25.10.6 | ok | ok | — |
+| 24.10.2.5 _(unreleased)_ | ok | ok | — |
+| 25.10.7 _(unreleased)_ | ok | ok | — |
+| 26.0.0-BETA.3 _(unreleased)_ | ok | ok | — |
+| master _(27-dev)_ | **BROKEN** | **BROKEN** | — |
+
+| verdict | meaning |
+| --- | --- |
+| **ok** | Every assumption the patch makes about middleware still holds. |
+| **BROKEN** | middleware changed underneath the patch. `apply.sh` **refuses to apply that module** on this version and leaves TrueNAS stock, so backups keep working — without the module's feature. |
+| **native** | TrueNAS does this itself now. The module retires; it is not a failure. |
+
+"ok" means *the patch's assumptions hold*, checked automatically against iX's
+source. It does not mean a human ran a backup on it — that is the
+**Hardware-verified** column, which is filled in by hand and only by doing it.
+
+**`master` is not the next release.** iX branches each major off to its own
+`release/` line and master rolls straight on to the one after — so master is
+`27-dev` while 26 is still in beta. A **BROKEN** master means iX has changed
+something that will reach users *a major release from now*, not in the version you
+are about to install. Read the numbered rows for that.
+
+A row like `25.10.5 _(unreleased)_` is the next maintenance release: branched by iX,
+not tagged yet, and the very next thing a 25.10.4 box gets. It is checked precisely
+because it is the one unshipped ref that reaches real users without warning.
+<!-- END COMPAT MATRIX -->
+
+The table is **regenerated daily by CI** against iXsystems' actual middleware source
+— it is not a claim somebody typed once and forgot.
+
+It is also **static analysis**: it proves the patch's assumptions still hold, which is
+a weaker claim than "a backup ran and a restore came back". For what has actually been
+run — which tasks, on which hardware, and the md5 of the file that came back — see
+[docs/verification.md](docs/verification.md).
+
+**TrueNAS 26 is supported** as of v0.7.0, and was verified on a real
+**26.0.0-BETA.1** install: a 274-snapshot recursive backup of a 292-dataset pool, and a
+byte-identical restore of a four-level-deep child dataset. The *Hardware-verified*
+column tracks the newest beta iX has tagged (currently BETA.3), so it does not carry
+that mark — a build nobody has actually run a backup on does not get credit for one.
+
+26 rewrites `cloud_backup` from async to synchronous and deletes the private ZFS
+methods this module used to call, so getting there took real work: the patch now
+injects the wrapper flavour that matches the installed middleware, reads dataset and
+snapshot lists **from ZFS rather than middleware** (whose queries hide TrueNAS's own
+datasets — 84 of 270 on a real pool, including live app data), and owns the snapshot
+sweep even when it stages nothing (26 decides `recursive` by a rule this patch does not
+share, and would otherwise orphan one snapshot per zvol on every run).
+
+**And if a future TrueNAS breaks it, you get a missing feature, not a broken backup.**
+`apply.sh` re-checks the patch's assumptions at every boot and **refuses to apply a
+module whose assumptions no longer hold** — TrueNAS is left stock, and the reason is
+named in `apply.log`. Details: [How it works](docs/how-it-works.md#truenas-26).
+
+---
+
+## Updating
 
 ```bash
-# Replace /mnt/tank/truenas-truecloud-patch with your clone path
-
-# List your cloud credentials to find the right ID
-python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py list-credentials
-
-# Create a task with a B2 credential (id=3).
-# The restic repo password is read from stdin, so it never lands in your shell
-# history — nor in any process's argv, where `ps` would expose it.
-printf '%s' 'restic-repo-password' | \
-python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py create \
-    --name "tank-to-b2" \
-    --path /mnt/tank/data \
-    --credential 3 \
-    --bucket my-bucket \
-    --folder backups/tank \
-    --password-stdin \
-    --cache-path /mnt/tank/.restic-cache \
-    --keep-last 14
+cd /mnt/tank/truenas-truecloud-patch
+sudo bash update.sh              # newest release
+sudo bash update.sh --check      # what would change?
+sudo bash update.sh --rollback   # back to the previous version
 ```
 
-Omit `--password-stdin` and you'll be prompted for the password instead. `--password
-<secret>` still works but warns: that password is the encryption key for the whole
-repository, and a CLI argument persists in your shell history forever.
+`update.sh` refuses to run on a dirty checkout, pins you to a release tag, and keeps
+the previous revision so a rollback is one command. **There is no auto-update**: this
+patches system internals as root, and a bad commit reaching your box unattended would
+detonate on the next reboot — v0.0.4 shipped exactly such a bug and took 54 apps
+down. The manual step *is* the safety gate.
 
-> **Always pass `--cache-path`.** Without it TrueNAS runs restic with `--no-cache`,
-> which re-fetches all repo metadata from the provider every run — glacially slow
-> on large repos. Point it at a writable dir on a pool with free space.
+---
 
-> Versions ≤ 0.1.0 used the `/api/v2.0` REST API with `--host`/`--api-key`; those
-> flags are now accepted-but-ignored (REST is removed in TrueNAS 26.04).
+## Update alerts
+
+When a newer release exists, the patch raises a **TrueNAS alert** (the bell in the
+UI) telling you so. It's on by default and checks once a day.
+
+```bash
+bash install.sh --no-update-alerts   # turn it off
+bash install.sh --update-alerts      # turn it back on
+```
+
+**It will not nag you about a README.** A release whose CHANGELOG contains only a
+`### Docs` section changed no code, and raises nothing. Anything that touched the
+system raises an INFO alert; a release with a `### Security` section raises a
+WARNING. The CHANGELOG's own section headings are the signal, and a security fix
+anywhere in the range escalates the whole span — a docs-only release on top of a
+security fix still reports as security.
+
+**Release candidates never alert.** They are invisible to `update.sh` and to the
+alert, which both take the newest plain `vX.Y.Z` tag. That is what lets debugging
+happen in `-rc` tags instead of in your notification bell — see
+[Releasing](docs/releasing.md).
+
+The changelog is read from whichever forge `origin` points at, derived from the
+remote rather than hard-coded. That is not cosmetic: when the changelog cannot be
+read, the alert deliberately fires **anyway** rather than risk hiding a security
+fix — so a wrong URL would not silence the alert, it would make it fire on *every*
+release, including the documentation-only ones this section promises to suppress.
+
+### How it works, and why it's built this way
+
+TrueNAS **cannot raise an alert from the CLI** — `midclt` exposes only
+`alert.dismiss`, `alert.list`, `alert.list_categories`, `alert.list_policies` and
+`alert.restore`. Alert *creation* is internal to middlewared, and none of its ~60
+one-shot alert classes is generic enough to reuse. So the only way to get a real
+alert is to register an `AlertSource`, which is what `patch/alert_source.py` does.
+
+That is also the **least invasive** thing this patch does:
+
+| | |
+|---|---|
+| providers module | **modifies** stock files (appends code to `b2.py`, `restic.py`) |
+| nested module | **modifies** stock files (3 middleware modules) |
+| **update alert** | **adds one file. Modifies nothing.** |
+
+It's the native mechanism — the same one every built-in TrueNAS alert uses — and
+TrueNAS polls it itself, so there is no cron job and no systemd timer.
+
+- **Fail-safe.** Every error path returns `None`. It cannot take middlewared down.
+- **Read-only.** `git ls-remote` plus an HTTPS fetch of the CHANGELOG. It never
+  writes to `.git`, so it cannot leave root-owned objects behind the way a
+  `git fetch` from middlewared (which runs as root) would.
+- **Removed by `uninstall.sh`.**
+
+It only *tells* you. It never updates anything — see
+[Updating](#updating).
 
 ---
 
@@ -416,230 +204,41 @@ and restores the original UI bundle from backup.
 
 ---
 
-## If TrueNAS adds native support
+## Documentation
 
-The patch is **two independent modules**, and each retires on its own — TrueNAS
-is likely to ship one of these natively long before the other, and a module
-going native must not take the other one down with it.
-
-| Module | What it does | Detected as native when |
-|---|---|---|
-| **providers** | B2/S3 credentials for TrueCloud Backup (`b2.py`, `restic.py`, UI dropdown) | `B2RcloneRemote` carries a real `get_restic_config()` |
-| **nested** | Snapshots on datasets with child datasets (`plugins/cloud/*`) | the *"no further nesting"* validation is gone from `plugins/cloud/crud.py` |
-
-At every boot `apply.sh` checks both:
-
-- **One module goes native** → that module is skipped and logged; the other keeps
-  working, and the patch stays installed.
-- **Both are done** (native, or nested was never enabled) → the kill switch
-  (`disabled` file) is set, overlays are unmounted, and `apply.log` tells you to
-  run `uninstall.sh`.
-
-So on a box using only the provider patch, native B2 support retires the whole
-thing as before. On a box that also uses nested snapshots, native B2 support
-retires *just* that half.
-
-Check the log after any TrueNAS update:
-```bash
-tail -20 /mnt/tank/truenas-truecloud-patch/apply.log
-```
-
-`hook_status.json` reports each module separately (`module.providers`,
-`module.nested_snapshots`) with an `active` flag and a reason.
-
-**Scenarios where the auto-detect may not fire** (manual check needed):
-
-| Scenario | What happens | Action |
-|---|---|---|
-| B2 support added to a **base class** (not `B2RcloneRemote` directly) | `__dict__` check misses it; our method shadows native | Uninstall manually |
-| B2 **credential schema changed** (e.g. `provider["account"]` renamed) | `KeyError` on first backup | Uninstall or update the patch |
-| **URL builder** fixed but B2 class unchanged | URL wrapper becomes a no-op; no harm, but patch is dead weight | Uninstall at your convenience |
-
----
-
-## After a TrueNAS update
-
-1. Check the log: `cat /mnt/tank/truenas-truecloud-patch/apply.log | tail -30`
-2. If you see "WARNING: … pattern not found", the UI patch needs updating.
-   [Open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues)
-   with your TrueNAS version number.
-3. The backend patch (B2 support + URL fix) is more stable — check that a
-   B2 backup job still completes successfully after any update.
-
----
-
-## Emergency recovery
-
-### middlewared won't start
-
-Run this from the TrueNAS shell (local console, SSH, or the debug shell in
-the UI):
-
-```bash
-bash /mnt/tank/truenas-truecloud-patch/recover.sh
-```
-
-Replace the path with your clone location. This creates a kill-switch file
-(`disabled`) in the repo root, unmounts the overlay so the original files are
-visible immediately, then restarts middlewared. No reboot required.
-
-If you cannot run a script and only have a bare shell prompt:
-
-```bash
-touch /mnt/tank/truenas-truecloud-patch/disabled
-systemctl restart middlewared
-```
-
-If you don't remember where you cloned the repo (midclt won't work while middlewared is
-down), find the path two ways:
-
-```bash
-# Option 1 — search the filesystem:
-find /mnt -name "recover.sh" -path "*/truenas-truecloud-patch/*" 2>/dev/null
-
-# Option 2 — query the TrueNAS database directly:
-sqlite3 /data/freenas-v1.db \
-    "SELECT script FROM initshutdownscript WHERE comment = 'TrueCloud provider patch (S3/B2)';"
-```
-
-The `script` column shows the full path to `patch/apply.sh`; your clone root is one
-level up (strip `/patch/apply.sh` from the end). Then run the `touch` command above
-with that path.
-
-If middlewared **still** won't start after the kill switch is set, the problem
-is unrelated to this patch. Check:
-
-```bash
-journalctl -u middlewared -n 50
-```
-
-To re-enable the patch once you have investigated:
-
-```bash
-rm /mnt/tank/truenas-truecloud-patch/disabled
-bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh
-systemctl restart middlewared   # manual apply.sh runs never restart for you
-```
-
----
-
-### Web UI is blank or broken
-
-If the TrueNAS web interface loads blank or shows JavaScript errors, the
-Angular bundle may have been interrupted mid-write (e.g. power cut during
-boot). The original bundle is always backed up before patching, so recovery
-is straightforward:
-
-```bash
-# Find the backup (the path varies by TrueNAS version):
-find /usr/share/truenas /usr/share/truenas-ui /var/www/truenas -name "*.js.pre-truecloud-patch" 2>/dev/null
-
-# Restore it — substitute the actual path from the find output:
-mv /usr/share/truenas/webui/main.XXXXXXXX.js.pre-truecloud-patch \
-   /usr/share/truenas/webui/main.XXXXXXXX.js
-```
-
-Refresh your browser. The UI will return to normal (Storj-only until the
-patch re-runs at next reboot, or you run
-`bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh` manually).
-
----
-
-### Backend verify shows FAIL
-
-```bash
-python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
-```
-
-`verify` reports one line per module:
-
-| Label | Meaning |
+| | |
 |---|---|
-| `[OK  ]` | Module is active and applied. |
-| `[SKIP]` | Module is inactive — either TrueNAS now does it natively, or it is opt-in and switched off. **Not a failure.** `nested_snapshots` shows SKIP on a default install. |
-| `[FAIL]` | Module is needed but did not apply. |
+| [Nested-dataset snapshots](docs/nested-snapshots.md) | Why stock refuses, what this does instead, and **how to verify your backups actually contain the data** |
+| [How it works](docs/how-it-works.md) | What is patched, how it survives updates, the boot sequence, and what happens when TrueNAS goes native |
+| [Recovery](docs/recovery.md) | middlewared won't start, blank web UI, `verify` shows FAIL |
+| [CLI](docs/cli.md) | Creating tasks with `create_task.py` |
+| [Development](docs/releasing.md) | Tests, CI, and the release process |
 
-If a module shows `[FAIL]`:
+## What's in the repo
 
-1. **Check the apply log** for errors during the last boot:
-   ```bash
-   tail -40 /mnt/tank/truenas-truecloud-patch/apply.log
-   ```
-2. **Check middlewared's own log** for Python tracebacks:
-   ```bash
-   grep -i "truecloud\|traceback\|error" /var/log/middlewared.log 2>/dev/null | tail -30
-   journalctl -u middlewared -n 50
-   ```
-3. **A FAIL is non-fatal.** middlewared runs normally and the other module is
-   unaffected; the failed one is simply inactive. Existing backups are not at
-   risk.
-4. **If the detail says the module doesn't exist**, a TrueNAS update renamed
-   or restructured the internal API.
-   [Open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues)
-   with your TrueNAS version number and the full verify output.
+| Path | What it is |
+|---|---|
+| `install.sh` | Register the boot hook, patch, restart middlewared. Also `--enable/--disable-nested-snapshots`. |
+| `update.sh` | Fetch and apply a newer release. `--check`, `--rollback`, `--to`, `--main`. |
+| `uninstall.sh` | Remove everything. |
+| `recover.sh` | Emergency: kill switch + restart against stock files. |
+| `patch/apply.sh` | The PREINIT script. Runs at **every boot**. |
+| `patch/truecloud_nested.py` | Nested-dataset staging: plan, mount, verify, tear down, sweep snapshots. |
+| `patch/create_task.py` | Create tasks with S3/B2 credentials; `verify` the patch state. |
+| `tools/compat.py` | What the patch assumes about middlewared — and the checker. Run daily by CI *and* at every boot. |
+| `release.sh` | Cut a release. Two stages, and the second is refused without the first. |
 
----
+## Before you install
 
-## Troubleshooting
+- This is **unofficial** and not affiliated with iXsystems.
+- It patches **internal middleware APIs** with no stability contract. Every patch is
+  fail-safe: if it cannot apply, middlewared starts normally and the reason is logged.
+- **Test your restores.** True of any backup; more so here. See [Verifying it
+  works](docs/nested-snapshots.md#verifying-it-works).
+- Filing a TrueNAS bug? **Remove the patch first** and reproduce on a stock system.
+- Provided as-is, no warranty. See LICENSE.
 
-**Backups fail with `NotImplementedError` after a reboot**
+## Support
 
-The traceback ends in `rclone/base.py` → `raise NotImplementedError` and
-contains no `_tc_` frames: the running middlewared is executing stock code.
-Either the deferred restart never fired, or the patch never landed on disk
-this boot. Diagnose in this order:
-
-```bash
-# Did apply.sh run this boot, at which version, and did it schedule the restart?
-tail -40 /mnt/tank/truenas-truecloud-patch/apply.log
-
-# Full check — compares the running process against the patch timestamp
-python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
-
-# Did the deferred restart unit run, fail, or never get created?
-systemctl status truecloud-mw-restart.service
-journalctl -u truecloud-mw-restart.service --no-pager | tail -20
-```
-
-- `verify` reports the process started **before** the patch → the restart
-  didn't happen. `systemctl restart middlewared` fixes it immediately; the
-  journal output above tells you why it was missed.
-- `apply.log` shows the kill switch is active → `rm .../disabled`, then
-  `bash install.sh`.
-- `apply.log` has no entry for this boot → the hook didn't run; re-run
-  `bash install.sh` to re-register it.
-- `apply.log` header shows `[v0.0.3]` or older → update:
-  `git pull && bash install.sh` (v0.0.4 fixed patches not loading after
-  reboot).
-
-**Apply log** (check after each reboot or install):
-```bash
-cat /mnt/tank/truenas-truecloud-patch/apply.log
-```
-
-**Verify backend patch is loaded** (while middlewared is running):
-```bash
-python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
-```
-Reads `hook_status.json` written by `apply.sh` at boot **and** checks that the
-running middlewared process started *after* the patches were applied — an
-on-disk patch that middlewared has not loaded yet is reported as FAIL with
-instructions. Does not require `--host` or `--api-key`.
-
-**Middlewared log:**
-```bash
-grep truecloud-patch /var/log/middlewared.log 2>/dev/null | tail -20
-journalctl -u middlewared -n 100 2>/dev/null | grep truecloud-patch
-```
-
-**Verify the UI patch** (should print your TrueNAS version):
-```bash
-grep -c 'STORJ_IX.*S3.*B2' \
-    $(find /usr/share/truenas -name '*.js' 2>/dev/null) 2>/dev/null \
-    | grep -v ':0'
-```
-
-**`create_task.py` — "midclt not found" or permission errors**
-`create_task.py` now talks to the local middleware via `midclt`, so run it **on
-the TrueNAS host** (not remotely) as a user with middleware access (root). There
-is no HTTPS/API-key call anymore, so there is no TLS certificate to configure.
+If truenas-truecloud-patch is useful to you, consider supporting development via
+[GitHub Sponsors](https://github.com/sponsors/sudolulo) or [Ko-fi](https://ko-fi.com/sudolulo).

docs/cli.md

@@ -0,0 +1,45 @@
+# Creating a task from the CLI
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## Creating a task via CLI
+
+If the UI still shows only Storj after refreshing (e.g. the JS bundle pattern
+changed in a new TrueNAS version), create tasks directly. Run this **on the
+TrueNAS host** — it talks to the local middleware via `midclt`, so it needs no
+host address or API key:
+
+```bash
+# Replace /mnt/tank/truenas-truecloud-patch with your clone path
+
+# List your cloud credentials to find the right ID
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py list-credentials
+
+# Create a task with a B2 credential (id=3).
+# The restic repo password is read from stdin, so it never lands in your shell
+# history — nor in any process's argv, where `ps` would expose it.
+printf '%s' 'restic-repo-password' | \
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py create \
+    --name "tank-to-b2" \
+    --path /mnt/tank/data \
+    --credential 3 \
+    --bucket my-bucket \
+    --folder backups/tank \
+    --password-stdin \
+    --cache-path /mnt/tank/.restic-cache \
+    --keep-last 14
+```
+
+Omit `--password-stdin` and you'll be prompted for the password instead. `--password
+<secret>` still works but warns: that password is the encryption key for the whole
+repository, and a CLI argument persists in your shell history forever.
+
+> **Always pass `--cache-path`.** Without it TrueNAS runs restic with `--no-cache`,
+> which re-fetches all repo metadata from the provider every run — glacially slow
+> on large repos. Point it at a writable dir on a pool with free space.
+
+> Versions ≤ 0.1.0 used the `/api/v2.0` REST API with `--host`/`--api-key`; those
+> flags are now accepted-but-ignored (REST is removed in TrueNAS 26.04).
+
+---
+

docs/how-it-works.md

@@ -0,0 +1,257 @@
+# How it works
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## What is actually patched
+
+**Nothing in TrueNAS's persistent database or configuration is modified**
+(other than the boot-hook entry itself). On every boot, `patch/apply.sh` runs
+as a PREINIT script. It mounts a writable
+[overlayfs](https://docs.kernel.org/filesystems/overlayfs.html) over the
+relevant directories in `/usr/` (upper layer in `/run` tmpfs), then patches
+`b2.py` and `restic.py` inside that overlay. The overlay is volatile — it
+exists only for the current boot — but the PREINIT script recreates it
+automatically on every subsequent boot. Nothing in `/usr/` is written to
+directly.
+
+PREINIT scripts are executed *by* middlewared, which by then has already
+imported the stock modules — so after patching, `apply.sh` schedules a single
+detached middlewared restart (transient systemd unit `truecloud-mw-restart`
+running `patch/wait_restart.sh`) that loads the patched modules once boot has
+*actually* settled: the script waits for the systemd boot job queue to drain
+and for the docker/apps state machine to reach a terminal state before
+restarting. Expect one middlewared restart shortly after every boot; the UI
+and API are briefly unavailable while it happens, and running services are
+not affected.
+
+| Module | What changes | Technique |
+|---|---|---|
+| **providers** | `B2RcloneRemote` gains `get_restic_config()` — skipped automatically if TrueNAS already provides one on the class. `restic.py` URL builder is fixed: strips the stray leading slash and converts the slash separator to a colon (`b2:bucket:path`), which is the format restic 0.16.x expects. URL wrapper is a no-op if the URL is already correctly formed. | File patch applied inside the overlayfs upper layer |
+| **providers** (UI) | The Angular bundle's `filterByProviders` binding is widened from `["STORJ_IX"]` to `["STORJ_IX","S3","B2"]` | In-place text replacement in the compiled JS chunk; original is backed up before patching |
+| **nested** (opt-in) | `_truecloud_nested.py` is installed into `plugins/cloud/`, and `plugins/cloud/{snapshot,crud}.py` + `plugins/cloud_backup/sync.py` are patched so `snapshot = true` works on a dataset that has child datasets. See [below](nested-snapshots.md). | New module + file patches inside the overlayfs upper layer |
+
+All changes are **fail-safe**: if a patch cannot be applied (e.g. TrueNAS
+restructured the relevant code), middlewared starts normally, the affected module
+is simply inactive, and the reason is logged to `apply.log` in your repo root. The
+two modules are independent — one failing or going native does not disable the
+other.
+
+
+## How persistence works
+
+Two different things must survive two different events:
+
+| Event | What would be lost | What makes it survive |
+|---|---|---|
+| **Reboot** | The overlay holding the patched files lives in `/run` (tmpfs) and vanishes | The PREINIT hook re-runs `apply.sh` on every boot and schedules one middlewared restart to load the result |
+| **TrueNAS update** | `/usr/` is replaced entirely; custom files in `/etc/` are wiped with the new boot environment | This repo lives on your **data pool**, and the hook registration lives in the **TrueNAS config database** — both survive updates. The first boot after an update is just a normal boot |
+
+### What happens on every boot
+
+1. **middlewared starts** with the stock (unpatched) modules. This is
+   unavoidable: PREINIT scripts are executed *by* middlewared
+   (`ix-preinit.service` → `midclt call initshutdownscript.execute_init_tasks`),
+   so nothing registered there can run before it.
+2. **Pools import** (`ix-zfs.service`), making `/mnt/<pool>` — and this
+   repository — available.
+3. **`apply.sh` runs** (`ix-preinit.service`). Before it patches anything it runs
+   the **compatibility preflight** ([`tools/compat.py`](../tools/compat.py)) against
+   the middlewared that is *actually installed*, and **any module whose assumptions
+   no longer hold is not applied** — see [TrueNAS
+   compatibility](../README.md#truenas-compatibility). What survives that check gets applied:
+   it mounts the writable overlay (upper layer in `/run`), patches `b2.py` and
+   `restic.py` on disk inside it, patches the UI bundle, and writes `apply.log` and
+   `hook_status.json`.
+
+   An incompatible module is skipped **for this boot only**. It is not the kill
+   switch: install a release that supports your TrueNAS and the patch re-applies
+   itself on the next boot, with no manual step. (The kill switch is permanent and
+   is set only when TrueNAS has made the patch *unnecessary* — a different
+   situation, and the opposite conclusion.)
+4. **A deferred restart is scheduled.** The middlewared that is running
+   imported the stock modules in step 1 and never re-imports, so the on-disk
+   patch alone is not enough. `apply.sh` detects it was invoked by middlewared
+   and creates a transient systemd unit (`truecloud-mw-restart`, via
+   `systemd-run --no-block`) running `patch/wait_restart.sh` — detached so it
+   cannot disrupt the remainder of the boot sequence.
+5. **Once boot has settled, middlewared restarts once** and imports the
+   patched modules from the overlay. `wait_restart.sh` holds the restart until
+   the systemd boot job queue has drained (so in-flight `ix-*` units like
+   `ix-reporting` finish first) *and* middlewared's docker/apps startup has
+   reached a terminal state — plain unit ordering cannot see either, and
+   restarting middlewared while they run kills apps and dashboard reporting
+   for the whole boot. S3/B2 backup support is then active until the next
+   reboot, when the cycle repeats.
+
+What you will observe: one middlewared restart shortly after every boot (a
+brief web UI/API blip; running services are unaffected). Between steps 3
+and 5 there is a short window — typically well under a minute — where the UI
+already shows S3/B2 (the JS bundle is read from disk per request) but the
+backend is still stock. A backup job that fires inside that window fails once
+with `NotImplementedError` and succeeds on its next run; see
+[Troubleshooting](recovery.md) if it persists beyond boot.
+
+Manual runs of `bash patch/apply.sh` never trigger the restart — that only
+happens in boot context. `install.sh` and `recover.sh` perform their own
+explicit restarts instead, which is why a manual re-apply must be followed by
+`systemctl restart middlewared`.
+
+---
+
+
+## If TrueNAS adds native support
+
+The patch is **two independent modules**, and each retires on its own — TrueNAS
+is likely to ship one of these natively long before the other, and a module
+going native must not take the other one down with it.
+
+| Module | What it does | Detected as native when |
+|---|---|---|
+| **providers** | B2/S3 credentials for TrueCloud Backup (`b2.py`, `restic.py`, UI dropdown) | `B2RcloneRemote` carries a real `get_restic_config()` |
+| **nested** | Snapshots on datasets with child datasets (`plugins/cloud/*`) | the *"no further nesting"* validation is gone from `plugins/cloud/crud.py` |
+
+At every boot `apply.sh` checks both:
+
+- **One module goes native** → that module is skipped and logged; the other keeps
+  working, and the patch stays installed.
+- **Both are done** (native, or nested was never enabled) → the kill switch
+  (`disabled` file) is set, overlays are unmounted, and `apply.log` tells you to
+  run `uninstall.sh`.
+
+So on a box using only the provider patch, native B2 support retires the whole
+thing as before. On a box that also uses nested snapshots, native B2 support
+retires *just* that half.
+
+Check the log after any TrueNAS update:
+```bash
+tail -20 /mnt/tank/truenas-truecloud-patch/apply.log
+```
+
+`hook_status.json` reports each module separately (`module.providers`,
+`module.nested_snapshots`) with an `active` flag and a reason.
+
+**Scenarios where the auto-detect may not fire** (manual check needed):
+
+| Scenario | What happens | Action |
+|---|---|---|
+| B2 support added to a **base class** (not `B2RcloneRemote` directly) | `__dict__` check misses it; our method shadows native | Uninstall manually |
+| B2 **credential schema changed** (e.g. `provider["account"]` renamed) | `KeyError` on first backup | Uninstall or update the patch |
+| **URL builder** fixed but B2 class unchanged | URL wrapper becomes a no-op; no harm, but patch is dead weight | Uninstall at your convenience |
+
+---
+
+## After a TrueNAS update
+
+A TrueNAS update replaces `/usr/` wholesale, wiping the patch. You do **not** need
+to reinstall: `patch/apply.sh` runs at every boot and re-applies itself from your
+clone. But it targets internal APIs with no stability contract, so an update *can*
+break it — and the failure is quiet by design (middlewared starts fine; the patch
+just doesn't).
+
+**Check the log after any TrueNAS update:**
+
+```bash
+tail -30 /mnt/tank/truenas-truecloud-patch/apply.log
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
+```
+
+| What you see | What it means |
+|---|---|
+| `[OK] providers`, `[OK]`/`[SKIP] nested_snapshots` | Fine. Nothing to do. |
+| `WARNING: … pattern not found` (UI) | The Angular bundle changed. The UI dropdown reverts to Storj-only, but **backups keep working** — create tasks with `create_task.py` meanwhile, and [open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues) with your TrueNAS version. |
+| `WARNING: truecloud-patch is NOT COMPATIBLE with this TrueNAS version` | This TrueNAS changed middleware underneath the patch, and the named module was **deliberately not applied** — see `incompatible.json` for exactly which assumption broke. TrueNAS is left stock, so nothing is half-patched. Check [TrueNAS compatibility](../README.md#truenas-compatibility), then `bash update.sh` once a release supports your version; it re-applies itself on the next boot. This is **not** the kill switch and needs no manual reset. |
+| `[FAIL] providers` | **Your B2/S3 backups will not run.** middlewared is fine, but the credential/URL handling is gone. Open an issue with your version. |
+| `[FAIL] nested_snapshots` | The stock guard is back, so tasks with `snapshot = true` on a nested dataset will fail validation. Turn the option off on those tasks until it's fixed. |
+
+"Fail-safe" means *the box stays up* — not that your backups keep running. A
+`[FAIL] providers` is a broken backup, so check the log rather than assume.
+
+Then update the patch itself if a newer release fixes it:
+
+```bash
+bash /mnt/tank/truenas-truecloud-patch/update.sh
+```
+
+---
+
+
+
+## TrueNAS 26
+
+TrueNAS 26 changes three things underneath the nested module. **Each one alone is
+backup-breaking**, and none of them is visible from the `cloud_backup` files:
+
+| what changed | what it would do |
+| --- | --- |
+| `cloud_backup` rewritten **async → synchronous** | an `async def` wrapper hands `sync.py` a coroutine where it unpacks a tuple |
+| `get_dataset_recursive()` **deleted** from `plugins/cloud/snapshot.py` | `NameError` — the injected block called it out of the host module's namespace |
+| `plugins/zfs_/dataset.py` and `zfs_/snapshot.py` **deleted** | `zfs.dataset.query`, `zfs.snapshot.query` and `zfs.snapshot.delete` all vanish. 26 uses `filesystem.statfs` and `zfs.resource.*` |
+
+All three are fixed as of **v0.7.0**, and 26 is supported.
+
+The first two were straightforward: the patch reads which flavour of `cloud_backup`
+your box declares and injects the wrapper that matches (one implementation of the real
+logic, two thin wrappers), and it carries its own copy of the deleted helper.
+
+The third was not, and it is the one that would have hurt most — `zfs.snapshot.delete`
+is what sweeps the recursive snapshot, and without it **every run would orphan one
+snapshot per descendant dataset (250 on a real pool), forever, while reporting
+success.**
+
+The obvious port is to the public `pool.dataset.query` / `pool.snapshot.query`. Those
+methods exist, are documented, and are covered by iX's deprecation policy — and they
+are **not like-for-like replacements**. They apply a *visibility policy*: they hide the
+datasets TrueNAS considers its own (`ix-apps/*`, `.system/*`, `.ix-virt/*`). On a real
+pool that is **84 of 270 datasets, including live application data.** Staging from that
+view would have omitted every one of them from the backup — and the planner would never
+have seen them, so they would not have appeared in its "skipped" list either. A green
+backup, quietly missing data. The snapshot query lies the same way, so the sweep would
+have orphaned one snapshot per hidden dataset.
+
+No source analysis could have caught that. The methods are all present and correctly
+shaped. Only running it could, which is why it took a real 26 box.
+
+So the module now follows one rule:
+
+> **Read the truth from ZFS. Make changes through middleware.**
+
+Enumeration is `zfs list` — no policy can filter it, and it behaves identically on every
+release, which also means one code path instead of a version conditional. Mutation stays
+a middleware call, so TrueNAS's own bookkeeping stays consistent; an exact-name delete
+works fine even on a dataset the query hides. It is only enumeration that lies.
+
+The snapshot *delete* still needs a namespace, and no single one spans every release —
+24.10 and 25.04 have `zfs.snapshot`, 26 has only `pool.snapshot`, 25.10 has both. So it
+is resolved at runtime, by asking whether the namespace can actually delete. `tools/compat.py`
+asks the identical question against iX's source, and a test binds the two lists together,
+so what CI verifies and what runs cannot drift apart.
+
+### The one 26 changed that nothing warned about
+
+Stock decides whether to take a **recursive** snapshot by its own rule, and on 26 that
+rule stopped being ours:
+
+| | decides `recursive` by |
+| --- | --- |
+| stock ≤ 25.10 | `get_dataset_recursive()` — the same function this patch vendors |
+| **stock 26** | `filesystem.statfs`: `recursive = (path == the dataset's mountpoint)` |
+| this patch | `get_dataset_recursive()` — is a mounted *filesystem* child under the path? |
+
+Up to 25.10 those were the *same question*, so a snapshot the patch declined to stage
+provably had no children and stock's non-recursive delete was correct. On 26 they
+disagree: a dataset whose only descendants are **zvols** or **legacy-mountpoint**
+datasets gets a recursive snapshot, while the patch sees nothing to stage. Stock then
+destroys the parent only — and with no staging tree there was no sidecar, and the
+garbage collector only ever ran from the staging path. Nothing on the box would ever
+have found the children.
+
+It was reproduced on a 26 VM (one orphan per zvol, every run, backup green) and closed:
+**ownership of the sweep is no longer conditional on staging.**
+
+`master` (development after 26) **does** report BROKEN: iXsystems are still reshaping
+these functions there, renaming `middleware` → `context` and `cloud_backup` → `entry`
+and adding a required `credentials` parameter. That is a moving target and is
+deliberately not chased; the check keeps reporting it until it settles into a beta,
+which is when it becomes worth fixing. Until then, a box running master would simply
+not get the modules — `apply.sh` refuses to apply a module whose assumptions no longer
+hold, and says why in `apply.log`.

docs/nested-snapshots.md

@@ -0,0 +1,194 @@
+# Nested-dataset snapshots
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## Nested-dataset snapshots
+
+**Opt-in, off by default.** It changes how backups read their source data, so it
+is never enabled implicitly:
+
+```bash
+bash install.sh --enable-nested-snapshots
+bash install.sh --disable-nested-snapshots
+```
+
+With neither flag `install.sh` leaves the setting alone, so `git pull && bash
+install.sh` won't flip it. The providers module is unaffected either way.
+
+Validated end to end on a live 252-dataset pool: an unattended scheduled backup
+of `/mnt/Tap` built a 173-mount staging tree, completed in **18m14s**, and left
+**zero** orphaned snapshots and **zero** stale mounts behind. The same backup
+previously stalled at 74% for over 12 hours reading live files.
+
+Still: verify your own first run actually contains child-dataset data before you
+rely on it — see [Verifying it works](#verifying-it-works). That advice is not
+boilerplate; it is the specific thing this feature exists to make true.
+
+TrueCloud Backup's **Take Snapshot** option makes restic read from a frozen ZFS
+snapshot instead of live files. Without it the backup reads data *while apps are
+writing to it* — databases get captured mid-write, and an app that rewrites its
+files continuously can stall a backup indefinitely as restic chases a moving
+target.
+
+Stock TrueNAS refuses to enable it on most real-world paths:
+
+```
+[EINVAL] cloud_backup_update.snapshot:
+  This option is only available for datasets that have no further nesting
+```
+
+That rules out **any pool running Apps** — every app is its own dataset, usually
+with `config`/`pgdata` children of its own. On a typical box that is 100+ nested
+datasets, so the feature is effectively unusable exactly where it matters most.
+
+### Why stock refuses
+
+The guard is **correct**. `plugins/cloud/snapshot.py` already takes a *recursive*
+ZFS snapshot — but it then points restic at the **parent** dataset's
+`.zfs/snapshot/<snap>/` directory, and ZFS does not expose child datasets
+through a parent's snapshot directory:
+
+```
+/mnt/Tap/.zfs/snapshot/<snap>/apps/                 ->  0 entries  (children invisible)
+/mnt/Tap/apps/lidarr/config/.zfs/snapshot/<snap>/   ->  the real data
+```
+
+So if you just remove the validation, restic walks a near-empty tree, reports
+SUCCESS, and uploads almost nothing — a green backup job protecting no data. iX
+gate the config rather than ship a backup that lies about succeeding.
+
+That is worth spelling out, because deleting those four lines in
+`plugins/cloud/crud.py` is the obvious "fix" and it is the wrong one. The guard
+is load-bearing: it has to be *replaced* with a working traversal, not removed.
+
+### What this patch does instead
+
+After the (already recursive) snapshot is taken, every descendant dataset's own
+`.zfs/snapshot/<snap>` is bind-mounted into a **staging tree** that mirrors the
+original layout, and restic is pointed at the staging root — a complete,
+consistent, point-in-time view of the whole subtree. Only then is the guard
+relaxed.
+
+Safety properties, in order of importance:
+
+- **Staging failure is loud.** If any descendant cannot be staged, the backup
+  *fails*. A silently-incomplete backup is precisely what the stock guard exists
+  to prevent, and it would be worse than not having the feature at all.
+- **Post-mount verification** asserts every planned target really is a mountpoint
+  and the staging root is non-empty — so this cannot regress into the empty
+  backup it exists to fix.
+- **The guard is relaxed last.** `apply.sh` installs the traversal, patches
+  `snapshot.py`, then `sync.py`, and only then `crud.py`. Any partial failure
+  leaves the guard intact and the option merely unavailable — never
+  "guard removed, traversal missing".
+- Datasets that cannot contribute to a file tree (`mountpoint=none|legacy`,
+  unmounted, locked/encrypted) are skipped and **reported to the log** — never
+  dropped silently.
+- Scoped to **cloud_backup only**. Cloud Sync (rclone) shares the same
+  validation mixin but has no staging teardown wired in, so its guard is
+  deliberately left in place.
+
+Side benefit: the staging root is a **stable path per task**, so restic can find
+its parent snapshot between runs. Stock's `.zfs/snapshot/<name>-<timestamp>/`
+path changes every run, defeating restic's parent detection and forcing a full
+re-scan each time.
+
+### Snapshot lifecycle
+
+> **Two mechanisms clean up, and the second exists because the first can be destroyed.**
+>
+> 1. **The sidecar** records exactly which snapshots a run pinned, and is removed only
+>    on a confirmed-clean sweep. Precise, and it survives a middlewared restart.
+> 2. **The garbage collector** finds leftovers by *name*, so it still works when the
+>    sidecar is gone — and it can be: **the sidecar lives in `/run`, which is tmpfs.** A
+>    reboot mid-backup takes it, and with it the only record of a 250-snapshot tree.
+>
+> The collector runs at the start of every backup, after the sidecar reclaim. It will
+> only touch a snapshot named `<dataset>@<task>-<timestamp>` that is not the current
+> run's, has **nothing mounted from it** (which is what protects a concurrently-running
+> backup), and is **over an hour old**. Periodic `auto-*` snapshots, other tasks'
+> snapshots, and anything you made by hand are structurally out of reach.
+
+
+> **A snapshot may survive a run, and that is expected.** ZFS **automounts**
+> `<dataset>/.zfs/snapshot/<snap>` the moment it is read, and holds it for
+> `zfs_expire_snapshot` seconds (**300** by default) after the last access. So
+> whatever restic read *last* is still pinned when we try to destroy it, and
+> `zfs destroy` refuses with `dataset is busy`.
+>
+> The patch unmounts those automounts itself and retries, which clears ~255 of 256 on
+> a real pool. The one that remains is **logged, its sidecar is kept, and the next run
+> reclaims it before doing anything else** — so the leak is bounded at a single cycle
+> instead of growing forever. Seeing one `could not delete snapshot … it will be
+> reclaimed on the next run` in the log is normal. Seeing the count *grow* run over run
+> is not, and would be a bug.
+>
+> This is why the sidecar is removed **only on a confirmed-clean sweep**: it is the
+> only record those snapshots exist, and a run that dropped it while they were still
+> around would orphan them permanently. That is precisely what happened before this was
+> fixed.
+
+
+`zfs.snapshot.delete` defaults to **`recursive=False`**, and stock
+`restic_backup()` calls it with no options. Stock is safe only because its
+validation means a *recursive* snapshot never actually happens in the field.
+Enabling nested datasets makes them real: on a 250-dataset pool,
+`zfs snapshot -r` creates **250 snapshots**, and stock's delete removes only the
+parent — orphaning **249 on every successful run** (measured, not theorised).
+
+So the patch owns the whole lifecycle:
+
+- **Sweeps the parent and every child**, and is idempotent against stock's
+  `finally` winning the race once the mounts are released.
+- **Records the snapshot in a sidecar file before mounting anything**, so a
+  middlewared restart mid-backup cannot orphan the tree (this patch *schedules*
+  a restart at boot, so that is not hypothetical).
+- **Reclaims the tree left by a crashed run** instead of overwriting the record.
+- **Deletes the tree when staging fails** — sync.py's own `finally` deletes
+  *nothing* in that case, because its `snapshot` local never gets assigned.
+- **Enumerates datasets *after* the snapshot, never before.** A list read
+  beforehand can miss a dataset created in the gap, which the recursive snapshot
+  *would* capture but the staging plan would not — a silent omission.
+
+**Expected log noise:** stock's delete fails with `EBUSY` while the staging
+mounts pin the snapshot. You will see one benign `Error deleting snapshot ...`
+warning per run; the patch then unmounts and deletes the tree for real.
+
+### Verifying it works
+
+This feature exists because a backup can report SUCCESS while containing
+nothing, so check the contents rather than the exit status:
+
+```bash
+# 1. Does the restic snapshot actually contain child-dataset data?
+#    Pick a path that lives in a CHILD dataset (e.g. an app's config).
+midclt call cloud_backup.list_snapshots <task_id> | head
+
+# 2. List a child-dataset path inside the newest restic snapshot.
+#    If this is empty, the staging tree did not work and you are backing up NOTHING.
+midclt call cloud_backup.list_snapshot_directory <task_id> "<snapshot_id>" "/apps/lidarr/config"
+```
+
+You should see the app's real files (`lidarr.db`, `config.xml`, …). An empty
+listing means the child datasets were not staged; disable the feature and open an
+issue.
+
+```bash
+# 3. No snapshots may be left behind after a run.
+zfs list -t snapshot -r <pool> | grep -c cloud_backup-   # expect 0 between runs
+
+# 4. No staging mounts may be left behind.
+mount | grep truecloud-nested                            # expect no output
+```
+
+### Troubleshooting
+
+| Symptom | Cause |
+|---|---|
+| `This option is only available for datasets that have no further nesting` | Feature not enabled. Run `install.sh --enable-nested-snapshots`, then restart middlewared. |
+| Backup fails: `dataset '…' has no snapshot '…'; refusing to back up an incomplete tree` | Working as designed — a descendant dataset was not covered by the snapshot. The backup is refused rather than silently omitting that data. |
+| Backup fails: `snapshot '…' cannot be read (Permission denied)` | The snapshot exists but is unreadable. Middleware runs as root, so this indicates a real permissions problem, not a missing snapshot. |
+| `cloud_backup-*` snapshots accumulating | The sweep is not running. Check `apply.log` for the nested patch applying, and confirm `sync.py` carries the `TRUECLOUD_PATCH` block. |
+| Web UI blank after a patch | A bad pattern unbalanced the bundle. `apply.sh` now refuses to write in that case, but if you hit it on an older version: restore `chunk-*.js.pre-truecloud-patch` over the live chunk, then re-run `install.sh`. (`MARKER` makes an already-patched file skip, so the patch cannot heal a corrupted bundle by itself.) |
+| Stale mounts under `/run/truecloud-nested` | A crashed run. The next backup tears them down. To clear them now: `python3 patch/truecloud_nested.py cleanup` (also run by `uninstall.sh` and `recover.sh`). It names any ZFS snapshot an interrupted run left pinned. |
+

docs/recovery.md

@@ -0,0 +1,179 @@
+# Recovery and troubleshooting
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## Emergency recovery
+
+### middlewared won't start
+
+Run this from the TrueNAS shell (local console, SSH, or the debug shell in
+the UI):
+
+```bash
+bash /mnt/tank/truenas-truecloud-patch/recover.sh
+```
+
+Replace the path with your clone location. This creates a kill-switch file
+(`disabled`) in the repo root, unmounts the overlay so the original files are
+visible immediately, then restarts middlewared. No reboot required.
+
+If you cannot run a script and only have a bare shell prompt:
+
+```bash
+touch /mnt/tank/truenas-truecloud-patch/disabled
+systemctl restart middlewared
+```
+
+If you don't remember where you cloned the repo (midclt won't work while middlewared is
+down), find the path two ways:
+
+```bash
+# Option 1 — search the filesystem:
+find /mnt -name "recover.sh" -path "*/truenas-truecloud-patch/*" 2>/dev/null
+
+# Option 2 — query the TrueNAS database directly:
+sqlite3 /data/freenas-v1.db \
+    "SELECT script FROM initshutdownscript WHERE comment = 'TrueCloud provider patch (S3/B2)';"
+```
+
+The `script` column shows the full path to `patch/apply.sh`; your clone root is one
+level up (strip `/patch/apply.sh` from the end). Then run the `touch` command above
+with that path.
+
+If middlewared **still** won't start after the kill switch is set, the problem
+is unrelated to this patch. Check:
+
+```bash
+journalctl -u middlewared -n 50
+```
+
+To re-enable the patch once you have investigated:
+
+```bash
+rm /mnt/tank/truenas-truecloud-patch/disabled
+bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh
+systemctl restart middlewared   # manual apply.sh runs never restart for you
+```
+
+---
+
+### Web UI is blank or broken
+
+If the TrueNAS web interface loads blank or shows JavaScript errors, the
+Angular bundle may have been interrupted mid-write (e.g. power cut during
+boot). The original bundle is always backed up before patching, so recovery
+is straightforward:
+
+```bash
+# Find the backup (the path varies by TrueNAS version):
+find /usr/share/truenas /usr/share/truenas-ui /var/www/truenas -name "*.js.pre-truecloud-patch" 2>/dev/null
+
+# Restore it — substitute the actual path from the find output:
+mv /usr/share/truenas/webui/main.XXXXXXXX.js.pre-truecloud-patch \
+   /usr/share/truenas/webui/main.XXXXXXXX.js
+```
+
+Refresh your browser. The UI will return to normal (Storj-only until the
+patch re-runs at next reboot, or you run
+`bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh` manually).
+
+---
+
+### Backend verify shows FAIL
+
+```bash
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
+```
+
+`verify` reports one line per module:
+
+| Label | Meaning |
+|---|---|
+| `[OK  ]` | Module is active and applied. |
+| `[SKIP]` | Module is inactive — either TrueNAS now does it natively, or it is opt-in and switched off. **Not a failure.** `nested_snapshots` shows SKIP on a default install. |
+| `[FAIL]` | Module is needed but did not apply. |
+
+If a module shows `[FAIL]`:
+
+1. **Check the apply log** for errors during the last boot:
+   ```bash
+   tail -40 /mnt/tank/truenas-truecloud-patch/apply.log
+   ```
+2. **Check middlewared's own log** for Python tracebacks:
+   ```bash
+   grep -i "truecloud\|traceback\|error" /var/log/middlewared.log 2>/dev/null | tail -30
+   journalctl -u middlewared -n 50
+   ```
+3. **A FAIL is non-fatal.** middlewared runs normally and the other module is
+   unaffected; the failed one is simply inactive. Existing backups are not at
+   risk.
+4. **If the detail says the module doesn't exist**, a TrueNAS update renamed
+   or restructured the internal API.
+   [Open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues)
+   with your TrueNAS version number and the full verify output.
+
+---
+
+## Troubleshooting
+
+**Backups fail with `NotImplementedError` after a reboot**
+
+The traceback ends in `rclone/base.py` → `raise NotImplementedError` and
+contains no `_tc_` frames: the running middlewared is executing stock code.
+Either the deferred restart never fired, or the patch never landed on disk
+this boot. Diagnose in this order:
+
+```bash
+# Did apply.sh run this boot, at which version, and did it schedule the restart?
+tail -40 /mnt/tank/truenas-truecloud-patch/apply.log
+
+# Full check — compares the running process against the patch timestamp
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
+
+# Did the deferred restart unit run, fail, or never get created?
+systemctl status truecloud-mw-restart.service
+journalctl -u truecloud-mw-restart.service --no-pager | tail -20
+```
+
+- `verify` reports the process started **before** the patch → the restart
+  didn't happen. `systemctl restart middlewared` fixes it immediately; the
+  journal output above tells you why it was missed.
+- `apply.log` shows the kill switch is active → `rm .../disabled`, then
+  `bash install.sh`.
+- `apply.log` has no entry for this boot → the hook didn't run; re-run
+  `bash install.sh` to re-register it.
+- `apply.log` header shows `[v0.0.3]` or older → update:
+  `git pull && bash install.sh` (v0.0.4 fixed patches not loading after
+  reboot).
+
+**Apply log** (check after each reboot or install):
+```bash
+cat /mnt/tank/truenas-truecloud-patch/apply.log
+```
+
+**Verify backend patch is loaded** (while middlewared is running):
+```bash
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
+```
+Reads `hook_status.json` written by `apply.sh` at boot **and** checks that the
+running middlewared process started *after* the patches were applied — an
+on-disk patch that middlewared has not loaded yet is reported as FAIL with
+instructions. Does not require `--host` or `--api-key`.
+
+**Middlewared log:**
+```bash
+grep truecloud-patch /var/log/middlewared.log 2>/dev/null | tail -20
+journalctl -u middlewared -n 100 2>/dev/null | grep truecloud-patch
+```
+
+**Verify the UI patch** (should print your TrueNAS version):
+```bash
+grep -c 'STORJ_IX.*S3.*B2' \
+    $(find /usr/share/truenas -name '*.js' 2>/dev/null) 2>/dev/null \
+    | grep -v ':0'
+```
+
+**`create_task.py` — "midclt not found" or permission errors**
+`create_task.py` now talks to the local middleware via `midclt`, so run it **on
+the TrueNAS host** (not remotely) as a user with middleware access (root). There
+is no HTTPS/API-key call anymore, so there is no TLS certificate to configure.

docs/releasing.md

@@ -0,0 +1,98 @@
+# Development and releasing
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## Development
+
+Parts of this project were written with AI assistance (Claude). All of it is
+reviewed and tested before release; the test suite and CI exist in large part to
+make that review meaningful. Bugs are mine.
+
+```bash
+pip install pytest ruff
+ruff check patch tests tools
+pytest tests
+```
+
+CI runs shellcheck, `bash -n`, ruff, and pytest on Python 3.11–3.13. Two checks
+are worth calling out, because nothing else would catch what they catch:
+
+- The tests **`compile()` the `*_BLOCK` strings** in `patch/apply.sh`. Those are
+  Python source appended into live `middlewared` modules — a syntax error there
+  breaks the box at boot, and they're string literals, so nothing else type-checks
+  them.
+- CI asserts **every script declares the same version**, and that it matches the
+  newest CHANGELOG entry. `VERSION=` had silently drifted to three different
+  values across the scripts before anything checked.
+
+The project is hosted on **Gitea** (`git.arch.fyi/flan/truenas-truecloud-patch`)
+and mirrored to GitHub. Both run the same workflows — Gitea reads
+`.github/workflows/` too — so a change is checked twice, on two independent runners.
+
+---
+
+## Releasing
+
+**Every release interrupts every user.** An update alert fires on each installed
+box (see [Update alerts](../README.md#update-alerts)), so a release that exists only to fix the
+last release teaches people to dismiss the alert — and one day that alert will be
+carrying a security fix. This project cut twelve releases in a single day once.
+Never again, and not by good intentions: by a gate.
+
+### The rule
+
+> A stable `vX.Y.Z` may only be published if a `vX.Y.Z-rcN` tag points at the
+> **same commit**.
+
+Release candidates are **invisible to users**: `update.sh` and the update alert both
+take the newest plain `vX.Y.Z` tag, so an `-rc` is never offered as an update. All
+the debugging therefore happens across `rc1`, `rc2`, `rc3` — at nobody's expense —
+instead of across `v0.5.0`, `v0.5.1`, `v0.5.2`, at everybody's.
+
+"The candidate passed, then I pushed one more little fix" is refused **by name**.
+That is not hypothetical; it is exactly how v0.5.1 happened.
+
+### Day to day
+
+You don't touch the release machinery. Write your changes under `## Unreleased` in
+`CHANGELOG.md` and push to `main`. `main` is a work surface — it is allowed to be
+mid-thought. Releasing is a separate, deliberate act.
+
+### Cutting a release
+
+```bash
+bash release.sh 0.6.0 --check      # what would ship? what is the next rc?
+
+bash release.sh 0.6.0 --rc         # promotes `## Unreleased` -> v0.6.0, stamps every
+                                   # VERSION=, tags v0.6.0-rc1, pushes. Users see nothing.
+
+#   ... install it on a real box. Exercise it. Break it. ...
+#   Found a bug? Fix it on main, then `bash release.sh 0.6.0 --rc` again -> rc2.
+
+bash release.sh 0.6.0 --promote    # publishes v0.6.0. REFUSED unless an rc points here.
+```
+
+### The gates, and where they live
+
+The logic is Python so it can be unit-tested; CI is the enforcement boundary
+because it is the only actor holding the token that publishes. `release.sh` runs the
+**same** code locally so you fail in 200 ms instead of after a push.
+
+| gate | enforces | where |
+| --- | --- | --- |
+| [`release_notes.py check`](../tools/release_notes.py) | every script's `VERSION=` matches the tag; the CHANGELOG section exists and is non-empty; **nothing is stranded under `## Unreleased`** | `release.sh` + CI |
+| [`release_gate.py`](../tools/release_gate.py) | **an rc points at this exact commit** | `release.sh` + CI |
+| the full suite | ruff, pytest, shellcheck, `bash -n` — re-run against the *tagged* commit | CI |
+
+A release's body **is** its `CHANGELOG.md` section — there is no second place to
+write release notes, and therefore no second place for them to go stale. Releases
+are published on both forges.
+
+### Why the alert doesn't nag
+
+A release whose CHANGELOG contains only a `### Docs` section changed no code, and
+raises **no alert**. Candidates raise no alert either. So the only thing that ever
+interrupts a user is a real, complete change — which is the entire point.
+
+---
+

docs/verification.md

@@ -0,0 +1,77 @@
+# What has actually been run
+
+The support matrix in the README is **static analysis**: it proves the patch's
+assumptions about middlewared still hold. That is a strictly weaker claim than "a
+backup ran and a restore came back". This file is the stronger claim, and it is
+maintained by hand, because the only way to fill it in is to do it.
+
+If you are deciding whether to trust this with your backups, read this file, not the
+matrix.
+
+---
+
+## v0.7.0 — TrueNAS 25.10.4 (production hardware)
+
+Six live TrueCloud tasks, all `snapshot = true`, backing up to Backblaze B2. Three were
+exercised end to end, chosen to cover the three shapes the code handles differently:
+
+| Task | Path | Shape | Result |
+| --- | --- | --- | --- |
+| 5 | `/mnt/Tap` | 191 nested datasets staged, 282-snapshot recursive tree | SUCCESS |
+| 7 | `/mnt/Tank/backups` | 215 filesystems **+ 2 zvols** | SUCCESS |
+| 9 | `/mnt/Tank/flan` | **no** nested filesystem children | SUCCESS |
+
+After every run: **0 orphaned snapshots, 0 leaked bind mounts, 0 stale sidecars.**
+
+**The restore.** `apps/vaultwarden/data/config.json` — a file inside a *child* dataset,
+which is exactly what stock TrueNAS cannot capture — was restored from B2 and compared
+against the live file:
+
+    live      f809df6ba231986b1ba824044228a03a   1808 bytes
+    restored  f809df6ba231986b1ba824044228a03a   1808 bytes
+    => byte-identical
+
+**The collector earned its keep on real data.** The pool was already carrying an orphan:
+`Tap/apps/prometheus@cloud_backup-5-20260713202355`, left behind by an earlier run when
+ZFS's automount held the snapshot busy past all four retries. The first v0.7.0 run found
+it by name, reclaimed it, and the pool's snapshot count went 2148 → 2147. That is the
+garbage collector doing the job it was written for, against a leak that was already
+there and that nothing else would ever have found.
+
+**Boot path.** `apply.sh` is registered as a PREINIT `initshutdownscript`; it was
+re-run against the live middleware and left exactly one `TRUECLOUD_PATCH` marker in
+each patched module (a second copy stacked into a live middlewared module would break
+the box at boot). It correctly detected the box as **async** (`cloud_backup is async
+(TrueNAS <= 25.10)`) and injected the matching wrappers.
+
+**Upgrade path.** `update.sh` was used to move the box from the release candidate to
+the stable tag, in detached HEAD at `v0.7.0`, which is how a user's box actually
+upgrades.
+
+## v0.7.0 — TrueNAS 26.0.0-BETA.1 (VM)
+
+A throwaway VM whose pool reproduces the production pool's *shape* — 292 datasets, 26
+`legacy` mountpoints, nesting five deep — because every bug found on the real box came
+from the shape of the pool, not the bytes in it. MinIO was not used; `rclone serve s3`
+(already on the box) provided the S3 target, so no real B2 credential ever entered the
+VM.
+
+* 274-snapshot recursive backup of the 292-dataset pool. 0 orphans, 0 leaked mounts.
+* Restored `ix-apps/app_mounts/vaultwarden/pgData` — **four levels deep, and a dataset
+  that middleware's own `pool.dataset.query` hides from itself** — byte-identical.
+* The zvol-orphan case was **reproduced with the fix disabled** (one orphan per zvol,
+  every run, backup green), then **closed with it enabled**. See the CHANGELOG entry
+  for why TrueNAS 26 decides `recursive` by a different rule than this patch decides
+  `nested`.
+
+## What is NOT covered
+
+* **24.10 and 25.04** are `ok` in the matrix — the assumptions hold, checked against
+  iX's source — but nobody has run a backup on them. The matrix says so.
+* **master** is BROKEN, and correctly reports so: iX renamed the leading parameters of
+  `get_restic_config` and `restic_backup`. It is not a shipped release; the daily
+  compatibility bot files it, and `apply.sh` would refuse to apply the modules on a box
+  running it.
+* A **reboot** of the production box has not been done on v0.7.0. `apply.sh` was
+  re-executed by hand against the live middleware, which exercises the same code path,
+  but the PREINIT ordering itself has only been proven on earlier versions.

install.sh

@@ -18,7 +18,7 @@
 
 set -euo pipefail
 
-VERSION="0.3.5"
+VERSION="0.7.0"
 
 # The directory containing install.sh is the permanent install location.
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -31,6 +31,8 @@ _NESTED_MARKER="$PATCH_DIR/nested_snapshots_enabled"
 # `git pull`) must never flip it on or off by itself: with neither flag given,
 # whatever was chosen previously is preserved.
 _nested_choice=""
+_alert_choice=""
+_ALERT_MARKER="$PATCH_DIR/update_alerts_disabled"
 
 usage() {
     cat <<USAGE
@@ -42,6 +44,8 @@ Options:
                               Stock TrueNAS refuses this; see README. Off by
                               default because it changes how backups read data.
   --disable-nested-snapshots  Turn it back off; the stock guard is restored.
+  --no-update-alerts          Do not raise a TrueNAS alert when an update exists.
+  --update-alerts             Re-enable those alerts (they are on by default).
   -h, --help                  Show this help.
 
 With neither flag, the current setting is left unchanged.
@@ -52,6 +56,8 @@ while [ $# -gt 0 ]; do
     case "$1" in
         --enable-nested-snapshots)  _nested_choice="on" ;;
         --disable-nested-snapshots) _nested_choice="off" ;;
+        --no-update-alerts)         _alert_choice="off" ;;
+        --update-alerts)            _alert_choice="on" ;;
         -h|--help)                  usage; exit 0 ;;
         *)
             echo "ERROR: unknown option: $1" >&2
@@ -82,6 +88,44 @@ if [ "$(id -u)" -ne 0 ]; then
     exit 1
 fi
 
+# ── Minimum TrueNAS version ───────────────────────────────────────────────────
+#
+# TrueCloud Backup -- the feature this whole project extends -- was introduced in
+# 24.10. On anything older, `plugins/cloud_backup/` does not exist at all: there is
+# no restic, no cloud_backup task type, and nothing for the patch to attach to. It
+# would not break the box, it would simply do nothing, silently, while the user
+# believed their backups were configured. Say no clearly instead.
+#
+# A version we cannot PARSE is not a version we may refuse on: warn and continue.
+# Refusing to install over a string we failed to read would be a worse failure than
+# the one being prevented.
+MIN_TRUENAS="24.10"
+
+_tc_version_raw=""
+if command -v midclt &>/dev/null; then
+    _tc_version_raw=$(midclt call system.version 2>/dev/null || true)   # TrueNAS-25.10.4
+fi
+# Both spellings are real: modern releases report `TrueNAS-25.10.4`, older ones
+# `TrueNAS-SCALE-24.04.2` -- and the SCALE- form is used by exactly the versions
+# this gate exists to turn away, so failing to parse it would let them through.
+_tc_version=$(printf '%s' "$_tc_version_raw" \
+    | sed -n 's/^TrueNAS-\(SCALE-\)\{0,1\}\([0-9]\{1,\}\.[0-9]\{1,\}\).*/\2/p')
+
+if [ -z "$_tc_version" ]; then
+    echo "WARNING: could not determine the TrueNAS version" \
+         "${_tc_version_raw:+(got '${_tc_version_raw}')}."
+    echo "WARNING: this patch requires TrueNAS SCALE ${MIN_TRUENAS} or newer. Continuing anyway."
+    echo ""
+elif [ "$(printf '%s\n%s\n' "$MIN_TRUENAS" "$_tc_version" | sort -V | head -1)" != "$MIN_TRUENAS" ]; then
+    echo "ERROR: TrueNAS ${_tc_version} is too old — this patch requires ${MIN_TRUENAS} or newer." >&2
+    echo "" >&2
+    echo "  TrueCloud Backup does not exist before ${MIN_TRUENAS}, so there is nothing" >&2
+    echo "  here for the patch to extend. Upgrade TrueNAS first." >&2
+    exit 1
+else
+    echo "TrueNAS ${_tc_version} (minimum ${MIN_TRUENAS}) — ok"
+fi
+
 if ! command -v midclt &>/dev/null; then
     echo "ERROR: midclt not found. Run this script on TrueNAS SCALE." >&2
     exit 1
@@ -95,8 +139,14 @@ fi
 # ── Set permissions ───────────────────────────────────────────────────────────
 
 echo "Setting permissions ..."
-chmod +x "$PATCH_DIR/patch/apply.sh" "$PATCH_DIR/patch/create_task.py" \
-          "$PATCH_DIR/recover.sh" "$PATCH_DIR/uninstall.sh"
+# Guard each path: under `set -e` a chmod on a missing file aborts the install.
+# The file set changes between versions, so `update.sh --rollback` to an older
+# revision must not be killed by a name this version happens to know about.
+for _exe in patch/apply.sh patch/create_task.py recover.sh uninstall.sh update.sh; do
+    if [ -f "$PATCH_DIR/$_exe" ]; then
+        chmod +x "$PATCH_DIR/$_exe"
+    fi
+done
 echo "Done."
 echo ""
 
@@ -186,6 +236,34 @@ case "$_nested_choice" in
 esac
 echo ""
 
+# ── Update alerts (on by default) ─────────────────────────────────────────────
+# TrueNAS cannot raise an alert from the CLI (midclt exposes only dismiss/list/
+# restore), so this installs an AlertSource into middlewared/alert/source/ — the
+# same mechanism every built-in TrueNAS alert uses. It ADDS a file and modifies
+# none, which makes it the least invasive thing this patch does.
+#
+# It only alerts for releases that changed something: a documentation-only release
+# raises nothing.
+
+case "$_alert_choice" in
+    off)
+        touch "$_ALERT_MARKER"
+        echo "Update alerts: DISABLED (apply.sh will remove the alert source)."
+        ;;
+    on)
+        rm -f "$_ALERT_MARKER"
+        echo "Update alerts: enabled."
+        ;;
+    *)
+        if [ -f "$_ALERT_MARKER" ]; then
+            echo "Update alerts: disabled (unchanged)."
+        else
+            echo "Update alerts: enabled (checks daily; docs-only releases are ignored)."
+        fi
+        ;;
+esac
+echo ""
+
 # ── Apply now ─────────────────────────────────────────────────────────────────
 
 echo "Applying patches ..."

patch/alert_source.py

@@ -0,0 +1,258 @@
+"""TrueNAS alert: a truecloud-patch update is available.
+
+Installed by patch/apply.sh into middlewared/alert/source/, where middlewared
+discovers and polls it natively — no cron job, no systemd timer.
+
+@PATCH_DIR@ is substituted at install time.
+
+Two rules govern this file:
+
+1. **It must never break middlewared.** It runs inside the alert framework on a
+   timer. Every failure path returns None (no alert) rather than raising.
+
+2. **It must not nag.** A release whose CHANGELOG only has a "### Docs" section
+   changed no code, and nobody wants an alert because a README was reworded. The
+   CHANGELOG's own section headings are the signal — see tools/release_notes.py.
+
+It also never writes to the repository. `git ls-remote` is read-only and the
+CHANGELOG is fetched over HTTPS, so this cannot leave root-owned objects in .git
+the way a `git fetch` from middlewared (running as root) would.
+"""
+
+import datetime
+import importlib.util
+import logging
+import os
+import re
+import subprocess
+import urllib.request
+
+from middlewared.alert.base import (
+    Alert,
+    AlertCategory,
+    AlertClass,
+    AlertLevel,
+    ThreadedAlertSource,
+)
+from middlewared.alert.schedule import IntervalSchedule
+
+logger = logging.getLogger(__name__)
+
+PATCH_DIR = "@PATCH_DIR@"
+DISABLED_MARKER = os.path.join(PATCH_DIR, "update_alerts_disabled")
+
+_TAG_RE = re.compile(r"^v\d+\.\d+\.\d+$")
+_VERSION_RE = re.compile(r'^VERSION="([^"]+)"', re.M)
+
+#: owner/repo out of any of:
+#:   git@github.com:sudolulo/repo.git
+#:   https://github.com/sudolulo/repo.git
+#:   ssh://git@git.arch.fyi:55214/flan/repo.git
+#:   https://git.arch.fyi/flan/repo.git
+#: The SSH port is deliberately not captured: it is not the web port.
+_REMOTE_RE = re.compile(
+    r"^(?:\w+://)?(?:[^@/]+@)?([^:/]+)(?::\d+)?[:/]([^/]+)/([^/]+?)(?:\.git)?/?$"
+)
+
+_TIMEOUT = 20
+
+
+class TrueCloudPatchUpdateAlertClass(AlertClass):
+    category = AlertCategory.SYSTEM
+    level = AlertLevel.INFO
+    title = "truecloud-patch update available"
+    text = (
+        "truecloud-patch %(current)s is installed; %(latest)s is available.%(summary)s "
+        "Update with:  bash %(dir)s/update.sh"
+    )
+
+
+class TrueCloudPatchSecurityUpdateAlertClass(AlertClass):
+    category = AlertCategory.SYSTEM
+    level = AlertLevel.WARNING
+    title = "truecloud-patch security update available"
+    text = (
+        "truecloud-patch %(current)s is installed; %(latest)s contains a SECURITY "
+        "fix.%(summary)s Update with:  bash %(dir)s/update.sh"
+    )
+
+
+class TrueCloudPatchUpdateAlertSource(ThreadedAlertSource):
+    schedule = IntervalSchedule(datetime.timedelta(hours=24))
+    run_on_backup_node = False
+
+    def check_sync(self):
+        try:
+            return self._check()
+        except Exception:
+            # An alert source must never take middlewared down with it.
+            logger.debug("truecloud-patch update check failed", exc_info=True)
+            return None
+
+    # ── internals ────────────────────────────────────────────────────────────
+
+    def _git(self, *args):
+        # List form, never shell=True, and every `args` value is a literal from
+        # this file -- nothing user-supplied reaches the command line. The partial
+        # `git` path is moot: this runs as root inside middlewared, so anyone who
+        # can poison PATH already has root.
+        return subprocess.run(  # noqa: S603
+            ["git", "-C", PATCH_DIR, *args],  # noqa: S607
+            capture_output=True, text=True, timeout=_TIMEOUT, check=True,
+        ).stdout
+
+    def _check(self):
+        if os.path.exists(DISABLED_MARKER):
+            return None
+        if not os.path.isdir(os.path.join(PATCH_DIR, ".git")):
+            return None
+
+        current = self._installed_version()
+        if not current:
+            return None
+
+        latest = self._latest_release_tag()
+        if not latest:
+            return None
+
+        rn = self._release_notes()
+        if rn is None:
+            return None
+        significance, version_tuple = rn.significance, rn.version_tuple
+
+        if version_tuple(latest) <= version_tuple(current):
+            return None
+
+        level, versions, summary = self._classify(
+            current, latest, significance
+        )
+
+        # Documentation-only releases are not worth an alert. This is the whole
+        # point: nobody should get a notification because a README was reworded.
+        if level == "docs":
+            logger.debug(
+                "truecloud-patch %s -> %s is documentation-only; not alerting",
+                current, latest,
+            )
+            return None
+
+        args = {
+            "current": f"v{current}",
+            "latest": latest,
+            "summary": summary,
+            "dir": PATCH_DIR,
+        }
+        klass = (
+            TrueCloudPatchSecurityUpdateAlertClass if level == "security"
+            else TrueCloudPatchUpdateAlertClass
+        )
+        return Alert(klass, args, key=[current, latest])
+
+    def _release_notes(self):
+        """Load tools/release_notes.py by path.
+
+        NOT via sys.path: prepending would shadow the stdlib for this interpreter,
+        and this runs in middlewared's thread pool, so mutating sys.path is a race.
+        """
+        path = os.path.join(PATCH_DIR, "tools", "release_notes.py")
+        try:
+            spec = importlib.util.spec_from_file_location("_tc_release_notes", path)
+            mod = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(mod)
+            return mod
+        except Exception:
+            logger.debug("could not load release_notes", exc_info=True)
+            return None
+
+    def _installed_version(self):
+        """The version of the patch actually checked out here."""
+        try:
+            with open(os.path.join(PATCH_DIR, "patch", "apply.sh"), encoding="utf-8") as fh:
+                m = _VERSION_RE.search(fh.read())
+        except OSError:
+            return None
+        return m.group(1) if m else None
+
+    def _latest_release_tag(self):
+        """Newest plain vX.Y.Z tag on the remote. Read-only: no .git writes.
+
+        Pre-release tags (-rc, -beta) are excluded: git's version sort ranks
+        v0.5.0-rc1 above v0.5.0, so including them would advertise a release
+        candidate as the latest stable.
+        """
+        try:
+            out = self._git("ls-remote", "--tags", "--refs", "origin")
+        except Exception:
+            return None
+
+        tags = []
+        for line in out.splitlines():
+            parts = line.split("refs/tags/")
+            if len(parts) == 2 and _TAG_RE.match(parts[1].strip()):
+                tags.append(parts[1].strip())
+        if not tags:
+            return None
+
+        return max(tags, key=lambda t: tuple(int(x) for x in t.lstrip("v").split(".")))
+
+    def _classify(self, current, latest, significance):
+        """(level, versions, one-line summary). Falls back to alerting."""
+        text = self._remote_changelog(latest)
+        if text is None:
+            # Cannot tell whether it matters. Alert rather than risk hiding a
+            # security fix -- but say that we could not tell.
+            return "notable", [], " (could not read the changelog)"
+
+        level, versions, headings = significance(text, current, latest)
+        if level == "docs":
+            return level, versions, ""
+
+        seen, ordered = set(), []
+        for h in headings:
+            if h not in seen:
+                seen.add(h)
+                ordered.append(h.capitalize())
+        detail = ", ".join(ordered)
+        return level, versions, f" Changes: {detail}." if detail else ""
+
+    def _changelog_url(self, tag):
+        """Where to read CHANGELOG.md at `tag`, derived from the origin remote.
+
+        Forge-agnostic on purpose. This project is canonically hosted on Gitea and
+        mirrored to GitHub, and hard-coding either one has a nastier failure than it
+        looks: when the changelog cannot be read, _classify() falls back to
+        "notable" and alerts ANYWAY, because the alternative is silently hiding a
+        security fix. So a stale URL does not disable the alert -- it makes the
+        alert fire on every release including documentation-only ones, which is
+        precisely the nagging this whole mechanism exists to prevent.
+        """
+        try:
+            remote = self._git("remote", "get-url", "origin").strip()
+        except Exception:
+            return None
+
+        m = _REMOTE_RE.match(remote)
+        if not m:
+            return None
+
+        host, owner, repo = m.group(1), m.group(2), m.group(3)
+
+        if host.endswith("github.com"):
+            return f"https://raw.githubusercontent.com/{owner}/{repo}/{tag}/CHANGELOG.md"
+
+        # Gitea and Forgejo both serve /{owner}/{repo}/raw/tag/{tag}/{path} over the
+        # web port, which is not the SSH port the remote may name.
+        return f"https://{host}/{owner}/{repo}/raw/tag/{tag}/CHANGELOG.md"
+
+    def _remote_changelog(self, tag):
+        """CHANGELOG.md at `tag`, over HTTPS. None if it cannot be read."""
+        url = self._changelog_url(tag)
+        if not url:
+            return None
+        try:
+            with urllib.request.urlopen(url, timeout=_TIMEOUT) as resp:  # noqa: S310
+                if resp.status != 200:
+                    return None
+                return resp.read().decode("utf-8", "replace")
+        except Exception:
+            return None

patch/apply.sh

@@ -32,7 +32,7 @@
 # Derive PATCH_DIR from this script's location (parent of the patch/ directory).
 PATCH_DIR="$(cd "$(dirname "$0")/.." && pwd)"
 LOG="$PATCH_DIR/apply.log"
-VERSION="0.3.5"
+VERSION="0.7.0"
 
 # Rotate log at 512 KB to avoid unbounded growth on a system volume.
 # Keep two prior generations (.1 and .2) so the last three boots are always available.
@@ -204,7 +204,91 @@ else
     _NESTED_ENABLED=0
 fi
 
-# Is either module still doing something useful?
+# ── compatibility preflight ──────────────────────────────────────────────────
+#
+# The native probes above ask "has iX made this module unnecessary?". This asks the
+# other question, the dangerous one: "has iX changed middleware so that this module
+# no longer WORKS?"
+#
+# middlewared is internal API with no stability contract, and TrueNAS 26 rewrites
+# the entire cloud_backup path from async to synchronous. Every block the nested
+# module injects is an `async def` wrapping an `await`ed original; on 26 that hands
+# sync.py a coroutine where it unpacks a tuple. The backup does not fail cleanly --
+# it fails at the point where you needed it.
+#
+# tools/compat.py records what each module assumes and checks it against the
+# middlewared ACTUALLY INSTALLED HERE. A module whose assumptions no longer hold is
+# not applied. Stock TrueNAS without a feature beats TrueNAS with a broken one.
+#
+# Fail direction, deliberately asymmetric:
+#   * a definite "assumption violated"  -> disable that module. Strong evidence.
+#   * the checker cannot run at all     -> change nothing. That is a tooling glitch,
+#     not evidence, and turning it into a disabled module would break working boxes.
+_TC_COMPAT_JSON="$PATCH_DIR/incompatible.json"
+rm -f "$_TC_COMPAT_JSON"
+
+_tc_incompatible=0
+_tc_compat=unknown
+
+# No middlewared directory means the checker has nothing to read -- every module
+# would look "broken" because every file is missing, which is the strongest possible
+# evidence derived from the weakest possible input. Skip the preflight entirely and
+# let the existing "Cannot determine middlewared directory" path handle it.
+if [ -z "$_MW_DIR" ]; then
+    echo "NOTICE: middlewared directory unknown; skipping the compatibility preflight."
+    _tc_compat=$(printf 'unknown\nunknown\n')
+else
+_tc_compat=$("$PYTHON" - "$PATCH_DIR" "$_MW_DIR" "$_TC_COMPAT_JSON" 2>/dev/null <<'PYEOF' || printf 'unknown\nunknown\n'
+import json, os, sys
+
+patch_dir, mw_dir, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
+
+# APPEND, never insert(0) -- see the note by the mw_patch import below. Shadowing
+# the stdlib for this interpreter is a much worse failure than not finding compat.
+sys.path.append(os.path.join(patch_dir, 'tools'))
+try:
+    import compat
+    result = compat.check_tree(mw_dir)
+except Exception:
+    print('unknown')
+    print('unknown')
+    raise SystemExit(0)
+
+def verdict(r):
+    # Deliberately NOT exempting 'native' here, unlike the CI matrix.
+    #
+    # 'native' answers "do we still NEED this module?"; 'ok' answers "is it still
+    # SAFE to inject?". They are different questions, and letting native mask a
+    # broken assumption conflates them: a future TrueNAS that both reworded the
+    # nesting guard (-> native) AND changed the signatures (-> broken) would read
+    # as safe, and we would patch it anyway.
+    #
+    # Refusing to apply is the correct action for BOTH answers -- a native module
+    # is unnecessary and a broken one is dangerous -- so the apply path only has to
+    # ask whether the assumptions hold. Whether the feature went native is decided
+    # separately, by the probes above, and only affects the wording of the notice.
+    return 'ok' if r['ok'] else 'broken'
+
+broken = {m: r for m, r in result.items() if verdict(r) == 'broken'}
+if broken:
+    # The alert source reads this. Written before we print, so a box that is
+    # incompatible always has the evidence on disk even if apply.sh dies later.
+    try:
+        with open(out_path, 'w', encoding='utf-8') as fh:
+            json.dump(broken, fh, indent=2)
+    except OSError:
+        pass
+
+print(verdict(result['providers']))
+print(verdict(result['nested']))
+PYEOF
+)
+fi
+
+_tc_compat_providers=$(printf '%s' "$_tc_compat" | sed -n '1p')
+_tc_compat_nested=$(printf '%s' "$_tc_compat"    | sed -n '2p')
+
+# Is either module still doing something useful -- and can it still be applied?
 _providers_needed=1
 [ "$_tc_native_b2" = "yes" ] && _providers_needed=0
 
@@ -213,6 +297,65 @@ if [ "$_NESTED_ENABLED" = "1" ] && [ "$_tc_native_nested" != "yes" ]; then
     _nested_needed=1
 fi
 
+# The native checks above have already zeroed _*_needed for anything TrueNAS now
+# does itself, and printed the (good) news. Only complain about a module that is
+# still NEEDED and no longer fits -- otherwise a version that took a feature native
+# AND reshaped the module would be announced as "NOT COMPATIBLE", which is alarming
+# and false.
+if [ "$_tc_compat_providers" = "broken" ] && [ "$_providers_needed" = "1" ]; then
+    echo "WARNING: truecloud-patch is NOT COMPATIBLE with this TrueNAS version."
+    echo "WARNING:   The B2/S3 providers module will NOT be applied. TrueCloud is"
+    echo "WARNING:   left stock, so B2/S3 tasks will not run until this is fixed."
+    echo "WARNING:   Details: $_TC_COMPAT_JSON"
+    _providers_needed=0
+    _tc_incompatible=1
+fi
+
+if [ "$_tc_compat_nested" = "broken" ] && [ "$_nested_needed" = "1" ]; then
+    echo "WARNING: truecloud-patch's nested-snapshot module is NOT COMPATIBLE with"
+    echo "WARNING:   this TrueNAS version and will NOT be applied. Backups still"
+    echo "WARNING:   run; datasets nested under the target are not included."
+    echo "WARNING:   Details: $_TC_COMPAT_JSON"
+    _nested_needed=0
+    _tc_incompatible=1
+fi
+
+_tc_unmount_overlays() {
+    for _tag in mw ui; do
+        if mount | grep -qF "truecloud-${_tag} on "; then
+            _mnt=$(mount | grep "truecloud-${_tag} on " | awk '{print $3}' | head -1)
+            if umount "$_mnt" 2>/dev/null; then
+                echo "NOTICE: Unmounted overlay on $_mnt"
+            fi
+        fi
+    done
+}
+
+# INCOMPATIBLE is not the same as RETIRED, and must never take the same exit.
+#
+# The kill switch below is permanent -- apply.sh checks for it and returns early on
+# every future boot -- and only install.sh removes it, NOT update.sh. That is right
+# for retirement ("TrueNAS does this natively now; stop forever"), and catastrophic
+# for incompatibility: on TrueNAS 26 the providers module fails its assumptions and
+# nested is opt-out by default, so BOTH would be zero, the kill switch would fire,
+# and the very release that fixes 26 could never re-enable itself. The user would
+# run `bash update.sh` -- exactly what the update alert tells them to do -- and the
+# patch would stay dead, silently, with their B2 backups off.
+#
+# So: incompatible means "apply nothing THIS boot, and try again next boot". The
+# fix ships, update.sh checks it out, the next boot re-runs the preflight, the
+# assumptions hold, and the patch comes back by itself.
+if [ "$_tc_incompatible" = "1" ] && [ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]; then
+    echo "NOTICE: Nothing can be applied on this TrueNAS version — see the WARNINGs above."
+    echo "NOTICE: The kill switch is deliberately NOT set: this is an incompatibility,"
+    echo "NOTICE: not a retirement. Install a release that supports this TrueNAS"
+    echo "NOTICE:   bash $PATCH_DIR/update.sh"
+    echo "NOTICE: and the patch will re-apply itself on the next boot."
+    _tc_unmount_overlays
+    echo "=== done ==="
+    exit 0
+fi
+
 if [ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]; then
     echo "NOTICE: Nothing left for truecloud-patch to do:"
     [ "$_tc_native_b2" = "yes" ] && echo "NOTICE:   - TrueNAS now provides native B2 restic support."
@@ -225,12 +368,7 @@ if [ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]; then
     echo "NOTICE: Run the following to fully remove the patch:"
     echo "NOTICE:   bash $PATCH_DIR/uninstall.sh"
     touch "$PATCH_DIR/disabled"
-    for _tag in mw ui; do
-        if mount | grep -qF "truecloud-${_tag} on "; then
-            _mnt=$(mount | grep "truecloud-${_tag} on " | awk '{print $3}' | head -1)
-            umount "$_mnt" 2>/dev/null && echo "NOTICE: Unmounted overlay on $_mnt" || true
-        fi
-    done
+    _tc_unmount_overlays
     echo "=== done ==="
     exit 0
 fi
@@ -353,19 +491,62 @@ else:
 # the guard removed but the traversal missing -- that would be a silently empty
 # backup, the worst possible outcome.
 
-SNAPSHOT_BLOCK = """
+# ── nested blocks: one core, two wrappers ─────────────────────────────────────
+#
+# TrueNAS <= 25.10 has an ASYNC cloud_backup path; TrueNAS 26 rewrote it SYNCHRONOUS
+# (`middleware.call_sync` throughout, no awaits). An `async def` wrapper on 26 hands
+# sync.py a coroutine where it unpacks a tuple, and a `def` wrapper on 25.10 blocks
+# the event loop. So each block is assembled from:
+#
+#   * a CORE, written once, synchronous, using middleware.call_sync -- which is safe
+#     from a worker thread and deadlocks on the event loop; and
+#   * a WRAPPER matching the stock function's own flavour, chosen at apply time by
+#     reading whether the installed middlewared declares it `async def`.
+#
+# On <= 25.10 the async wrapper hops to a thread via `await middleware.run_in_thread`
+# -- exactly the thread call_sync needs. On 26 the stock function is already running
+# in middlewared's thread pool (its own code calls call_sync), so the sync wrapper
+# calls the core directly.
+#
+# The logic that matters -- snapshots, bind mounts, failure modes -- exists once.
+# An async twin would mean every future fix had to land twice, and the one that got
+# missed would be the one that eats a backup.
+
+_NESTED_IMPORT = """
 # TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh
 try:
     from middlewared.plugins.cloud import _truecloud_nested as _tc_nested
 except ImportError:
     _tc_nested = None
+"""
 
+SNAPSHOT_CORE = _NESTED_IMPORT + """
 if _tc_nested is not None:
     _tc_orig_create_snapshot = create_snapshot
 
-    async def create_snapshot(middleware, path, name="cloud_task-onetime"):
-        # Stock takes the (already recursive) snapshot; we only replace the PATH.
-        snapshot, snap_path = await _tc_orig_create_snapshot(middleware, path, name)
+    def _tc_stage(middleware, path, name, snapshot, snap_path):
+        # Synchronous, and always called from a worker thread (see above).
+        #
+        # ONLY cloud_backup. Bail out before touching anything otherwise.
+        #
+        # create_snapshot is module-global in plugins/cloud/snapshot.py and is
+        # imported by cloud_sync.py as well as cloud_backup/sync.py -- so this
+        # wrapper sits in the path of every rclone/Storj CloudSync task with
+        # snapshot=true, not just ours. Two consequences, and the second is worse:
+        #
+        #   * everything below is a NEW failure mode for tasks that worked before we
+        #     were installed. A `zfs list` that errors would break a CloudSync job
+        #     we have no business touching.
+        #   * if a CloudSync task ever were staged, nothing would ever tear it down:
+        #     the teardown is wired into cloud_backup's restic_backup finally, and
+        #     CRUD_BLOCK deliberately leaves CloudSync's nesting guard intact. The
+        #     bind mounts would pin the snapshot forever.
+        #
+        # cloud_backup names its snapshot "cloud_backup-<id>"; cloud_sync names it
+        # "cloud_sync-<id>"; the stock default is "cloud_task-onetime". Anything that
+        # is not ours gets stock behaviour, untouched, with no extra middleware call.
+        if not name.startswith("cloud_backup"):
+            return snapshot, snap_path
 
         _logger = getattr(middleware, "logger", None)
         try:
@@ -375,49 +556,78 @@ if _tc_nested is not None:
             # our staging plan would not -- silently omitting it from the backup.
             # Read afterwards, an unsnapshotted dataset instead trips the isdir()
             # check in plan_staging and fails the run loudly. Loud beats silent.
-            datasets = await middleware.call(
-                "zfs.dataset.query", [["type", "=", "FILESYSTEM"]]
-            )
-            dataset, nested = get_dataset_recursive(datasets, path)
+            # query_filesystems() reads ZFS directly. It deliberately does NOT use
+            # pool.dataset.query: that applies a visibility policy and hides
+            # TrueNAS-internal datasets (ix-apps/*, .system/*, .ix-virt/*) -- 84 of
+            # 270 on a real pool, including live app data. Staging from the filtered
+            # view omits them silently, which is the one thing this must never do.
+            datasets = _tc_nested.query_filesystems(middleware)
+            # OUR copy of get_dataset_recursive, not the host module's: TrueNAS 26
+            # deleted that helper (create_snapshot uses filesystem.statfs now), so
+            # calling it out of the module namespace is a NameError there.
+            dataset, nested = _tc_nested.get_dataset_recursive(datasets, path)
 
             if not nested:
-                # No children: stock behaviour, untouched. Stock's `finally` owns
-                # the snapshot from here (its non-recursive delete is correct,
-                # because a non-nested snapshot has no children).
+                # Nothing to STAGE -- but we still own the SWEEP, and that is not a
+                # formality. Stock decides `recursive` by its own rule, and on 26 that
+                # rule is no longer ours: it snapshots recursively whenever the backup
+                # path IS the dataset's mountpoint (filesystem.statfs), while
+                # get_dataset_recursive() sees nothing to stage when the only
+                # descendants are ZVOLs or legacy/none-mountpoint datasets. Stock then
+                # deletes the PARENT ONLY. Without this, one snapshot per descendant is
+                # orphaned on every run, forever, with no sidecar and no GC to find it --
+                # and the backup still reports success.
+                _tc_nested.own_snapshot(middleware, name, snapshot, logger=_logger)
                 return snapshot, snap_path
 
-            staging_root = await _tc_nested.stage_nested(
+            staging_root = _tc_nested.stage_nested(
                 middleware, path, snapshot,
                 dataset["name"], dataset["properties"]["mountpoint"]["value"],
                 name, datasets, logger=_logger,
             )
         except Exception:
             # The snapshot exists, but this exception means sync.py never completes
-            # `snapshot, local_path = await create_snapshot(...)`, so its local
-            # `snapshot` stays None and its `finally` deletes NOTHING. Sweep the
-            # tree ourselves or leak the parent plus one snapshot per descendant
-            # dataset (160+ here) on every failed run.
-            await _tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger)
+            # `snapshot, local_path = create_snapshot(...)`, so its local `snapshot`
+            # stays None and its `finally` deletes NOTHING. Sweep the tree ourselves
+            # or leak the parent plus one snapshot per descendant dataset (160+ here)
+            # on every failed run.
+            #
+            # The sweep is itself wrapped: a cleanup that raises would REPLACE the
+            # original exception with its own, hiding why the backup actually failed.
+            # An error handler must not be able to lose the error.
+            try:
+                _tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger)
+            except Exception as _tc_sweep_err:
+                if _logger:
+                    _logger.error(
+                        "truecloud-patch: could not sweep %s after a staging failure "
+                        "(%r) -- it is orphaned and must be deleted by hand",
+                        snapshot, _tc_sweep_err,
+                    )
             raise
 
         return snapshot, staging_root
+"""
+
+SNAPSHOT_ASYNC = SNAPSHOT_CORE + """
+    async def create_snapshot(middleware, path, name="cloud_task-onetime"):
+        snapshot, snap_path = await _tc_orig_create_snapshot(middleware, path, name)
+        return await middleware.run_in_thread(
+            _tc_stage, middleware, path, name, snapshot, snap_path
+        )
 
     create_snapshot._truecloud_patched = True
 """
 
-CRUD_BLOCK = """
-# TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh
-try:
-    from middlewared.plugins.cloud import _truecloud_nested as _tc_nested
-except ImportError:
-    _tc_nested = None
+SNAPSHOT_SYNC = SNAPSHOT_CORE + """
+    def create_snapshot(middleware, path, name="cloud_task-onetime"):
+        snapshot, snap_path = _tc_orig_create_snapshot(middleware, path, name)
+        return _tc_stage(middleware, path, name, snapshot, snap_path)
 
-if _tc_nested is not None:
-    _tc_orig_validate = CloudTaskServiceMixin._validate
-
-    async def _tc_validate(self, app, verrors, name, data):
-        await _tc_orig_validate(self, app, verrors, name, data)
+    create_snapshot._truecloud_patched = True
+"""
 
+_CRUD_FILTER = """
         # Only cloud_backup: staging teardown is wired into cloud_backup.sync's
         # finally. cloudsync would leak bind mounts, so leave its guard intact.
         if getattr(getattr(self, "_config", None), "namespace", "") != "cloud_backup":
@@ -438,25 +648,69 @@ if _tc_nested is not None:
     CloudTaskServiceMixin._validate._truecloud_patched = True
 """
 
-SYNC_BLOCK = """
-# TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh
-try:
-    from middlewared.plugins.cloud import _truecloud_nested as _tc_nested
-except ImportError:
-    _tc_nested = None
+CRUD_ASYNC = _NESTED_IMPORT + """
+if _tc_nested is not None:
+    _tc_orig_validate = CloudTaskServiceMixin._validate
 
+    async def _tc_validate(self, app, verrors, name, data):
+        await _tc_orig_validate(self, app, verrors, name, data)
+""" + _CRUD_FILTER
+
+CRUD_SYNC = _NESTED_IMPORT + """
+if _tc_nested is not None:
+    _tc_orig_validate = CloudTaskServiceMixin._validate
+
+    def _tc_validate(self, app, verrors, name, data):
+        _tc_orig_validate(self, app, verrors, name, data)
+""" + _CRUD_FILTER
+
+# *args/**kwargs, not the stock signature spelled out.
+#
+# 24.10 and 25.04 have `restic_backup(middleware, job, cloud_backup, dry_run)`;
+# 25.10 added `rate_limit`. Naming them and forwarding all five raised
+# `TypeError: takes 4 positional arguments but 5 were given` on every nested backup
+# on the two older releases. Forwarding whatever we were handed makes this wrapper
+# indifferent to iX adding or dropping a trailing parameter -- which they have now
+# done twice.
+#
+# Our bind mounts pin the ZFS snapshot, so stock's `finally` cannot destroy it
+# (EBUSY) and logs one benign warning. We unmount here and then delete it for real.
+SYNC_ASYNC = _NESTED_IMPORT + """
 if _tc_nested is not None:
     _tc_orig_restic_backup = restic_backup
 
-    async def restic_backup(middleware, job, cloud_backup, dry_run=False, rate_limit=None):
-        # Our bind mounts pin the ZFS snapshot, so stock's `finally` cannot
-        # destroy it (EBUSY) and logs one benign warning. We unmount here and
-        # then delete the snapshot for real.
+    async def restic_backup(middleware, job, cloud_backup, *args, **kwargs):
         try:
-            return await _tc_orig_restic_backup(middleware, job, cloud_backup, dry_run, rate_limit)
+            return await _tc_orig_restic_backup(middleware, job, cloud_backup, *args, **kwargs)
         finally:
             try:
-                await _tc_nested.cleanup_task(
+                # logger= must be passed here too, exactly as the sync variant does.
+                # run_in_thread forwards **kwargs (functools.partial), and without it
+                # cleanup_task gets logger=None -- so every teardown warning ("could
+                # not unmount X") is silently swallowed on <= 25.10, which is most
+                # boxes. The two wrappers must differ ONLY in how they reach the core.
+                await middleware.run_in_thread(
+                    _tc_nested.cleanup_task,
+                    middleware,
+                    f"cloud_backup-{cloud_backup.get('id', 'onetime')}",
+                    logger=getattr(middleware, "logger", None),
+                )
+            except Exception as e:
+                middleware.logger.warning("truecloud-patch: staging cleanup failed: %r", e)
+
+    restic_backup._truecloud_patched = True
+"""
+
+SYNC_SYNC = _NESTED_IMPORT + """
+if _tc_nested is not None:
+    _tc_orig_restic_backup = restic_backup
+
+    def restic_backup(middleware, job, cloud_backup, *args, **kwargs):
+        try:
+            return _tc_orig_restic_backup(middleware, job, cloud_backup, *args, **kwargs)
+        finally:
+            try:
+                _tc_nested.cleanup_task(
                     middleware,
                     f"cloud_backup-{cloud_backup.get('id', 'onetime')}",
                     logger=getattr(middleware, "logger", None),
@@ -559,10 +813,39 @@ else:
         if missing:
             raise FileNotFoundError('missing: ' + ', '.join(missing))
 
+        # Which flavour of cloud_backup is installed? <= 25.10 is async; TrueNAS 26
+        # rewrote it synchronous. Inject the wrapper that matches: an `async def` on
+        # 26 hands sync.py a coroutine where it unpacks a tuple, and a plain `def` on
+        # 25.10 blocks the event loop.
+        #
+        # None means the three stock functions disagree, or one could not be read.
+        # Refuse rather than guess -- a half-converted middleware is one this patch
+        # has never seen, and guessing wrong there costs a backup, not a feature.
+        # nested_src is <repo>/patch/truecloud_nested.py, so tools/ is its sibling.
+        # APPEND, never insert(0) -- shadowing the stdlib for this interpreter is a
+        # far worse failure than not finding compat.
+        sys.path.append(
+            os.path.join(os.path.dirname(os.path.dirname(nested_src)), 'tools')
+        )
+        import compat
+        _flavour = compat.async_flavour_tree(mw_dir)
+        if _flavour is None:
+            raise RuntimeError(
+                'cannot tell whether this TrueNAS cloud_backup path is async or '
+                'sync (the wrapped functions disagree, or could not be read)'
+            )
+
+        _snapshot_block = SNAPSHOT_ASYNC if _flavour else SNAPSHOT_SYNC
+        _sync_block     = SYNC_ASYNC     if _flavour else SYNC_SYNC
+        _crud_block     = CRUD_ASYNC     if _flavour else CRUD_SYNC
+
         shutil.copyfile(nested_src, nested_dst)   # 1. traversal implementation
-        patch_file(snapshot_py, SNAPSHOT_BLOCK)   # 2. build the staging tree
-        patch_file(sync_path, SYNC_BLOCK)         # 3. tear it down afterwards
-        patch_file(crud_py, CRUD_BLOCK)           # 4. ONLY NOW allow nested tasks
+        patch_file(snapshot_py, _snapshot_block)  # 2. build the staging tree
+        patch_file(sync_path, _sync_block)        # 3. tear it down afterwards
+        patch_file(crud_py, _crud_block)          # 4. ONLY NOW allow nested tasks
+
+        print('OK: cloud_backup is %s; injected the matching wrappers.'
+              % ('async (TrueNAS <= 25.10)' if _flavour else 'synchronous (TrueNAS 26+)'))
 
         nested_ok = True
         nested_detail = 'nested-dataset snapshots enabled (staging tree)'
@@ -578,6 +861,51 @@ else:
 # that is inactive (superseded, or opt-in and off) is ok -- reporting a disabled
 # opt-in feature as FAIL would make `create_task.py verify` fail on a default
 # install. `active` says whether the module is doing anything.
+# ── update-available alert ────────────────────────────────────────────────────
+# Dropped into middlewared/alert/source/, where middlewared discovers and polls it
+# natively — no cron, no timer. It only raises an alert for releases that actually
+# changed something: a docs-only release is ignored (see tools/release_notes.py).
+alert_ok = False
+alert_detail = ''
+_patch_dir = os.path.dirname(os.path.dirname(nested_src))
+alert_src = os.path.join(os.path.dirname(nested_src), 'alert_source.py')
+alert_dst = os.path.join(mw_dir, 'alert', 'source', 'truecloud_patch_update.py')
+
+if os.path.exists(os.path.join(_patch_dir, 'update_alerts_disabled')):
+    alert_detail = 'disabled (update_alerts_disabled)'
+    try:
+        os.unlink(alert_dst)
+        print('OK: Removed update alert (disabled).')
+    except OSError:
+        pass
+elif not os.path.exists(alert_src):
+    alert_detail = 'alert_source.py not found'
+    print(f'WARNING: {alert_src} missing — no update alert.')
+else:
+    try:
+        with open(alert_src, encoding='utf-8') as fh:
+            _body = fh.read()
+
+        # repr() so ANY path becomes a valid Python literal -- a directory
+        # containing a quote or backslash would otherwise produce a syntax error.
+        _body = _body.replace('"@PATCH_DIR@"', repr(_patch_dir))
+
+        # COMPILE BEFORE WRITING. middlewared's alert.load() imports every file in
+        # alert/source/ with NO try/except, and it runs at startup -- a module that
+        # raises on import takes middlewared's setup down with it. An uninstalled
+        # alert is a missing convenience; a broken one is a broken box.
+        compile(_body, alert_dst, 'exec')
+
+        with open(alert_dst, 'w', encoding='utf-8') as fh:
+            fh.write(_body)
+        alert_ok = True
+        alert_detail = 'update alert installed'
+        print(f'OK: Installed update alert → {alert_dst}')
+    except Exception as e:
+        alert_detail = f'not applied: {e}'
+        print(f'WARNING: could not install update alert: {e}')
+        print('WARNING: no update alert; everything else is unaffected.')
+
 patches = {
     'providers': {
         'ok': (not providers_needed) or bool(b2_ok and restic_ok),
@@ -589,6 +917,11 @@ patches = {
         'active': nested_needed,
         'detail': nested_detail,
     },
+    'update_alert': {
+        'ok': True,          # never a failure: it is a convenience, not a patch
+        'active': alert_ok,
+        'detail': alert_detail,
+    },
 }
 payload = {'patched_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'patches': patches}
 tmp = status_path + '.tmp'

patch/create_task.py

@@ -52,7 +52,7 @@ import subprocess
 import sys
 import time
 
-__version__ = "0.3.5"
+__version__ = "0.7.0"
 
 _PATCH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 _STATUS_FILE = os.path.join(_PATCH_DIR, "hook_status.json")

patch/mw_patch.py

@@ -37,6 +37,10 @@ NESTED_RELPATHS = [
 #: The importable module the nested blocks depend on.
 NESTED_MODULE = ("plugins", "cloud", "_truecloud_nested.py")
 
+#: The update-available alert source. Not a "patch" (it appends nothing to a stock
+#: file), but it is a file we install into middlewared and must therefore remove.
+ALERT_MODULE = ("alert", "source", "truecloud_patch_update.py")
+
 
 def patch_file(path, block):
     """Append `block`, replacing any block we appended before. Idempotent."""
@@ -101,8 +105,14 @@ def revert_nested(mw_dir):
 
 
 def revert_all(mw_dir):
-    """Undo every patch this project applies."""
-    return revert(mw_dir, NESTED_RELPATHS + PROVIDER_RELPATHS, NESTED_MODULE)
+    """Undo every patch this project applies, and remove every file it installs."""
+    reverted = revert(mw_dir, NESTED_RELPATHS + PROVIDER_RELPATHS, NESTED_MODULE)
+    try:
+        os.unlink(os.path.join(mw_dir, *ALERT_MODULE))
+        reverted.append(ALERT_MODULE[-1])
+    except OSError:
+        pass
+    return reverted
 
 
 def find_middlewared_dir():

recover.sh

@@ -17,7 +17,7 @@
 #   bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh
 #   systemctl restart middlewared
 
-VERSION="0.3.5"
+VERSION="0.7.0"
 
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 

release.sh

@@ -0,0 +1,332 @@
+#!/usr/bin/env bash
+# Cut a release. Two stages, and you cannot skip the first one.
+#
+#   bash release.sh 0.6.0 --rc        stage 1: candidate. Invisible to users.
+#   bash release.sh 0.6.0 --promote   stage 2: stable. Only if an rc passed HERE.
+#
+# WHY IT WORKS THIS WAY
+#
+# This repo once cut twelve releases in a day, several of them fixing the release
+# before. Every one of those raises an update alert on every user's box. An alert
+# people learn to ignore is worse than no alert, because one day it will be
+# carrying a security fix.
+#
+# So: debugging happens across rc1, rc2, rc3 -- which update.sh and the alert
+# source both filter out, so no user ever sees them -- and a stable tag is only
+# reachable from a candidate that already went green on the identical commit.
+# tools/release_gate.py enforces that here, and .github/workflows/release.yml
+# enforces it again where it cannot be bypassed.
+#
+# Day to day you do not touch this script. You write your changes under
+# `## Unreleased` in CHANGELOG.md and push to main. Releasing is a separate,
+# deliberate act.
+
+set -euo pipefail
+
+# This file IS on every user's box -- update.sh clones the whole repo -- so it
+# carries no VERSION= not because it is "not shipped", but because nothing reads
+# it. VERSION= exists so the running system can say which patch it is; this script
+# never runs on a running system. (Anything that DOES carry a VERSION= must be in
+# release_notes.VERSIONED_FILES or it silently rots: create_task.py sat three
+# releases behind for exactly that reason.)
+#
+# Running it on a user's box is a no-op by construction, and that is checked below
+# rather than left to luck: update.sh pins the checkout to a tag in detached HEAD,
+# and this refuses to run anywhere but an up-to-date `main` with push access.
+
+cd "$(dirname "$(readlink -f "$0")")"
+
+die() { printf '\033[31merror:\033[0m %s\n' "$*" >&2; exit 1; }
+note() { printf '\033[36m==>\033[0m %s\n' "$*"; }
+ok() { printf '\033[32m ok\033[0m %s\n' "$*"; }
+
+usage() {
+  sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'
+  exit "${1:-0}"
+}
+
+# ── args ─────────────────────────────────────────────────────────────────────
+
+target=""
+mode=""
+assume_yes=0
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --rc)       mode="rc" ;;
+    --promote)  mode="promote" ;;
+    --check)    mode="check" ;;
+    -y|--yes)   assume_yes=1 ;;
+    -h|--help)  usage 0 ;;
+    -*)         die "unknown option: $1" ;;
+    *)
+      [ -n "$target" ] && die "give exactly one version"
+      target="${1#v}"
+      ;;
+  esac
+  shift
+done
+
+[ -n "$target" ] || usage 2
+[ -n "$mode" ] || die "pick a stage: --rc (candidate) or --promote (stable)"
+
+printf '%s' "$target" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$' \
+  || die "version must be plain X.Y.Z (the -rcN suffix is added for you)"
+
+# ── preflight ────────────────────────────────────────────────────────────────
+
+[ -d .git ] || die "not a git checkout"
+
+# UNTRACKED files count as dirty, because --rc runs `git add -A`: an untracked file
+# lying around would be swept into the release commit and pushed. This checkout is
+# routinely shared between sessions, so stray files are the normal state here, not
+# an exotic one. (Anything genuinely ignorable belongs in .gitignore.)
+if [ -n "$(git status --porcelain)" ]; then
+  echo "  Working tree is not clean:" >&2
+  git status --short >&2
+  die "commit, stash, or ignore the above first -- a release must be reproducible
+       from a commit, not from whatever happened to be on disk. --rc runs
+       'git add -A', so an untracked file here ships inside the release."
+fi
+
+branch="$(git rev-parse --abbrev-ref HEAD)"
+if [ "$branch" = "HEAD" ]; then
+  # This is what an INSTALLED patch looks like: update.sh pins the checkout to a
+  # release tag in detached HEAD. Someone has found this script in their clone and
+  # run it. Say so plainly rather than emitting a confusing branch error.
+  die "this is an installed checkout (detached at $(git describe --tags --always)),
+       not a development one. release.sh is the maintainer tool that publishes new
+       versions of the patch; it is not how you install or update one.
+
+       To update this box:   bash update.sh"
+fi
+[ "$branch" = "main" ] || die "releases are cut from main, not '$branch'"
+
+note "fetching tags"
+git fetch --tags --quiet origin
+
+if [ -n "$(git log --oneline "origin/$branch..$branch" 2>/dev/null)" ]; then
+  die "local main has commits that are not pushed. Push first: the tag must point
+       at a commit the world can actually fetch."
+fi
+
+# BEHIND is just as bad as ahead, and less obvious. --rc commits the version stamp,
+# creates the tag, and only THEN pushes -- so on a stale main the push is rejected
+# (non-fast-forward) *after* the tag exists and `## Unreleased` has already been
+# consumed. Re-running then sees rc1, cuts rc2, and silently skips the stamping
+# step; the rc1 tag dangles locally forever.
+if [ -n "$(git log --oneline "$branch..origin/$branch" 2>/dev/null)" ]; then
+  die "local main is BEHIND origin. Pull first:  git pull --ff-only
+       Releasing from a stale main half-completes: the tag is cut locally, the push
+       is rejected, and '## Unreleased' has already been consumed."
+fi
+
+# ── the gates: identical to the ones CI will run ─────────────────────────────
+
+run_gates() {
+  local tag="$1"
+  note "gate: versions agree, CHANGELOG is complete"
+  python3 tools/release_notes.py check "$tag" \
+    || die "content gate failed (see above)"
+  ok "content"
+
+  note "gate: provenance"
+  python3 tools/release_gate.py "$tag" -C . \
+    || die "provenance gate failed (see above)"
+  ok "provenance"
+}
+
+# ── tests, because a tag that fails its own tests is not a release ───────────
+
+# The interpreter that actually has the dev deps. A bare `python3` is usually the
+# system one with no pytest -- and "No module named pytest" would read as "tests
+# fail", i.e. the gate blocking a release for a reason that is not true.
+PY=python3
+[ -x .venv/bin/python ] && PY=.venv/bin/python
+
+RUFF=""
+if [ -x .venv/bin/ruff ]; then RUFF=.venv/bin/ruff
+elif command -v ruff >/dev/null 2>&1; then RUFF=ruff
+fi
+
+run_tests() {
+  note "running the suite"
+  "$PY" -c 'import pytest' 2>/dev/null || die "no pytest in $PY. Install the dev deps:
+       python3 -m venv .venv && .venv/bin/pip install pytest ruff"
+  "$PY" -m pytest tests -q || die "tests fail. Fix them; do not release around them."
+  [ -n "$RUFF" ] && { "$RUFF" check patch tests tools || die "lint fails"; }
+  local f
+  while IFS= read -r f; do
+    bash -n "$f" || die "bash syntax error in $f"
+  done < <(find . -name '*.sh' -not -path './.git/*')
+  ok "suite"
+}
+
+confirm() {
+  [ "$assume_yes" -eq 1 ] && return 0
+  printf '\n%s [y/N] ' "$1"
+  read -r reply </dev/tty
+  case "$reply" in [yY]*) return 0 ;; *) die "aborted" ;; esac
+}
+
+# ── check ────────────────────────────────────────────────────────────────────
+
+if [ "$mode" = "check" ]; then
+  echo
+  python3 - "$target" <<'PY'
+import sys, os
+sys.path.insert(0, os.path.join(os.getcwd(), "tools"))
+from release_notes import unreleased_body
+with open("CHANGELOG.md", encoding="utf-8") as fh:
+    body = unreleased_body(fh.read())
+if body:
+    print("Unreleased, and would ship as v%s:\n" % sys.argv[1])
+    print("\n".join("    " + line for line in body.splitlines()))
+else:
+    print("Nothing under `## Unreleased`. There is nothing to release.")
+PY
+  echo
+  next_rc="$(python3 tools/release_gate.py "$target" --next-rc -C .)"
+  echo "Next candidate would be: $next_rc"
+  exit 0
+fi
+
+# ── stage 1: release candidate ───────────────────────────────────────────────
+
+if [ "$mode" = "rc" ]; then
+  tag="$(python3 tools/release_gate.py "$target" --next-rc -C .)"
+
+  # Tests BEFORE the stamping commit, deliberately.
+  #
+  # Stamping consumes `## Unreleased` and makes a "release vX.Y.Z" commit. If the
+  # suite then failed, that commit was already on main and a re-run died inside
+  # promote() with "no `## Unreleased` content" -- the release was wedged, and the
+  # only way out was to hand-unpick a commit. Failing first leaves the tree
+  # untouched.
+  run_tests
+
+  # The first candidate promotes `## Unreleased` and stamps the version into every
+  # script. Later candidates (rc2+) are re-cuts of an already-stamped version, so
+  # they only tag -- the CHANGELOG section for this version already exists, and
+  # fixes found during rc go into it.
+  #
+  # "Already stamped" is decided by the TREE, not by the rc1 tag: if a previous run
+  # stamped and committed but died before tagging (or before pushing), the tag is
+  # absent while the stamp is present, and re-stamping would try to promote an
+  # `## Unreleased` section that is no longer there.
+  if python3 tools/release_notes.py check "v$target-rc0" >/dev/null 2>&1; then
+    note "v$target is already stamped; cutting a follow-up candidate"
+  else
+    note "promoting '## Unreleased' -> v$target and stamping the scripts"
+    python3 - "$target" <<'PY'
+import datetime, os, re, sys
+sys.path.insert(0, os.path.join(os.getcwd(), "tools"))
+from release_notes import VERSIONED_FILES, promote
+
+version = sys.argv[1]
+today = datetime.date.today().isoformat()
+
+with open("CHANGELOG.md", encoding="utf-8") as fh:
+    text = fh.read()
+try:
+    out = promote(text, version, today)
+except ValueError as e:
+    sys.exit(f"error: {e}")
+with open("CHANGELOG.md", "w", encoding="utf-8") as fh:
+    fh.write(out)
+print(f"    CHANGELOG.md  Unreleased -> v{version} - {today}")
+
+for rel in VERSIONED_FILES:
+    with open(rel, encoding="utf-8") as fh:
+        src = fh.read()
+    new, n = re.subn(
+        r'^(VERSION=|__version__\s*=\s*)"[^"]+"',
+        lambda m: f'{m.group(1)}"{version}"',
+        src, count=1, flags=re.M,
+    )
+    if not n:
+        sys.exit(f"error: {rel} has no VERSION= line to stamp")
+    if new != src:
+        with open(rel, "w", encoding="utf-8") as fh:
+            fh.write(new)
+        print(f"    {rel}  -> {version}")
+PY
+    git add -A
+    git commit -q -m "release v$target"
+    ok "stamped"
+  fi
+
+  # Gated as the rc tag it is: content is checked against the base version, and the
+  # provenance gate is a no-op for candidates -- being one is the whole point.
+  # (run_tests already ran, above, before anything was committed.)
+  run_gates "$tag"
+
+  echo
+  note "about to cut $tag"
+  echo "    commit:  $(git rev-parse --short HEAD)  $(git log -1 --format=%s)"
+  echo
+  echo "    A candidate is invisible to users: update.sh and the update alert both"
+  echo "    ignore -rc tags. Install it on a real box, exercise it, and only then"
+  echo "    run:  bash release.sh $target --promote"
+  confirm "cut $tag?"
+
+  # Push the branch FIRST. If it is rejected, no tag has been created yet -- a tag
+  # pointing at a commit nobody else has is worse than no tag, because the next run
+  # sees it, counts it as a candidate, and cuts rc2 against a commit that was never
+  # published.
+  git push --quiet origin main
+  git tag -a "$tag" -m "$tag"
+  git push --quiet origin "$tag"
+  ok "pushed $tag"
+  echo
+  echo "CI is now testing $tag and publishing it as a PRE-RELEASE."
+  echo "When you are satisfied:  bash release.sh $target --promote"
+  exit 0
+fi
+
+# ── stage 2: promote to stable ───────────────────────────────────────────────
+
+if [ "$mode" = "promote" ]; then
+  tag="v$target"
+
+  if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then
+    next="$(echo "$target" | awk -F. '{printf "%d.%d.%d", $1, $2, $3+1}')"
+    die "$tag already exists. A published version is immutable -- if it is broken,
+       the fix ships as v$next, and it goes through a candidate like everything else."
+  fi
+
+  # The barrier. Fails unless an rc points at THIS commit.
+  note "gate: was this exact commit a release candidate?"
+  python3 tools/release_gate.py "$tag" -C . || {
+    echo
+    die "not promotable (see above)"
+  }
+  ok "provenance"
+
+  run_gates "$tag"
+  run_tests
+
+  rcs="$(python3 - "$target" <<'PY'
+import os, sys
+sys.path.insert(0, os.path.join(os.getcwd(), "tools"))
+from release_gate import rc_tags
+print(", ".join(rc_tags(sys.argv[1])) or "none")
+PY
+)"
+
+  echo
+  note "about to publish $tag to every user"
+  echo "    commit:      $(git rev-parse --short HEAD)"
+  echo "    candidates:  $rcs"
+  echo
+  echo "    This raises an update alert on every installed box (unless the only"
+  echo "    CHANGELOG section is Docs). Make sure it is worth interrupting people."
+  confirm "publish $tag?"
+
+  git tag -a "$tag" -m "$tag"
+  git push --quiet origin "$tag"
+  ok "pushed $tag"
+  echo
+  echo "CI is publishing the release. Users will be alerted within 24h."
+  exit 0
+fi

tests/test_alert_source.py

@@ -0,0 +1,132 @@
+"""Tests for the update-available alert source.
+
+This file is imported by middlewared's `alert.load()`, which runs at STARTUP and
+has **no try/except**:
+
+    def load(self):
+        for module in load_modules(.../alert/source):
+            for cls in load_classes(module, AlertSource, (ThreadedAlertSource,)):
+                source = cls(self.middleware)
+                if source.name in ALERT_SOURCES:
+                    raise RuntimeError(...)
+
+So a module that raises on import takes middlewared's setup down with it. These
+tests guard the realistic ways that could happen.
+"""
+
+import ast
+import os
+import re
+
+import pytest
+
+ALERT_SRC = os.path.join(os.path.dirname(__file__), "..", "patch", "alert_source.py")
+APPLY_SH = os.path.join(os.path.dirname(__file__), "..", "patch", "apply.sh")
+
+
+def source():
+    with open(ALERT_SRC, encoding="utf-8") as fh:
+        return fh.read()
+
+
+def tree():
+    return ast.parse(source())
+
+
+class TestCannotBreakMiddlewaredAtImport:
+    def test_it_compiles(self):
+        compile(source(), "alert_source.py", "exec")
+
+    def test_apply_sh_compiles_it_before_writing_it(self):
+        # The substituted file is what middlewared imports. If it does not compile,
+        # installing it would break startup — so apply.sh must refuse to write it.
+        with open(APPLY_SH, encoding="utf-8") as fh:
+            sh = fh.read()
+        i = sh.index("alert_dst = os.path.join(mw_dir, 'alert', 'source'")
+        block = sh[i:i + 1600]
+        assert "compile(_body, alert_dst, 'exec')" in block
+        assert block.index("compile(_body") < block.index("open(alert_dst, 'w'")
+
+    def test_patch_dir_is_substituted_with_repr(self):
+        # A directory containing a quote or backslash would otherwise produce a
+        # syntax error in the installed module.
+        with open(APPLY_SH, encoding="utf-8") as fh:
+            sh = fh.read()
+        assert "_body.replace('\"@PATCH_DIR@\"', repr(_patch_dir))" in sh
+
+    @pytest.mark.parametrize("path", [
+        "/mnt/tank/patch",
+        '/mnt/we"ird/patch',      # a quote in the path
+        "/mnt/back\\slash/patch",  # a backslash
+    ])
+    def test_substituted_module_compiles_for_awkward_paths(self, path):
+        body = source().replace('"@PATCH_DIR@"', repr(path))
+        compile(body, "alert_source.py", "exec")  # must not raise
+
+    def test_no_io_at_module_import_time(self):
+        # Anything at module scope runs during alert.load(). Only imports,
+        # constants and class definitions are allowed.
+        allowed = (ast.Import, ast.ImportFrom, ast.Assign, ast.AnnAssign,
+                   ast.ClassDef, ast.FunctionDef, ast.Expr)
+        for node in tree().body:
+            assert isinstance(node, allowed), f"module-level {type(node).__name__}"
+            if isinstance(node, ast.Expr):
+                assert isinstance(node.value, ast.Constant), "only the docstring"
+
+
+class TestAlertClassNaming:
+    """middlewared's AlertClassMeta raises NameError unless the name ends in
+    'AlertClass' — at import, inside alert.load(), which has no try/except."""
+
+    def alert_classes(self):
+        return [n for n in tree().body
+                if isinstance(n, ast.ClassDef)
+                and any(getattr(b, "id", "") == "AlertClass" for b in n.bases)]
+
+    def test_there_are_alert_classes(self):
+        assert self.alert_classes()
+
+    def test_every_alert_class_name_ends_in_AlertClass(self):
+        for cls in self.alert_classes():
+            assert cls.name.endswith("AlertClass"), (
+                f"{cls.name}: AlertClassMeta raises NameError on this"
+            )
+
+    def test_every_alert_class_defines_the_required_attrs(self):
+        # category/level/title are NotImplemented on the base; a missing one shows
+        # up as a broken alert rather than an error.
+        for cls in self.alert_classes():
+            names = {t.id for n in cls.body if isinstance(n, ast.Assign)
+                     for t in n.targets if isinstance(t, ast.Name)}
+            assert {"category", "level", "title", "text"} <= names, cls.name
+
+    def test_alert_text_placeholders_match_the_args_we_pass(self):
+        src = source()
+        placeholders = set(re.findall(r"%\((\w+)\)s", src))
+        # These are the keys built in _check().
+        assert placeholders <= {"current", "latest", "summary", "dir"}
+
+
+class TestNoSysPathMutation:
+    def test_release_notes_is_loaded_by_path_not_sys_path(self):
+        # sys.path.insert(0, ...) would shadow the stdlib for this interpreter, and
+        # ThreadedAlertSource runs in a thread pool — mutating sys.path is a race.
+        #
+        # Check for actual MUTATION, not the string: the docstring legitimately
+        # mentions sys.path to explain why it is avoided.
+        src = source()
+        assert "sys.path.insert" not in src
+        assert "sys.path.append" not in src
+        assert not re.search(r"^import sys$", src, re.M), "sys is not needed"
+        assert "spec_from_file_location" in src
+
+
+class TestNeverWritesToGit:
+    def test_only_read_only_git_commands(self):
+        # A `git fetch` from middlewared (running as root) would leave root-owned
+        # objects in .git and break every later non-root git command — which is
+        # exactly the breakage this project already hit once.
+        src = source()
+        for forbidden in ("fetch", "pull", "checkout", "clone", "reset"):
+            assert f'"{forbidden}"' not in src, f"git {forbidden} writes to .git"
+        assert '"ls-remote"' in src

tests/test_apply_blocks.py

@@ -14,14 +14,27 @@ import pytest
 
 APPLY_SH = os.path.join(os.path.dirname(__file__), "..", "patch", "apply.sh")
 
+#: Every block that is actually injected into a middlewared module.
+#:
+#: The three nested blocks come in two flavours. TrueNAS <= 25.10 has an ASYNC
+#: cloud_backup path; TrueNAS 26 rewrote it synchronous. apply.sh reads which one is
+#: installed and injects the matching wrapper -- an `async def` on 26 would hand
+#: sync.py a coroutine where it unpacks a tuple, and a plain `def` on 25.10 would
+#: block the event loop. Both flavours must therefore be valid Python, always.
 EXPECTED_BLOCKS = {
     "B2_BLOCK",
     "RESTIC_BLOCK",
-    "SNAPSHOT_BLOCK",
-    "CRUD_BLOCK",
-    "SYNC_BLOCK",
+    "SNAPSHOT_ASYNC",
+    "SNAPSHOT_SYNC",
+    "CRUD_ASYNC",
+    "CRUD_SYNC",
+    "SYNC_ASYNC",
+    "SYNC_SYNC",
 }
 
+NESTED_BLOCKS = ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC", "CRUD_ASYNC", "CRUD_SYNC",
+                 "SYNC_ASYNC", "SYNC_SYNC"]
+
 
 def heredoc_source():
     with open(APPLY_SH, encoding="utf-8") as fh:
@@ -32,18 +45,30 @@ def heredoc_source():
 
 
 def extract_blocks():
+    """The blocks as apply.sh actually builds them.
+
+    EVALUATED, not read off as string literals: each nested block is a CORE
+    concatenated with a flavour-specific wrapper, so reading only `ast.Constant`
+    would silently return nothing for them -- a green suite over blocks nobody
+    checked. Assignments that need the runtime (argv, imports) simply fail to
+    evaluate and are skipped.
+    """
     tree = ast.parse(heredoc_source())
-    blocks = {}
-    for node in ast.walk(tree):
-        if isinstance(node, ast.Assign):
-            for tgt in node.targets:
-                if (
-                    isinstance(tgt, ast.Name)
-                    and tgt.id.endswith("_BLOCK")
-                    and isinstance(node.value, ast.Constant)
-                    and isinstance(node.value.value, str)
-                ):
-                    blocks[tgt.id] = node.value.value
+    ns, blocks = {}, {}
+    for node in tree.body:
+        if not isinstance(node, ast.Assign):
+            continue
+        try:
+            value = eval(  # noqa: S307 - our own shipped source, on purpose
+                compile(ast.Expression(node.value), "<blocks>", "eval"), {}, ns
+            )
+        except Exception:
+            continue
+        for tgt in node.targets:
+            if isinstance(tgt, ast.Name) and isinstance(value, str):
+                ns[tgt.id] = value
+                if tgt.id in EXPECTED_BLOCKS:
+                    blocks[tgt.id] = value
     return blocks
 
 
@@ -98,7 +123,7 @@ def test_injected_block_carries_the_idempotency_marker(name):
     assert extract_blocks()[name].lstrip("\n").startswith("# TRUECLOUD_PATCH")
 
 
-@pytest.mark.parametrize("name", ["SNAPSHOT_BLOCK", "CRUD_BLOCK", "SYNC_BLOCK"])
+@pytest.mark.parametrize("name", NESTED_BLOCKS)
 def test_nested_blocks_degrade_safely_without_the_module(name):
     # If _truecloud_nested failed to install, every nested block must no-op.
     # Critically this includes CRUD_BLOCK: relaxing the guard without the
@@ -115,24 +140,66 @@ class TestSnapshotLeak:
     every path that creates one must also sweep the whole tree.
     """
 
-    def test_staging_failure_deletes_the_snapshot_tree(self):
-        # On a staging failure, sync.py's `snapshot, local_path = await
-        # create_snapshot(...)` never completes, so its local `snapshot` stays
-        # None and its finally deletes nothing. We must sweep it ourselves.
-        block = extract_blocks()["SNAPSHOT_BLOCK"]
-        assert "except Exception:" in block
-        assert "delete_snapshot_tree" in block
-        assert "raise" in block
+    # The behaviour these once asserted as substrings -- the sweep, the re-raise, the
+    # teardown in the finally -- is now asserted STRUCTURALLY, against the parsed
+    # block: see TestTheStagingFailurePathReallyReRaises and
+    # TestTheSyncBlockAlwaysTearsDown. As substring checks they were satisfied by
+    # COMMENTS ("a cleanup that raises...", "cleanup_task gets logger=None"), so
+    # deleting the actual `raise` and the actual cleanup call both left the suite
+    # green -- reinstating a silently-empty backup and ~250 orphans per run.
 
-    def test_sync_block_cleans_up_on_every_path(self):
-        block = extract_blocks()["SYNC_BLOCK"]
-        assert "finally:" in block
-        assert "cleanup_task" in block
+    def test_the_snapshot_block_still_owns_the_snapshot_when_not_staging(self):
+        # The TrueNAS 26 zvol/legacy orphan: stock decides `recursive` by its own rule
+        # (path == mountpoint) and deletes only the parent, so we must record the
+        # snapshot even on the path where we stage nothing.
+        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
+            stage = functions(tree_of(name), "_tc_stage")[0]
+            assert calls_to(stage, "_tc_nested.own_snapshot"), (
+                f"{name} hands an unstaged snapshot back to stock, whose delete is "
+                f"non-recursive -- every zvol/legacy child is orphaned, every run"
+            )
+
+    def test_the_staging_plan_is_enumerated_from_ZFS(self):
+        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
+            stage = functions(tree_of(name), "_tc_stage")[0]
+            assert calls_to(stage, "_tc_nested.query_filesystems"), (
+                "the staging plan must come from query_filesystems() (which reads ZFS "
+                "unfiltered); middleware's query hides ix-apps/*, .system/*, .ix-virt/*"
+            )
+            assert not calls_to(stage, "middleware.call_sync"), (
+                "the block calls middleware directly again -- its dataset/snapshot "
+                "queries are FILTERED and silently omit 84 of 270 datasets"
+            )
+
+    def test_the_vendored_helper_is_used_not_the_host_module(self):
+        # TrueNAS 26 DELETED get_dataset_recursive from plugins/cloud/snapshot.py, so
+        # calling it out of the host module's namespace is a NameError there.
+        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
+            stage = functions(tree_of(name), "_tc_stage")[0]
+            assert calls_to(stage, "_tc_nested.get_dataset_recursive"), (
+                "must call OUR vendored copy: TrueNAS 26 deleted the host's"
+            )
+
+    def test_datasets_are_enumerated_AFTER_the_snapshot(self):
+        # A dataset created between the listing and the snapshot would be captured by
+        # the recursive snapshot but missing from the staging plan -- silently omitted.
+        # Read afterwards, it instead trips plan_staging's probe and fails loudly.
+        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
+            src = extract_blocks()[name]
+            code = "\n".join(
+                ln for ln in src.splitlines() if not ln.lstrip().startswith("#")
+            )
+            # _tc_stage receives `snapshot` as a parameter -- i.e. it is taken by the
+            # caller, before any of this runs. If the enumeration ever moves ahead of
+            # create_snapshot it can only do so by leaving _tc_stage.
+            assert "def _tc_stage(middleware, path, name, snapshot, snap_path)" in code
+            assert "query_filesystems" in code
 
 
 def test_crud_block_is_scoped_to_cloud_backup():
     # cloudsync has no staging teardown wired in, so its guard must stay.
-    assert '!= "cloud_backup"' in extract_blocks()["CRUD_BLOCK"]
+    for name in ("CRUD_ASYNC", "CRUD_SYNC"):
+        assert '!= "cloud_backup"' in extract_blocks()[name]
 
 
 class TestIndependentModules:
@@ -220,7 +287,7 @@ class TestIndependentModules:
         # find the string in our own patch and never detect native support.
         sh = self._sh()
         assert "split('\\n# TRUECLOUD_PATCH', 1)[0]" in sh
-        assert "no further nesting" in extract_blocks()["CRUD_BLOCK"], (
+        assert "no further nesting" in extract_blocks()["CRUD_ASYNC"], (
             "if this ever stops being true, the probe comment is stale"
         )
 
@@ -264,7 +331,7 @@ class TestOptIn:
         # The guard-relaxing crud.py patch must be inside the enabled branch.
         src = heredoc_source()
         gate = src.index("if not nested_needed:")
-        crud = src.index("patch_file(crud_py, CRUD_BLOCK)")
+        crud = src.index("patch_file(crud_py, _crud_block)")
         assert gate < crud, "crud.py patch must sit inside the opt-in branch"
 
     def test_disabling_REVERTS_the_patch_rather_than_merely_skipping_it(self):
@@ -282,7 +349,7 @@ class TestOptIn:
         assert "from mw_patch import patch_file, revert_nested" in src
         gate = src.index("if not nested_needed:")
         revert = src.index("reverted = revert_nested(")
-        patch = src.index("patch_file(crud_py, CRUD_BLOCK)")
+        patch = src.index("patch_file(crud_py, _crud_block)")
         assert gate < revert < patch, "revert belongs in the not-needed branch"
 
     def test_import_failure_skips_the_patch_rather_than_crashing(self):
@@ -302,8 +369,385 @@ def test_guard_is_relaxed_only_after_traversal_is_installed():
     src = heredoc_source()
     order = [
         src.index("shutil.copyfile(nested_src, nested_dst)"),
-        src.index("patch_file(snapshot_py, SNAPSHOT_BLOCK)"),
-        src.index("patch_file(sync_path, SYNC_BLOCK)"),
-        src.index("patch_file(crud_py, CRUD_BLOCK)"),
+        src.index("patch_file(snapshot_py, _snapshot_block)"),
+        src.index("patch_file(sync_path, _sync_block)"),
+        src.index("patch_file(crud_py, _crud_block)"),
     ]
     assert order == sorted(order), "crud.py must be patched last"
+
+
+class TestWrappersDoNotHardcodeStockArity:
+    """iX changes the tail of these signatures between releases.
+
+    SYNC_BLOCK used to spell out `(middleware, job, cloud_backup, dry_run, rate_limit)`
+    and forward all five. But 24.10 and 25.04 declare only four -- `rate_limit` arrived
+    in 25.10 -- so every nested backup on those two releases raised
+    `TypeError: restic_backup() takes 4 positional arguments but 5 were given`.
+    It shipped broken and nothing noticed, because the compat check at the time only
+    asked whether the parameter NAMES still appeared somewhere in the signature.
+
+    Forwarding *args/**kwargs makes the wrapper indifferent to a trailing parameter
+    being added or dropped, which is the only part iX actually churns.
+    """
+
+    def test_restic_backup_forwards_rather_than_naming_stock_params(self):
+        block = extract_blocks()["SYNC_ASYNC"]
+        assert "async def restic_backup(middleware, job, cloud_backup, *args, **kwargs)" in block
+        assert "_tc_orig_restic_backup(middleware, job, cloud_backup, *args, **kwargs)" in block
+
+        # Comments stripped: the block's own commentary explains the rate_limit
+        # history, and that must not be mistaken for the code re-declaring it.
+        code = "\n".join(
+            line for line in block.splitlines()
+            if not line.lstrip().startswith("#")
+        )
+        assert "rate_limit" not in code, (
+            "naming a trailing stock parameter re-introduces the arity bug"
+        )
+
+
+class TestTheTwoNativeProbesCannotDrift:
+    """The split-literal trick is implemented TWICE: inline in apply.sh's probe, and
+    as compat._squash. It has already caused one silent bug.
+
+    Stock middleware writes the guard as an implicitly-concatenated literal, so the
+    contiguous phrase never appears in the source. A naive search finds nothing,
+    concludes iX removed the guard, and reports "native" -- which means "retire the
+    module". That would disable nested snapshots on every box that depends on them.
+
+    apply.sh (runtime, on the box) and compat.py (static, in CI) must therefore agree
+    on every input, or one of them is wrong about whether to retire a module.
+    """
+
+    CASES = [
+        # (crud.py source, expected native?)
+        ("verrors.add('x', 'datasets that have no further nesting')", False),
+        # THE case: split across adjacent literals, as stock actually writes it.
+        ("verrors.add('x', 'datasets that have no further '\n"
+         "                 'nesting')", False),
+        ('verrors.add("x", "no further "\n              "nesting")', False),
+        # Guard genuinely gone -> iX implemented it -> native.
+        ("verrors.add('x', 'some other validation entirely')", True),
+        ("", True),
+    ]
+
+    @pytest.mark.parametrize("src,expect_native", CASES)
+    def test_both_probes_agree(self, src, expect_native):
+        import sys as _sys
+        _sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
+        import compat
+
+        shipped = _nested_native_detector()(src)
+        assert (shipped == "yes") == expect_native, (
+            f"apply.sh's probe says native={shipped!r} for {src!r}"
+        )
+
+        path, phrase, native_when_present = compat.NATIVE_PROBES[compat.NESTED]
+        present = compat._squash(phrase) in compat._squash(src)
+        static_native = (present == native_when_present)
+        assert static_native == expect_native, (
+            f"compat.py says native={static_native} for {src!r}"
+        )
+
+
+class TestOnlyOurOwnTasksAreTouched:
+    """create_snapshot is module-global, and cloud_sync.py imports it too.
+
+    plugins/cloud/snapshot.py::create_snapshot is imported by BOTH
+    cloud_backup/sync.py and cloud_sync.py, so our wrapper sits in the path of every
+    rclone/Storj CloudSync task with snapshot=true -- tasks this patch has no business
+    touching. Two consequences, the second much worse than the first:
+
+      * every middleware call we add is a NEW failure mode for a job that worked
+        before we were installed;
+      * a staged CloudSync task would NEVER be torn down. The teardown is wired into
+        cloud_backup's restic_backup finally, and CRUD_BLOCK deliberately leaves
+        CloudSync's nesting guard intact -- so the bind mounts would pin the ZFS
+        snapshot forever.
+
+    cloud_backup names its snapshot "cloud_backup-<id>", cloud_sync "cloud_sync-<id>",
+    and stock's default is "cloud_task-onetime".
+    """
+
+    @pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
+    def test_the_staging_path_is_gated_on_cloud_backup(self, name):
+        block = extract_blocks()[name]
+        assert 'if not name.startswith("cloud_backup"):' in block
+
+    @pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
+    def test_the_bail_out_precedes_every_middleware_call(self, name):
+        # The point is to add NO new failure mode to a CloudSync task. If any
+        # middleware call happened before the bail-out, we would already have broken
+        # the thing we are trying not to touch.
+        #
+        # Checked against whichever interactions the block ACTUALLY contains, not a
+        # fixed list: the dataset query moved behind `_tc_nested.query_filesystems()`
+        # when it switched to the public pool.* API, and a hardcoded
+        # `middleware.call_sync(` simply stopped being found -- a test that silently
+        # stops testing is worse than no test.
+        block = extract_blocks()[name]
+        gate = block.index('if not name.startswith("cloud_backup"):')
+
+        interactions = [
+            "middleware.call_sync(",
+            "_tc_nested.query_filesystems(",
+            "_tc_nested.stage_nested(",
+            "_tc_nested.delete_snapshot_tree(",
+        ]
+        present = [c for c in interactions if c in block]
+        assert present, "found no middleware interaction at all -- the test is vacuous"
+        for call in present:
+            assert gate < block.index(call), f"{call} runs before the cloud_backup gate"
+
+
+# ── structural assertions ────────────────────────────────────────────────────
+#
+# `assert "raise" in block` was TRUE because a COMMENT in the block says "a cleanup
+# that raises would replace the original exception". `assert "cleanup_task" in block`
+# was TRUE because a comment says "cleanup_task gets logger=None". Deleting the actual
+# `raise`, and deleting the actual cleanup call from the `finally`, both left the suite
+# green -- while reinstating, respectively, a silently-empty backup and ~250 orphaned
+# snapshots per run.
+#
+# A test that a comment can satisfy is not a test. These parse the block and assert on
+# the CODE.
+
+def tree_of(name):
+    return ast.parse(textwrap.dedent(extract_blocks()[name]))
+
+
+def functions(tree, name):
+    return [
+        n for n in ast.walk(tree)
+        if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef) and n.name == name
+    ]
+
+
+def calls_to(node, dotted):
+    """Every Call in `node` whose callee renders as `dotted` (e.g. a.b.c)."""
+    out = []
+    for n in ast.walk(node):
+        if isinstance(n, ast.Call):
+            try:
+                if ast.unparse(n.func) == dotted:
+                    out.append(n)
+            except Exception:                                    # noqa: BLE001
+                pass
+    return out
+
+
+class TestTheStagingFailurePathReallyReRaises:
+    """If staging fails and we swallow it, restic backs up the UN-STAGED path.
+
+    That is the silently-empty backup this entire module exists to prevent: stock
+    points the tool at the parent's `.zfs/snapshot/`, where child datasets are
+    invisible. The exception MUST propagate.
+    """
+
+    @pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
+    def test_the_handler_sweeps_the_snapshot_and_re_raises(self, name):
+        stage = functions(tree_of(name), "_tc_stage")
+        assert stage, "_tc_stage is gone"
+
+        handlers = [
+            h for t in ast.walk(stage[0]) if isinstance(t, ast.Try)
+            for h in t.handlers
+        ]
+        assert handlers, "the staging failure handler is gone"
+
+        sweeps = any(calls_to(h, "_tc_nested.delete_snapshot_tree") for h in handlers)
+        assert sweeps, (
+            "a staging failure no longer sweeps the snapshot. sync.py's `snapshot` "
+            "local stays None, so ITS finally deletes nothing -- the whole tree leaks "
+            "on every failed run."
+        )
+
+        # A bare `raise` directly in the handler body -- not one nested inside the
+        # defensive try/except that wraps the sweep.
+        reraises = any(
+            any(isinstance(s, ast.Raise) and s.exc is None for s in h.body)
+            for h in handlers
+        )
+        assert reraises, (
+            "the staging failure is SWALLOWED. restic then runs against the un-staged "
+            "path and uploads a near-empty tree, reporting SUCCESS."
+        )
+
+
+class TestTheSyncBlockAlwaysTearsDown:
+    """The teardown is what unmounts the staging tree and sweeps the snapshot.
+
+    It must run on EVERY exit from restic_backup -- success, failure, or exception --
+    or the bind mounts pin the snapshot and the tree is orphaned.
+    """
+
+    @pytest.mark.parametrize("name", ["SYNC_ASYNC", "SYNC_SYNC"])
+    def test_cleanup_runs_in_a_finally(self, name):
+        fns = functions(tree_of(name), "restic_backup")
+        assert fns, "the restic_backup wrapper is gone"
+
+        tries = [t for t in ast.walk(fns[0]) if isinstance(t, ast.Try) and t.finalbody]
+        assert tries, "restic_backup no longer has a try/finally"
+
+        cleans = any(
+            "cleanup_task" in ast.unparse(stmt)
+            for t in tries for stmt in t.finalbody
+        )
+        assert cleans, (
+            "cleanup_task is not called in the finally. The staging tree is never torn "
+            "down, its bind mounts pin the snapshot, and ~250 snapshots leak per run."
+        )
+
+
+class TestTheBlockingWorkNeverRunsOnTheEventLoop:
+    """`zfs list` and `call_sync` are BLOCKING. On <=25.10 these blocks are async.
+
+    Running them directly on middlewared's event loop stalls the whole daemon.
+    """
+
+    @pytest.mark.parametrize("name,fn", [
+        ("SNAPSHOT_ASYNC", "create_snapshot"),
+        ("SYNC_ASYNC", "restic_backup"),
+    ])
+    def test_the_async_flavour_hops_to_a_thread(self, name, fn):
+        fns = functions(tree_of(name), fn)
+        assert fns and isinstance(fns[0], ast.AsyncFunctionDef)
+        assert calls_to(fns[0], "middleware.run_in_thread"), (
+            f"{name}.{fn} does the blocking work on the asyncio event loop"
+        )
+
+    @pytest.mark.parametrize("name,fn", [
+        ("SNAPSHOT_SYNC", "create_snapshot"),
+        ("SYNC_SYNC", "restic_backup"),
+    ])
+    def test_the_sync_flavour_does_not(self, name, fn):
+        # On 26 stock already runs this in the thread pool; hopping again would be
+        # wrong (and there is no event loop to protect).
+        fns = functions(tree_of(name), fn)
+        assert fns and isinstance(fns[0], ast.FunctionDef)
+        assert not calls_to(fns[0], "middleware.run_in_thread")
+
+
+def test_the_flavour_mapping_is_not_inverted():
+    # `_snapshot_block = SNAPSHOT_ASYNC if _flavour else SNAPSHOT_SYNC` -- inverting it
+    # injects an async wrapper on 26 (a coroutine gets unpacked as a tuple) or a sync
+    # one on 25.10 (the event loop blocks). Every nested backup breaks, both ways.
+    with open(APPLY_SH, encoding="utf-8") as fh:
+        code = " ".join(
+            ln for ln in fh.read().splitlines() if not ln.lstrip().startswith("#")
+        )
+    code = re.sub(r"\s+", " ", code)          # the assignments are space-aligned
+    for block in ("SNAPSHOT", "CRUD", "SYNC"):
+        assert f"{block}_ASYNC if _flavour else {block}_SYNC" in code, (
+            f"the {block} flavour mapping is missing or inverted: _flavour is True for "
+            f"an ASYNC middleware, so it must select {block}_ASYNC"
+        )
+
+
+# ── the compat preflight ─────────────────────────────────────────────────────
+#
+# This is the guard that stands between a broken middleware and a live NAS: at every
+# boot, apply.sh checks the patch's assumptions against the middlewared actually
+# installed, and REFUSES to apply a module whose assumptions no longer hold.
+#
+# It had no test. An audit turned it into a no-op eight different ways -- `verdict()`
+# always returning 'ok', the broken branch never firing, the kill switch never honoured
+# -- and the suite stayed green every time. The most consequential safety net in the
+# project was unguarded.
+
+def preflight_heredoc():
+    """The preflight's Python, lifted out of apply.sh and made runnable.
+
+    Extracted, not reimplemented: a reimplementation would happily pass while the
+    SHIPPED preflight stayed broken, which is exactly the failure being guarded.
+    """
+    with open(APPLY_SH, encoding="utf-8") as fh:
+        sh = fh.read()
+    # Line-based: the compat heredoc opens with `<<'PYEOF'` on the _tc_compat line and
+    # closes at the next bare PYEOF. (A regex that matched `<< 'PYEOF'` silently found
+    # the OTHER heredoc and ran a different script entirely.)
+    lines = sh.splitlines()
+    start = next(
+        i for i, ln in enumerate(lines)
+        if ln.startswith("_tc_compat=$(") and "<<'PYEOF'" in ln
+    )
+    end = next(i for i in range(start + 1, len(lines)) if lines[i].strip() == "PYEOF")
+    m = "\n".join(lines[start + 1:end])
+    assert m, "could not find the compat preflight heredoc in apply.sh"
+    return m
+
+
+def run_preflight(result, tmp_path):
+    """Run the SHIPPED preflight against a fake compat.check_tree result.
+
+    The heredoc does `import sys`, so a fake `sys` in the namespace is immediately
+    rebound to the real module -- drive the real one instead.
+    """
+    import contextlib
+    import io
+    import sys
+    import types
+
+    src = preflight_heredoc()
+    fake = types.ModuleType("compat")
+    fake.check_tree = lambda _mw: result
+
+    saved_mod = sys.modules.get("compat")
+    saved_argv = sys.argv
+    sys.modules["compat"] = fake
+    sys.argv = ["x", "/patch", "/mw", str(tmp_path / "compat.json")]
+
+    buf = io.StringIO()
+    try:
+        with contextlib.redirect_stdout(buf):
+            exec(compile(src, "apply.sh:preflight", "exec"), {"__name__": "__main__"})  # noqa: S102
+    except SystemExit:
+        pass
+    finally:
+        sys.argv = saved_argv
+        if saved_mod is not None:
+            sys.modules["compat"] = saved_mod
+        else:
+            sys.modules.pop("compat", None)
+    return buf.getvalue().splitlines()
+
+
+def _mod(ok=True, native=False, unknown=False, problems=()):
+    return {"ok": ok, "native": native, "unknown": unknown, "problems": list(problems)}
+
+
+class TestTheBootPreflightRefusesABrokenMiddleware:
+    def test_a_healthy_tree_is_ok(self, tmp_path):
+        out = run_preflight({"providers": _mod(), "nested": _mod()}, tmp_path)
+        assert out[:2] == ["ok", "ok"]
+
+    def test_a_broken_module_is_reported_broken(self, tmp_path):
+        out = run_preflight({
+            "providers": _mod(),
+            "nested": _mod(ok=False, problems=[
+                {"id": "x", "detail": "gone", "why": "orphans every run"},
+            ]),
+        }, tmp_path)
+        assert "broken" in out, (
+            "the preflight did not report a module whose assumptions FAILED. It would "
+            "be injected into a middleware it does not fit -- broken backups, "
+            "discovered at restore time."
+        )
+
+    def test_a_module_that_went_NATIVE_is_also_not_applied(self, tmp_path):
+        # 'native' answers "do we still need it?", 'ok' answers "is it safe to inject?".
+        # Applying a module TrueNAS now implements itself is not safe either.
+        out = run_preflight({
+            "providers": _mod(),
+            "nested": _mod(ok=False, native=True),
+        }, tmp_path)
+        assert "broken" in out
+
+    def test_an_UNKNOWN_verdict_is_not_reported_as_broken(self, tmp_path):
+        # A network error or an unreadable file is not iX deleting our symbols. Calling
+        # it broken would switch a working module off on a healthy box.
+        out = run_preflight({
+            "providers": _mod(unknown=True),
+            "nested": _mod(unknown=True),
+        }, tmp_path)
+        assert "broken" not in out

tests/test_compat.py

@@ -0,0 +1,808 @@
+"""Tests for the middlewared compatibility manifest.
+
+Two failure directions, and they are NOT symmetric:
+
+  * a false **BROKEN** makes a module decline to apply on a box where it works.
+    Worse, if both modules go quiet, apply.sh used to set a PERMANENT kill switch
+    that only install.sh clears -- so a network blip or an innocent refactor could
+    take a working box's B2 backups down until someone noticed by hand.
+
+  * a false **OK** lets the patch inject into middleware it does not fit, which is a
+    broken backup discovered at restore time.
+
+Both are tested. The `native` verdict gets its own scrutiny because it is the most
+dangerous thing this file can say -- it means "TrueNAS does this now, retire the
+module" -- and it rests on nothing more than a substring match.
+"""
+
+import json
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
+
+import compat  # noqa: E402
+import compat_publish  # noqa: E402
+from compat import (  # noqa: E402
+    NESTED,
+    PROVIDERS,
+    Unreadable,
+    check,
+    is_broken,
+)
+
+# A middlewared that the patch fits: TrueNAS 25.10 in miniature.
+GOOD = {
+    "rclone/remote/b2.py": "class B2RcloneRemote(BaseRcloneRemote):\n    pass\n",
+    "plugins/cloud_backup/restic.py": (
+        "class ResticConfig:\n    cmd: list\n\n"
+        "def get_restic_config(cloud_backup):\n    return ResticConfig([], {})\n"
+    ),
+    "plugins/cloud/snapshot.py": (
+        'async def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+    ),
+    "plugins/cloud/crud.py": (
+        "class CloudTaskServiceMixin:\n"
+        "    async def _validate(self, app, verrors, name, data):\n"
+        "        verrors.add('x', 'datasets that have no further '\n"
+        "                         'nesting')\n"
+    ),
+    "plugins/cloud_backup/sync.py": (
+        "async def restic_backup(middleware, job, cloud_backup, dry_run=False, "
+        "rate_limit=None):\n    pass\n"
+    ),
+    # The middlewared METHODS the injected code calls. These now go through the
+    # PUBLIC pool.* API: TrueNAS 26 deleted plugins/zfs_/ outright, taking the whole
+    # private zfs.* service with it -- see TestMiddlewareMethodsWeCall.
+    #
+    # This default tree is a MODERN box (25.10/26): it has pool.snapshot and no
+    # zfs.snapshot. The older shape is built explicitly where it is tested.
+    # Not a plugin: a method on the middleware OBJECT. `snapshot_service()` resolves
+    # the snapshot namespace through it, so if it vanishes the module cannot sweep the
+    # snapshot it just took.
+    "utils/plugins.py": (
+        "class LoadPluginsMixin:\n"
+        "    def get_service(self, name):\n        pass\n"
+    ),
+    "plugins/pool_/dataset.py": (
+        "class PoolDatasetService(CRUDService):\n"
+        "    class Config:\n"
+        "        namespace = 'pool.dataset'\n"
+        "    def query(self, filters, options):\n        pass\n"
+    ),
+    "plugins/pool_/snapshot.py": (
+        "class PoolSnapshotService(CRUDService):\n"
+        "    class Config:\n"
+        "        namespace = 'pool.snapshot'\n"
+        "    def query(self, filters, options):\n        pass\n"
+        "    def delete(self, id_, options={}):\n        pass\n"
+    ),
+}
+
+#: A 24.10/25.04 box: `pool.snapshot` does not exist yet and the snapshot CRUD
+#: service still answers to the (then-public) `zfs.snapshot`.
+ZFS_ERA_SNAPSHOT = (
+    "class ZFSSnapshot(CRUDService):\n"
+    "    class Config:\n"
+    "        namespace = 'zfs.snapshot'\n"
+    "    def query(self, filters, options):\n        pass\n"
+    "    def do_delete(self, id_, options={}):\n        pass\n"
+)
+
+
+def loader(files):
+    def load(path):
+        if path not in files:
+            return None
+        v = files[path]
+        if isinstance(v, Exception):
+            raise v
+        return v
+    return load
+
+
+def check_files(files, modules=None):
+    return check(loader(files), modules)
+
+
+def with_(**overrides):
+    files = dict(GOOD)
+    files.update(overrides)
+    return files
+
+
+class TestTheBaseline:
+    def test_a_good_tree_is_ok_and_not_native(self):
+        r = check_files(GOOD)
+        for mod in (PROVIDERS, NESTED):
+            assert r[mod]["ok"], r[mod]["problems"]
+            assert not r[mod]["native"]
+            assert not r[mod]["unknown"]
+
+
+class TestFalseOkWouldBreakBackups:
+    """The patch calls the originals POSITIONALLY. A name-subset check passed all of
+    these, and each is a TypeError or -- worse -- silently swapped arguments."""
+
+    def test_reordered_parameters_are_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                'async def create_snapshot(name, path, middleware):\n    return 1, 2\n',
+        }))
+        assert is_broken(r[NESTED])
+
+    def test_a_keyword_only_conversion_is_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                'async def create_snapshot(middleware, *, path, name="x"):\n    return 1, 2\n',
+        }))
+        assert is_broken(r[NESTED])
+
+    def test_a_new_required_parameter_is_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                'async def create_snapshot(middleware, path, name, dataset):\n    return 1, 2\n',
+        }))
+        assert is_broken(r[NESTED])
+
+    def test_a_new_optional_parameter_is_fine(self):
+        # The patch simply will not pass it. Refusing here would be false BROKEN.
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                'async def create_snapshot(middleware, path, name="x", quiet=False):\n'
+                "    return 1, 2\n",
+        }))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+    def test_the_master_signature_change_is_caught(self):
+        # iX really did rename this on master: get_restic_config(entry, credentials).
+        # RESTIC_BLOCK rebinds the module-level name to a 1-arg wrapper, so getting
+        # this wrong kills EVERY TrueCloud task -- Storj included.
+        r = check_files(with_(**{
+            "plugins/cloud_backup/restic.py":
+                "class ResticConfig:\n    cmd: list\n\n"
+                "def get_restic_config(entry, credentials):\n    pass\n",
+        }))
+        assert is_broken(r[PROVIDERS])
+
+    def test_a_vanished_symbol_is_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py": "def something_else():\n    pass\n",
+        }))
+        assert is_broken(r[NESTED])
+
+
+class TestFalseBrokenWouldDisableWorkingBoxes:
+    def test_a_conditionally_defined_symbol_is_not_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                "try:\n"
+                "    from .fast import create_snapshot\n"
+                "except ImportError:\n"
+                '    async def create_snapshot(middleware, path, name="x"):\n'
+                "        return 1, 2\n",
+        }))
+        assert not is_broken(r[NESTED]), r[NESTED]["problems"]
+
+    def test_a_re_exported_symbol_is_unknown_not_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud_backup/restic.py":
+                "from ._impl import ResticConfig, get_restic_config\n",
+        }))
+        assert not is_broken(r[PROVIDERS])
+        assert r[PROVIDERS]["unknown"]
+
+    def test_an_unreadable_source_is_unknown_not_broken(self):
+        # A rate limit (the matrix makes ~30 unauthenticated requests) must not be
+        # able to say "iX deleted six files, both modules are broken".
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py": Unreadable("HTTP 429"),
+        }))
+        assert not is_broken(r[NESTED])
+        assert r[NESTED]["unknown"]
+
+    def test_a_definite_break_still_wins_over_an_unknown(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py": Unreadable("HTTP 429"),
+            "plugins/cloud_backup/sync.py":
+                "async def restic_backup(job, middleware, cloud_backup):\n    pass\n",
+        }))
+        assert is_broken(r[NESTED]), "unknown must not launder away a proven break"
+
+
+class TestTheNativeVerdict:
+    """"native" means "retire the module". It is the most destructive thing this file
+    can say, and it is only a substring match — so it must never outrank BROKEN."""
+
+    def test_broken_outranks_native(self):
+        # Guard reworded (reads as native) AND the signatures changed (really broken).
+        # This used to render as good news: green CI, no bug report, and a README row
+        # telling users the feature went native while it was in fact broken.
+        r = check_files(with_(**{
+            "plugins/cloud/crud.py":
+                "class CloudTaskServiceMixin:\n"
+                "    async def _validate(self, verrors, name):\n"
+                "        verrors.add('x', 'no children allowed')\n",
+        }))
+        assert r[NESTED]["native"]
+        assert is_broken(r[NESTED])
+        assert compat._verdict(r[NESTED]) == "BROKEN"
+
+    def test_an_already_patched_tree_does_not_read_as_native(self):
+        # B2_BLOCK writes `B2RcloneRemote.restic = True` into b2.py. Scanning the whole
+        # file finds OUR OWN line and concludes TrueNAS went native — so the command
+        # compat.py's docstring recommends for a live box (`--tree /usr/lib/...`)
+        # reported providers as native on every patched machine.
+        r = check_files(with_(**{
+            "rclone/remote/b2.py":
+                "class B2RcloneRemote(BaseRcloneRemote):\n    pass\n"
+                "\n# TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh\n"
+                "B2RcloneRemote.restic = True\n",
+        }))
+        assert not r[PROVIDERS]["native"], "read its own patch as native support"
+        assert r[PROVIDERS]["ok"]
+
+    def test_a_genuinely_native_b2_is_native(self):
+        r = check_files(with_(**{
+            "rclone/remote/b2.py":
+                "class B2RcloneRemote(BaseRcloneRemote):\n    restic = True\n",
+        }))
+        assert r[PROVIDERS]["native"]
+
+
+class TestUpdateReadmeCannotPublishAGuess:
+    def test_it_refuses_when_anything_is_unknown(self, tmp_path):
+        readme = tmp_path / "README.md"
+        readme.write_text(f"x\n{compat.BEGIN}\nold\n{compat.END}\ny\n")
+        rows = [{
+            "ref": "TS-25.10.4", "unreleased": False,
+            "modules": check_files(with_(**{
+                "plugins/cloud/snapshot.py": Unreadable("HTTP 429"),
+            })),
+        }]
+        with pytest.raises(Unreadable):
+            compat.update_readme(rows, path=str(readme))
+        assert "old" in readme.read_text(), "a blip must not repaint the matrix"
+
+
+class TestAsyncFlavour:
+    """TrueNAS <= 25.10 is async; 26 is synchronous. Both are supported -- apply.sh
+    injects the wrapper that matches. So asyncness is DETECTED, never assumed."""
+
+    def test_async_middleware_is_detected(self):
+        assert compat.async_flavour(loader(GOOD)) is True
+
+    def test_sync_middleware_is_detected(self):
+        sync = dict(GOOD)
+        sync["plugins/cloud/snapshot.py"] = (
+            'def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+        )
+        sync["plugins/cloud/crud.py"] = (
+            "class CloudTaskServiceMixin:\n"
+            "    def _validate(self, app, verrors, name, data):\n"
+            "        verrors.add('x', 'no further nesting')\n"
+        )
+        sync["plugins/cloud_backup/sync.py"] = (
+            "def restic_backup(middleware, job, cloud_backup, dry_run=False, "
+            "rate_limit=None):\n    pass\n"
+        )
+        assert compat.async_flavour(loader(sync)) is False
+
+    def test_a_HALF_converted_middleware_is_refused(self):
+        # The dangerous middle. If iX converts create_snapshot but not restic_backup,
+        # there is no single wrapper flavour that works -- and guessing means either
+        # a coroutine unpacked as a tuple, or the event loop blocked. None means
+        # "do not patch"; apply.sh turns that into a skip, not a guess.
+        half = dict(GOOD)
+        half["plugins/cloud/snapshot.py"] = (
+            'def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+        )
+        assert compat.async_flavour(loader(half)) is None
+
+    def test_an_unreadable_source_refuses_rather_than_guesses(self):
+        broken = dict(GOOD)
+        broken["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429")
+        assert compat.async_flavour(loader(broken)) is None
+
+    def test_it_reads_STOCK_source_not_our_own_injected_block(self):
+        # This was a byte-identical copy of test_async_middleware_is_detected under a
+        # name that promised more. The fact worth pinning: apply.sh re-runs on an
+        # ALREADY-PATCHED overlay, so the probe must cut our block off first -- our own
+        # SNAPSHOT_SYNC wrapper is a plain `def create_snapshot`, and reading it would
+        # report a 25.10 box as synchronous and inject the wrong flavour.
+        patched = dict(GOOD)
+        patched["plugins/cloud/snapshot.py"] = (
+            GOOD["plugins/cloud/snapshot.py"]
+            + "\n# TRUECLOUD_PATCH\n"
+            + 'def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+        )
+        assert compat.async_flavour(loader(patched)) is True, (
+            "the flavour probe read our own injected block and concluded the box is "
+            "synchronous -- it would then inject a sync wrapper into an async "
+            "middleware, and every nested backup would break"
+        )
+
+
+class TestMiddlewareMethodsWeCall:
+    """The assumption class that was MISSING, and that hid a catastrophic break.
+
+    The manifest recorded the symbols the patch WRAPS. It said nothing about the
+    middlewared methods the patch CALLS -- and TrueNAS 26 deleted
+    plugins/zfs_/dataset.py and plugins/zfs_/snapshot.py outright, taking
+    `zfs.dataset.query`, `zfs.snapshot.query` and `zfs.snapshot.delete` with them.
+
+    Nothing about the five cloud_backup files reveals that. The patch applied
+    perfectly and every other check went green. The first backup would have failed --
+    or, far worse, snapshotted fine and then failed to DELETE, orphaning one snapshot
+    per descendant dataset (250 on a real pool) on every run, forever.
+    """
+
+    POOL_SNAPSHOT = GOOD["plugins/pool_/snapshot.py"]
+
+    def _tree(self, **over):
+        files = dict(GOOD)
+        files.update(over)
+        return files
+
+    #: A 26 box: pool.snapshot only.
+    def _modern(self, **over):
+        return self._tree(**over)
+
+    #: A 24.10/25.04 box: zfs.snapshot only -- plugins/pool_/snapshot.py does not
+    #: exist yet.
+    def _zfs_era(self, **over):
+        return self._tree(**{
+            "plugins/pool_/snapshot.py": None,
+            "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
+            **over,
+        })
+
+    def test_present_methods_are_ok(self):
+        r = check_files(self._modern())
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+    def test_the_OLD_zfs_era_snapshot_service_also_satisfies_the_call(self):
+        # 24.10 and 25.04 have no `pool.snapshot` at all -- the CRUD service is the
+        # then-public `zfs.snapshot`. Pinning only the modern spelling marked both of
+        # those releases BROKEN and would have switched nested snapshots OFF on boxes
+        # where they work perfectly. The runtime picks the same way; see
+        # pick_snapshot_service().
+        r = check_files(self._zfs_era())
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+    def test_it_is_broken_only_when_NEITHER_namespace_exists(self):
+        # The real failure: middleware drops the last spelling we know how to call.
+        r = check_files(self._tree(**{
+            "plugins/pool_/snapshot.py": None,
+            "plugins/zfs_/snapshot.py": None,
+        }))
+        assert is_broken(r[NESTED])
+        details = " ".join(p["detail"] for p in r[NESTED]["problems"])
+        assert "pool.snapshot.delete" in details
+        assert "zfs.snapshot.delete" in details, (
+            "the report must say BOTH spellings were tried, or whoever reads it will "
+            "think we simply never looked for the one their box has"
+        )
+
+    def test_we_do_NOT_depend_on_a_middleware_dataset_query_at_all(self):
+        # iX could delete plugins/pool_/dataset.py tomorrow and the patch would not
+        # care, because the staging plan is enumerated from ZFS, not from middleware.
+        #
+        # That is deliberate, and it was expensive to learn. `pool.dataset.query`
+        # exists and is correctly shaped -- and it LIES: it applies a visibility
+        # policy that hides ix-apps/*, .system/* and .ix-virt/* (84 of 270 datasets
+        # on the real pool, including live app data). No source check could ever
+        # have caught that; only running it could. So there is no assumption here
+        # left to break.
+        r = check_files(self._modern(**{"plugins/pool_/dataset.py": None}))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+        ids = {c.id for c in compat.MIDDLEWARE_CALLS}
+        assert not any("dataset" in i or "query" in i for i in ids), (
+            "a dataset/snapshot QUERY assumption crept back into the manifest -- "
+            "middleware's queries are filtered; enumerate from ZFS"
+        )
+
+    def test_a_renamed_namespace_is_broken(self):
+        r = check_files(self._tree(**{
+            "plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace(
+                "'pool.snapshot'", "'zfs.resource.snapshot'"),
+            "plugins/zfs_/snapshot.py": None,
+        }))
+        assert is_broken(r[NESTED])
+
+    def test_the_CRUDService_do_prefix_is_accepted(self):
+        # A CRUDService exposes `delete` from a method NAMED `do_delete`. Both
+        # spellings are live across the matrix. Accepting only the literal name
+        # reported working releases as broken.
+        r = check_files(self._modern(**{
+            "plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace(
+                "def delete(", "def do_delete("),
+        }))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+    def test_the_snapshot_delete_reason_names_the_orphan_risk(self):
+        # If this ever regresses, whoever reads the bug report must understand that
+        # it is not a cosmetic failure.
+        r = check_files(self._tree(**{
+            "plugins/pool_/snapshot.py": None,
+            "plugins/zfs_/snapshot.py": None,
+        }))
+        whys = " ".join(p["why"] for p in r[NESTED]["problems"])
+        assert "orphan" in whys
+
+
+class TestTheMethodCheckIsNotJustANamespaceCheck:
+    """compat must verify the METHOD, not merely that the namespace still exists.
+
+    Deleting the method check entirely used to leave all 304 tests green -- so the
+    "namespace AND method" claim was unenforced and silently revertible. It is the
+    half of the predicate that catches iX gutting a method while keeping its service,
+    which they have already done to `pool.snapshot.do_update` on master.
+    """
+
+    def test_a_namespace_that_no_longer_defines_delete_is_broken(self):
+        gutted = (
+            "class PoolSnapshotService(CRUDService):\n"
+            "    class Config:\n"
+            "        namespace = 'pool.snapshot'\n"
+            "    def query(self, filters, options):\n        pass\n"
+            # do_delete is GONE -- the service is still registered and still a
+            # CRUDService, so it still INHERITS a callable `delete`.
+        )
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": gutted,
+            "plugins/zfs_/snapshot.py": None,      # no fallback either
+        }))
+        assert is_broken(r[NESTED]), (
+            "a namespace with no delete must be BROKEN. Checking only that the "
+            "namespace exists would apply the patch to a box that cannot sweep its "
+            "own snapshots."
+        )
+
+    def test_the_alternative_still_saves_it_when_only_the_primary_is_gutted(self):
+        gutted = (
+            "class PoolSnapshotService(CRUDService):\n"
+            "    class Config:\n"
+            "        namespace = 'pool.snapshot'\n"
+            "    def query(self, filters, options):\n        pass\n"
+        )
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": gutted,
+            "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
+        }))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+
+class TestUnreadableIsNeverOkAndNeverBroken:
+    """A rate limit is not a regression, and it is not a clean bill of health either.
+
+    compat runs ~30 unauthenticated GitHub requests per matrix; 429 is a real outcome.
+    It also runs at BOOT against the installed tree, where a read can fail with EACCES.
+
+      * treating unreadable as BROKEN repaints the README, files a bug report, and
+        makes apply.sh refuse the module on a box where it works.
+      * treating it as OK injects a module whose delete may be gone.
+
+    Both mutations used to pass the whole suite.
+    """
+
+    def test_both_spellings_unreadable_is_unknown_not_broken(self):
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": Unreadable("HTTP 429"),
+            "plugins/zfs_/snapshot.py": Unreadable("HTTP 429"),
+        }))
+        assert not is_broken(r[NESTED]), "a 429 is not iX deleting the snapshot service"
+        assert r[NESTED]["unknown"]
+
+    def test_an_unreadable_primary_with_a_healthy_alternative_is_ok(self):
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": Unreadable("HTTP 429"),
+            "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
+        }))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+        assert not r[NESTED]["unknown"], (
+            "one spelling answered the question; the other's 429 is irrelevant"
+        )
+
+    def test_a_missing_primary_with_an_unreadable_alternative_is_unknown(self):
+        # We cannot tell whether the box is broken. Saying either would be a guess.
+        r = check_files(with_(**{
+            "plugins/pool_/snapshot.py": None,
+            "plugins/zfs_/snapshot.py": Unreadable("HTTP 429"),
+        }))
+        assert not is_broken(r[NESTED])
+        assert r[NESTED]["unknown"]
+
+
+class TestGetServiceIsChecked:
+    """The runtime resolves the snapshot namespace through `middleware.get_service`.
+
+    It is not a plugin method, so the manifest had no way to express it and never
+    checked it. If it vanishes, `_can_delete` reports BOTH namespaces unusable and
+    every nested backup fails -- on a box the preflight had declared healthy.
+    """
+
+    def test_a_middleware_without_get_service_is_broken(self):
+        r = check_files(with_(**{"utils/plugins.py": None}))
+        assert is_broken(r[NESTED])
+        details = " ".join(p["detail"] for p in r[NESTED]["problems"])
+        assert "get_service" in details
+
+
+class TestATransientNetworkBlipDoesNotWakeAnybody:
+    """The fingerprint must digest what iX BROKE, not what GitHub failed to serve.
+
+    `unknown` problems (a 429 on one of ~30 unauthenticated fetches, an EACCES at boot)
+    used to be folded into an already-broken module's problem list, so one blip flipped
+    the fingerprint, `compat_publish` rewrote the issue body, and the next clean run
+    rewrote it back. Daily churn is what teaches people to ignore the bot -- which is
+    the whole thing this fingerprint exists to prevent.
+    """
+
+    def _rows(self, files):
+        return [{"ref": "master", "modules": check_files(files)}]
+
+    def test_an_unreadable_file_does_not_change_the_fingerprint_of_a_broken_ref(self):
+        # The blip must land in the SAME module that is broken. Put it in `providers`
+        # (which is healthy) and `fingerprint()` skips the whole module via
+        # `is_broken(m)` -- so the `state` filter under test never runs and the test
+        # passes no matter what the code does. `nested` is the broken one here, so the
+        # unreadable file goes in `nested` too.
+        broken = with_(**{
+            "plugins/cloud/snapshot.py":
+                "async def create_snapshot(name, path, middleware):\n    return 1, 2\n",
+        })
+        clean = compat.fingerprint(self._rows(broken))
+
+        blipped = dict(broken)
+        blipped["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429")   # nested
+        assert compat.fingerprint(self._rows(blipped)) == clean, (
+            "a rate-limited fetch changed the fingerprint, so the bot rewrites the "
+            "issue body and then rewrites it back tomorrow"
+        )
+
+    def test_a_REAL_new_finding_still_changes_it(self):
+        # ...and the anti-noise measure must not have made it deaf.
+        broken = with_(**{
+            "plugins/cloud/snapshot.py":
+                "async def create_snapshot(name, path, middleware):\n    return 1, 2\n",
+        })
+        worse = dict(broken)
+        worse["plugins/cloud_backup/restic.py"] = (
+            "class ResticConfig:\n    cmd: list\n\n"
+            "def get_restic_config(entry, credentials):\n    pass\n"
+        )
+        assert compat.fingerprint(self._rows(worse)) != compat.fingerprint(self._rows(broken))
+
+
+class TestTheBotFindsItsOwnIssueOnBOTHForges:
+    """`find_issue` decides "have I already filed this?" -- and it ran on two forges.
+
+    It used to skip pull requests with `"pull_request" not in i`. GitHub omits that key
+    on a plain issue; **Gitea sends it as `null`**. So on Gitea every issue looked like
+    a PR, the match list was always empty, and the bot took the "nothing filed yet"
+    branch on EVERY run: nine duplicate copies of the same report on the canonical
+    forge, four of them filed after the commit that was supposed to stop exactly this.
+
+    It is the same failure the spam fix was written to prevent, moved from comments to
+    issues -- and it survived because `find_issue` was the one function here with no
+    test. So the payload shapes are pinned, per forge, by hand.
+    """
+
+    TITLE = compat_publish.TITLE
+
+    def _find(self, monkeypatch, payload):
+        monkeypatch.setattr(compat_publish, "_call", lambda *a, **k: payload)
+        return compat_publish.find_issue("https://forge/api", "tok", self.TITLE)
+
+    def test_gitea_sends_pull_request_as_null_and_the_issue_is_still_found(self, monkeypatch):
+        found = self._find(monkeypatch, [
+            {"number": 7, "title": self.TITLE, "state": "open", "pull_request": None},
+            {"number": 1, "title": self.TITLE, "state": "open", "pull_request": None},
+        ])
+        assert found is not None, (
+            "find_issue missed a Gitea issue, so the bot files a NEW duplicate report "
+            "every run -- which is how nine of them piled up"
+        )
+        assert found["number"] == 1, "lowest-numbered wins"
+
+    def test_github_omits_the_key_entirely_and_the_issue_is_still_found(self, monkeypatch):
+        found = self._find(monkeypatch, [
+            {"number": 2, "title": self.TITLE, "state": "open"},
+        ])
+        assert found is not None and found["number"] == 2
+
+    def test_a_real_PR_with_the_same_title_is_still_skipped_on_both(self, monkeypatch):
+        # The reason the filter exists at all: both forges list PRs on /issues, and
+        # commenting on a PR instead of the bug report would be worse than useless.
+        assert self._find(monkeypatch, [
+            {"number": 3, "title": self.TITLE, "state": "open",           # Gitea PR
+             "pull_request": {"merged": False}},
+            {"number": 4, "title": self.TITLE, "state": "open",           # GitHub PR
+             "pull_request": {"url": "https://api.github.com/..."}},
+        ]) is None
+
+    def test_an_unrelated_issue_is_not_mistaken_for_the_report(self, monkeypatch):
+        assert self._find(monkeypatch, [
+            {"number": 1, "title": "TypeError when create B2 backup on Electric Eel",
+             "state": "closed", "pull_request": None},
+        ]) is None
+
+
+class TestTheNextMaintenanceReleaseIsChecked:
+    """`release/25.10.5` fell through every sieve, and it is the one that reaches users.
+
+    Shipped versions come from `TS-*` TAGS; unreleased ones come from `release/*`
+    BRANCHES that carry `-BETA`/`-RC`. A branched-but-untagged MAINTENANCE release is
+    neither: no tag, and its line (25.10) has already shipped, so the "prereleases of
+    a shipped line are history" filter threw it out. It was invisible.
+
+    That is backwards. `release/24.10-RC.2` is history -- nobody can install it. But
+    `release/25.10.5` is the FUTURE of a shipped line: it is what a 25.10.4 box gets
+    on its next update. A break there ships to real users before the daily check has
+    ever looked at it.
+    """
+
+    TAGS = ["TS-24.10.2.4", "TS-25.04.2.6", "TS-25.10.4"]
+    HEADS = [
+        "release/25.10.4.1",
+        "release/25.10.5",          # branched, untagged -- the next maintenance release
+        "release/24.10-RC.2",       # history: its line shipped long ago
+        "release/25.20.2.2",        # iX's typo branch: 25.20 is not a TrueNAS version
+        "release/26.0.0-BETA.3",
+        "master",
+    ]
+
+    def _refs(self, monkeypatch):
+        monkeypatch.setattr(
+            compat, "_ls_remote",
+            lambda remote, what: self.TAGS if what == "--tags" else self.HEADS)
+        return compat.discover_refs("origin")
+
+    def test_the_next_maintenance_release_is_checked(self, monkeypatch):
+        assert "release/25.10.5" in self._refs(monkeypatch), (
+            "the next thing a 25.10.4 box updates to is not checked, so a break in it "
+            "reaches users before the bot ever sees it"
+        )
+
+    def test_a_superseded_maintenance_branch_is_not(self, monkeypatch):
+        # 25.10.4.1 sorts OLDER than the newest tag TS-25.10.4? No -- it is NEWER, and
+        # both are on the 25.10 line, so only the newest branch on the line is taken.
+        refs = self._refs(monkeypatch)
+        assert "release/25.10.4.1" not in refs, "only the newest branch per line"
+
+    def test_the_typo_branch_stays_out(self, monkeypatch):
+        # 25.20 has no TS tag, so it is not a release line at all. A typo branch in the
+        # matrix reads as a real supported release we are silently broken on.
+        assert "release/25.20.2.2" not in self._refs(monkeypatch)
+
+    def test_a_prerelease_of_an_already_shipped_line_stays_out(self, monkeypatch):
+        assert "release/24.10-RC.2" not in self._refs(monkeypatch)
+
+    def test_an_untagged_branch_counts_as_UNRELEASED(self, monkeypatch):
+        # The exit code keys off this. Calling 25.10.5 "shipped" would fail the build
+        # as a live outage on a version nobody is running yet.
+        assert compat.is_unreleased("release/25.10.5")
+        assert compat.is_unreleased("master")
+        assert not compat.is_unreleased("TS-25.10.4")
+
+
+class TestMasterIsNotTheNextRelease:
+    """A red `master` row used to read as "the version you are about to install".
+
+    On 2026-07-14 master was 27-dev -- every recent commit targeted 27.0.0-BETA.1 --
+    while 26 was still in beta on its own branches. So `master BROKEN` meant "iX will
+    break us a major release from now", but the matrix said "master _(unreleased)_",
+    which any reader takes as the next thing out the door. For a table whose whole job
+    is helping somebody decide whether to trust this with their backups, that is a
+    false alarm in the worst possible place.
+    """
+
+    def _rows(self, refs):
+        return [{"ref": r, "unreleased": compat.is_unreleased(r), "modules": {}}
+                for r in refs]
+
+    def test_master_is_labelled_with_the_major_AFTER_the_newest_known_one(self):
+        rows = self._rows(["TS-25.10.4", "release/26.0.0-BETA.3", "master"])
+        assert compat.dev_label(rows) == "27-dev"
+
+    def test_it_rolls_over_on_its_own_when_the_next_beta_branches(self):
+        # Derived, not hardcoded: when release/27.0.0-BETA.1 appears, master is 28-dev.
+        rows = self._rows(["TS-26.0.0", "release/27.0.0-BETA.1", "master"])
+        assert compat.dev_label(rows) == "28-dev"
+
+    def test_the_rendered_matrix_says_dev_not_unreleased(self):
+        healthy = check_files(with_())
+        rows = [
+            {"ref": r, "unreleased": compat.is_unreleased(r), "modules": healthy}
+            for r in ("TS-25.10.4", "release/26.0.0-BETA.3", "master")
+        ]
+        md = compat.render_markdown(rows)
+        assert "master _(27-dev)_" in md
+        assert "master _(unreleased)_" not in md
+        # ...and the ordinary rows are untouched.
+        assert "| 25.10.4 |" in md
+        assert "| 26.0.0-BETA.3 _(unreleased)_ |" in md
+
+
+class TestTheBodyIsTruthAndCommentsAreTheChangelog:
+    """An unchanged fingerprint used to freeze the BODY, not just silence the comments.
+
+    Two different questions were sharing one answer. "Have the findings changed?" gates
+    COMMENTS -- they notify, and a daily "still broken, same as yesterday" is what
+    teaches people to ignore the one that finally matters. But "is the body still
+    true?" gates the BODY, and editing a body notifies nobody, so keeping it honest
+    costs nothing.
+
+    Conflated, an unchanged fingerprint meant the report could never be corrected --
+    and the fingerprint deliberately ignores everything that moves on its own, which
+    includes how a row is LABELLED. Relabelling master `27-dev` would have reached the
+    README and never the issue anybody opens.
+    """
+
+    ROWS = [{"ref": "master", "unreleased": True,
+             "modules": check_files(with_(**{
+                 "plugins/cloud_backup/restic.py":
+                     "class ResticConfig:\n    cmd: list\n\n"
+                     "def get_restic_config(entry, credentials):\n    pass\n",
+             }))}]
+
+    def _run(self, monkeypatch, tmp_path, existing_body):
+        calls = []
+
+        def fake(url, token, method="GET", data=None):
+            calls.append((method, url, data))
+            if url.endswith("/issues?state=all&per_page=100&limit=100"):
+                return [{"number": 1, "title": compat_publish.TITLE,
+                         "state": "open", "body": existing_body,
+                         "pull_request": None}]
+            return {"number": 1}
+
+        monkeypatch.setattr(compat_publish, "_call", fake)
+        matrix = tmp_path / "m.json"
+        matrix.write_text(json.dumps(self.ROWS))
+        compat_publish.main([
+            "prog", "--api", "https://forge/api", "--token", "t",
+            "--matrix", str(matrix)])
+        return calls
+
+    def _writes(self, calls):
+        patched = [c for c in calls if c[0] == "PATCH"]
+        commented = [c for c in calls if c[0] == "POST" and c[1].endswith("/comments")]
+        return patched, commented
+
+    def test_identical_body_and_findings_touches_nothing(self, monkeypatch, tmp_path):
+        body = compat.render_issue(self.ROWS)
+        patched, commented = self._writes(self._run(monkeypatch, tmp_path, body))
+        assert not patched and not commented, "a quiet run must be completely silent"
+
+    def test_a_relabel_refreshes_the_body_but_says_NOTHING(self, monkeypatch, tmp_path):
+        # Same findings (same fingerprint), different rendering -- the exact shape of
+        # the master -> 27-dev relabel.
+        stale = compat.render_issue(self.ROWS).replace("27-dev", "unreleased")
+        assert compat.extract_fingerprint(stale) == compat.fingerprint(self.ROWS)
+
+        patched, commented = self._writes(self._run(monkeypatch, tmp_path, stale))
+        assert patched, "the body was left stale, so the issue keeps telling lies"
+        assert "27-dev" in patched[0][2]["body"]
+        assert not commented, (
+            "a rendering change is not news -- commenting on it is how the bot gets "
+            "muted before the next real finding"
+        )
+
+    def test_a_REAL_findings_change_still_comments(self, monkeypatch, tmp_path):
+        # ...and the fix must not have made it mute.
+        stale = compat.render_issue(self.ROWS).replace(
+            compat.fingerprint(self.ROWS), "0" * 16)
+        patched, commented = self._writes(self._run(monkeypatch, tmp_path, stale))
+        assert patched and commented, "a genuine change must still notify"
+
+    def test_a_body_differing_only_by_CRLF_is_not_rewritten(self, monkeypatch, tmp_path):
+        # Forges round-trip line endings. Without normalising, every run would rewrite
+        # the body -- silent, but it churns updated_at and looks freshly touched daily.
+        body = compat.render_issue(self.ROWS).replace("\n", "\r\n")
+        patched, _ = self._writes(self._run(monkeypatch, tmp_path, body))
+        assert not patched

tests/test_docs.py

@@ -0,0 +1,133 @@
+"""The docs must not lie about themselves.
+
+The README was 969 lines with the install instructions at line 517. Splitting it into
+docs/ fixed that and broke every cross-reference in the process -- which is the normal
+outcome of moving Markdown around, and exactly why this is a test rather than a
+careful afternoon.
+
+A dead link in a recovery doc is worse than a dead link anywhere else: the person
+following it is, by definition, already having a bad day.
+"""
+
+import os
+import re
+
+import pytest
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DOCS = os.path.join(ROOT, "docs")
+
+LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
+HEADING_RE = re.compile(r"^#{1,6}\s+(.*)$", re.M)
+
+
+def markdown_files():
+    files = [os.path.join(ROOT, "README.md"), os.path.join(ROOT, "CHANGELOG.md")]
+    if os.path.isdir(DOCS):
+        files += [os.path.join(DOCS, f) for f in sorted(os.listdir(DOCS))
+                  if f.endswith(".md")]
+    return files
+
+
+def anchors(text):
+    """GitHub/Gitea slugs for every heading in `text`."""
+    out = set()
+    for h in HEADING_RE.findall(text):
+        slug = re.sub(r"[^a-z0-9 -]", "", h.lower()).replace(" ", "-")
+        out.add(slug)
+    return out
+
+
+@pytest.mark.parametrize("path", markdown_files(), ids=os.path.basename)
+def test_every_internal_link_resolves(path):
+    with open(path, encoding="utf-8") as fh:
+        text = fh.read()
+    here = anchors(text)
+    base = os.path.dirname(path)
+
+    broken = []
+    for label, target in LINK_RE.findall(text):
+        if target.startswith(("http://", "https://", "mailto:")):
+            continue
+        rel, _, anchor = target.partition("#")
+
+        if not rel:                                  # same-file anchor
+            if anchor and anchor not in here:
+                broken.append(f"[{label}](#{anchor}) — no such heading here")
+            continue
+
+        dest = os.path.normpath(os.path.join(base, rel))
+        if not os.path.exists(dest):
+            broken.append(f"[{label}]({target}) — file does not exist")
+            continue
+
+        if anchor and dest.endswith(".md"):
+            with open(dest, encoding="utf-8") as fh:
+                if anchor not in anchors(fh.read()):
+                    broken.append(f"[{label}]({target}) — no such heading there")
+
+    assert not broken, "broken links in {}:\n  {}".format(
+        os.path.basename(path), "\n  ".join(broken)
+    )
+
+
+class TestTheReadmeStaysAReadme:
+    def test_install_is_near_the_top(self):
+        # It was at line 517 of 969, under a wall of internals. Somebody deciding
+        # whether to use this should not have to scroll past the boot sequence.
+        with open(os.path.join(ROOT, "README.md"), encoding="utf-8") as fh:
+            lines = fh.read().splitlines()
+        install = next(i for i, ln in enumerate(lines, 1) if ln.startswith("## Install"))
+        assert install < 40, f"## Install is at line {install}"
+
+    def test_the_readme_does_not_grow_back(self):
+        with open(os.path.join(ROOT, "README.md"), encoding="utf-8") as fh:
+            n = len(fh.read().splitlines())
+        assert n < 300, (
+            f"README is {n} lines. Detail belongs in docs/ — the README is what "
+            f"someone reads before they trust this with their backups."
+        )
+
+    def test_the_minimum_version_is_stated_before_the_install_command(self):
+        with open(os.path.join(ROOT, "README.md"), encoding="utf-8") as fh:
+            text = fh.read()
+        assert "24.10" in text[:text.index("## Install")], (
+            "the minimum TrueNAS version must be visible above the install steps"
+        )
+
+
+class TestInstallDoesNotDirtyTheCheckout:
+    """install.sh chmod +x's scripts. If git records them as 100644, that chmod is a
+    TRACKED MODIFICATION -- and update.sh refuses to run over a dirty tree.
+
+    So installing once permanently blocked updating, for every user, with a message
+    telling them to `git checkout -- .` (which would just undo the exec bit and let
+    the next install re-dirty it). Found on a real box that had been stuck on an old
+    version for exactly this reason.
+
+    Every script install.sh makes executable must already be executable in git.
+    """
+
+    def test_every_chmodded_script_is_already_executable_in_git(self):
+        import re
+        import subprocess
+
+        with open(os.path.join(ROOT, "install.sh"), encoding="utf-8") as fh:
+            m = re.search(r"^for _exe in (.+?); do", fh.read(), re.M)
+        assert m, "could not find install.sh's chmod loop"
+        scripts = m.group(1).split()
+
+        out = subprocess.run(
+            ["git", "ls-files", "-s", *scripts],
+            cwd=ROOT, capture_output=True, text=True, check=True,
+        ).stdout
+
+        not_exec = [
+            line.split("\t")[-1] for line in out.strip().splitlines()
+            if not line.startswith("100755")
+        ]
+        assert not not_exec, (
+            "install.sh chmod +x's these, but git records them as non-executable — "
+            "so installing dirties the checkout and update.sh then refuses to run:\n  "
+            + "\n  ".join(not_exec)
+        )

tests/test_release_gate.py

@@ -0,0 +1,264 @@
+"""Tests for the barrier: a stable release must have been a release candidate.
+
+This exists because the repo cut twelve releases in one day, several of them
+fixing the release before -- and with the update alert live, every one of those
+interrupts every user. The gate makes that path impossible rather than impolite.
+
+The provenance gate is the one thing here that can wrongly PASS in a way nobody
+notices (a wrongly-failing gate is loud; a wrongly-passing gate silently restores
+the old behaviour), so it gets tested against real git repositories, not mocks.
+"""
+
+import os
+import subprocess
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
+
+from release_gate import (  # noqa: E402
+    check_promotable,
+    next_rc,
+    rc_tags,
+)
+from release_notes import (  # noqa: E402
+    base_version,
+    is_prerelease,
+    promote,
+    unreleased_body,
+)
+
+
+# ── a real git repo, because the gate reads real tags ────────────────────────
+
+class Repo:
+    """A throwaway git repo. The gate reads real tags, so the tests build real ones."""
+
+    def __init__(self, path):
+        self.path = path
+
+    def __str__(self):
+        return str(self.path)
+
+    def git(self, *args):
+        return subprocess.run(
+            ["git", *args], cwd=self.path, capture_output=True, text=True, check=True,
+        ).stdout.strip()
+
+    def commit(self, msg):
+        (self.path / "f").write_text(msg)
+        self.git("add", "-A")
+        self.git("commit", "-q", "-m", msg)
+        return self.git("rev-parse", "HEAD")
+
+
+@pytest.fixture
+def repo(tmp_path):
+    d = tmp_path / "repo"
+    d.mkdir()
+    r = Repo(d)
+    r.git("init", "-q", "-b", "main")
+    r.git("config", "user.email", "t@example.com")
+    r.git("config", "user.name", "t")
+    r.commit("initial")
+    return r
+
+
+class TestTheBarrierBeforeTheTagExists:
+    """release.sh calls the gate BEFORE creating the stable tag.
+
+    Every other test here tags first, and that is what let a fatal bug ship green:
+    `check_promotable` began with `commit_for(vX.Y.Z)` and returned "does not exist",
+    while release.sh's own guard refuses to run at all IF the tag exists. The two
+    conditions were mutually exclusive, so `--promote` could never succeed -- the
+    only way to cut a stable release was to hand-tag, bypassing every gate.
+
+    A gate that can only be satisfied after the thing it gates is not a gate.
+    """
+
+    def test_promote_is_allowed_when_head_was_a_candidate_and_the_tag_is_absent(self, repo):
+        repo.git("tag", "v1.0.0-rc1")          # rc on HEAD, no stable tag yet
+        assert check_promotable("v1.0.0", cwd=str(repo)) == []
+
+    def test_promote_is_refused_when_head_was_never_a_candidate(self, repo):
+        assert check_promotable("v1.0.0", cwd=str(repo))
+
+    def test_promote_is_refused_when_head_moved_past_the_candidate(self, repo):
+        repo.git("tag", "v1.0.0-rc1")
+        repo.commit("one more little fix")     # HEAD is no longer the candidate
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "no release candidate does" in problems[0]
+
+
+class TestTheBarrier:
+    def test_a_tag_with_no_candidate_is_refused(self, repo):
+        repo.git("tag", "v1.0.0")
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "never a release candidate" in problems[0]
+
+    def test_a_tag_whose_candidate_is_on_the_same_commit_is_allowed(self, repo):
+        repo.git("tag", "v1.0.0-rc1")
+        repo.git("tag", "v1.0.0")
+        assert check_promotable("v1.0.0", cwd=str(repo)) == []
+
+    def test_one_more_little_fix_after_the_rc_is_refused(self, repo):
+        # THE case this whole mechanism exists for. The candidate passed, then a
+        # "trivial" commit landed, and the stable tag ships code no candidate ever
+        # tested. That is how v0.5.1 happened.
+        repo.git("tag", "v1.0.0-rc1")
+        repo.commit("just a tiny fix, surely fine")
+        repo.git("tag", "v1.0.0")
+
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "no release candidate does" in problems[0]
+        assert "v1.0.0-rc2" in problems[0], "must say how to fix it"
+
+    def test_an_rc_for_a_different_version_does_not_count(self, repo):
+        repo.git("tag", "v0.9.0-rc1")
+        repo.git("tag", "v1.0.0")
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "never a release candidate" in problems[0]
+
+    def test_candidates_themselves_are_never_gated(self, repo):
+        # Requiring an rc to have an rc would be a deadlock.
+        repo.git("tag", "v1.0.0-rc1")
+        assert check_promotable("v1.0.0-rc1", cwd=str(repo)) == []
+
+    def test_a_later_candidate_on_the_right_commit_rescues_it(self, repo):
+        repo.git("tag", "v1.0.0-rc1")
+        repo.commit("fix found during rc1")
+        repo.git("tag", "v1.0.0-rc2")   # re-cut on the fixed commit
+        repo.git("tag", "v1.0.0")
+        assert check_promotable("v1.0.0", cwd=str(repo)) == []
+
+    def test_a_tag_the_numbering_does_not_understand_is_not_a_candidate(self, repo):
+        # `v1.0.0-rc*` also globs `v1.0.0-rc1-hotfix`, which _rc_number reads as 0.
+        # The barrier must be satisfied only by something that really was a candidate.
+        repo.git("tag", "v1.0.0-rc1-hotfix")
+        repo.git("tag", "v1.0.0")
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "never a release candidate" in problems[0]
+        assert rc_tags("1.0.0", cwd=str(repo)) == []
+
+
+class TestRcNumbering:
+    def test_first_candidate_is_rc1(self, repo):
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc1"
+
+    def test_it_counts_up(self, repo):
+        repo.git("tag", "v1.0.0-rc1")
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc2"
+        repo.git("tag", "v1.0.0-rc2")
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc3"
+
+    def test_rc10_sorts_after_rc9_not_before(self, repo):
+        # Lexicographic sorting would rank rc10 before rc9 and hand out a duplicate.
+        for n in range(1, 11):
+            repo.git("tag", f"v1.0.0-rc{n}")
+        assert rc_tags("1.0.0", cwd=str(repo))[-1] == "v1.0.0-rc10"
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc11"
+
+    def test_other_versions_do_not_leak_in(self, repo):
+        repo.git("tag", "v0.9.0-rc7")
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc1"
+
+
+# ── the content half of the gate ─────────────────────────────────────────────
+
+class TestPrereleaseDetection:
+    @pytest.mark.parametrize("tag", ["v1.2.3-rc1", "v1.2.3-rc10", "1.2.3-beta",
+                                     "v1.2.3-alpha2", "V1.2.3-RC1"])
+    def test_prereleases(self, tag):
+        assert is_prerelease(tag)
+
+    @pytest.mark.parametrize("tag", ["v1.2.3", "1.2.3", "v0.0.1"])
+    def test_stable(self, tag):
+        assert not is_prerelease(tag)
+
+    def test_base_version_strips_the_suffix(self):
+        assert base_version("v1.2.3-rc4") == "1.2.3"
+        assert base_version("v1.2.3") == "1.2.3"
+
+
+class TestUnreleasedSection:
+    def test_body_is_extracted(self):
+        text = "# C\n\n## Unreleased\n\n### Fixed\n- a thing\n\n## v1.0.0 — 2026-01-01\n\n- old\n"
+        assert "- a thing" in unreleased_body(text)
+        assert "old" not in unreleased_body(text)
+
+    def test_empty_section_reads_as_empty(self):
+        text = "# C\n\n## Unreleased\n\n## v1.0.0 — 2026-01-01\n\n- old\n"
+        assert unreleased_body(text) == ""
+
+    def test_absent_section_reads_as_empty(self):
+        assert unreleased_body("# C\n\n## v1.0.0 — 2026-01-01\n\n- old\n") == ""
+
+    def test_promote_renames_the_heading_and_keeps_the_body(self):
+        text = "# C\n\n## Unreleased\n\n### Fixed\n- a thing\n\n## v1.0.0 — 2026-01-01\n"
+        out = promote(text, "1.1.0", "2026-07-13")
+        assert "## v1.1.0 — 2026-07-13" in out
+        assert "## Unreleased" not in out
+        assert "- a thing" in out
+        assert "## v1.0.0 — 2026-01-01" in out, "older sections survive"
+
+    def test_promoting_nothing_is_refused(self):
+        # A release with no content is a release nobody needed -- and it still
+        # alerts every box.
+        with pytest.raises(ValueError, match="nothing to release"):
+            promote("# C\n\n## v1.0.0 — 2026-01-01\n", "1.1.0", "2026-07-13")
+
+
+class TestStrandedWorkBlocksAStableRelease:
+    """`check()` refuses a stable tag that leaves work under `## Unreleased`.
+
+    Either it is finished and belongs in the release, or the release is premature.
+    """
+
+    def _tree(self, tmp_path, changelog):
+        from release_notes import VERSIONED_FILES
+        for rel in VERSIONED_FILES:
+            p = tmp_path / rel
+            p.parent.mkdir(parents=True, exist_ok=True)
+            marker = "__version__ = " if rel.endswith(".py") else "VERSION="
+            p.write_text(f'{marker}"1.0.0"\n')
+        (tmp_path / "CHANGELOG.md").write_text(changelog)
+        return str(tmp_path)
+
+    def test_stranded_work_is_refused_for_a_stable_tag(self, tmp_path):
+        from release_notes import check
+        root = self._tree(tmp_path, (
+            "# C\n\n## Unreleased\n\n### Fixed\n- not done yet\n\n"
+            "## v1.0.0 — 2026-07-13\n\n### Added\n- the thing\n"
+        ))
+        problems = check("v1.0.0", root=root)
+        assert any("Unreleased" in p for p in problems)
+
+    def test_stranded_work_is_fine_for_a_candidate(self, tmp_path):
+        # An rc may legitimately have more work queued behind it.
+        from release_notes import check
+        root = self._tree(tmp_path, (
+            "# C\n\n## Unreleased\n\n### Fixed\n- later\n\n"
+            "## v1.0.0 — 2026-07-13\n\n### Added\n- the thing\n"
+        ))
+        assert check("v1.0.0-rc1", root=root) == []
+
+    def test_a_clean_stable_release_passes(self, tmp_path):
+        from release_notes import check
+        root = self._tree(tmp_path, (
+            "# C\n\n## v1.0.0 — 2026-07-13\n\n### Added\n- the thing\n"
+        ))
+        assert check("v1.0.0", root=root) == []
+
+    def test_a_candidate_checks_against_its_base_version(self, tmp_path):
+        # The scripts say 1.0.0; the tag says v1.0.0-rc3. That must agree, not clash.
+        from release_notes import check
+        root = self._tree(tmp_path, (
+            "# C\n\n## v1.0.0 — 2026-07-13\n\n### Added\n- the thing\n"
+        ))
+        assert check("v1.0.0-rc3", root=root) == []

tests/test_release_notes.py

@@ -9,6 +9,7 @@ across install.sh / uninstall.sh / recover.sh / apply.sh and nothing noticed.
 """
 
 import os
+import re
 import sys
 
 import pytest
@@ -21,6 +22,8 @@ from release_notes import (  # noqa: E402
     extract_notes,
     normalise,
     script_versions,
+    significance,
+    version_tuple,
 )
 
 REPO = os.path.join(os.path.dirname(__file__), "..")
@@ -106,9 +109,14 @@ class TestAgainstTheRealRepo:
             f"scripts say v{version} but the newest CHANGELOG entry is v{newest}"
         )
 
-    def test_check_passes_for_the_current_version(self):
+    def test_the_repo_is_always_releasable_as_a_candidate(self):
+        # Deliberately checked as an rc, not as a stable release. `main` carries
+        # work under `## Unreleased` most of the time, and the stable gate refuses
+        # that on purpose -- shipping with work stranded mid-section is how you get
+        # a release that needs another release. So the invariant main must uphold is
+        # the candidate one: versions agree, and the CHANGELOG section exists.
         version = next(iter({normalise(v) for v in script_versions(REPO).values()}))
-        assert check(version, REPO) == []
+        assert check(f"v{version}-rc1", REPO) == []
 
 
 class TestCheckCatchesMistakes:
@@ -120,3 +128,166 @@ class TestCheckCatchesMistakes:
     def test_reports_a_missing_changelog_section(self):
         problems = check("v9.9.9", REPO)
         assert any("no section" in p for p in problems)
+
+
+class TestSignificance:
+    """Drives the TrueNAS update alert: what is worth bothering a human about.
+
+    The rule: a release whose CHANGELOG only has a "### Docs" section changed no
+    code, and nobody should get an alert because a README was reworded.
+    """
+
+    TEXT = """\
+# Changelog
+
+## v0.4.2 — 2026-07-13
+
+### Docs
+
+- reworded the README
+
+## v0.4.1 — 2026-07-13
+
+### Fixed
+
+- a real bug
+
+## v0.4.0 — 2026-07-13
+
+### Added
+
+- a feature
+
+## v0.3.3 — 2026-07-13
+
+### Security
+
+- keep a password out of argv
+
+## v0.3.2 — 2026-07-13
+
+### Fixed
+
+- something
+"""
+
+    def test_docs_only_release_does_not_alert(self):
+        level, versions, _ = significance(self.TEXT, "0.4.1", "0.4.2")
+        assert level == "docs"
+        assert versions == ["0.4.2"]
+
+    def test_a_real_fix_alerts(self):
+        level, _v, _h = significance(self.TEXT, "0.4.0", "0.4.1")
+        assert level == "notable"
+
+    def test_security_in_range_escalates(self):
+        level, _v, _h = significance(self.TEXT, "0.3.2", "0.3.3")
+        assert level == "security"
+
+    def test_security_wins_even_when_the_newest_release_is_docs_only(self):
+        # A docs-only v0.4.2 sitting on top of a security-fixing v0.3.3 must still
+        # be reported as security — classify the whole span, not just the tip.
+        level, versions, _ = significance(self.TEXT, "0.3.2", "0.4.2")
+        assert level == "security"
+        assert set(versions) == {"0.3.3", "0.4.0", "0.4.1", "0.4.2"}
+
+    def test_same_version_is_never_notable(self):
+        level, versions, _ = significance(self.TEXT, "0.4.2", "0.4.2")
+        assert level == "docs"
+        assert versions == []
+
+    def test_range_is_exclusive_of_current_inclusive_of_latest(self):
+        _l, versions, _h = significance(self.TEXT, "0.4.0", "0.4.2")
+        assert "0.4.0" not in versions
+        assert "0.4.2" in versions
+
+    def test_version_tuple_orders_correctly(self):
+        assert version_tuple("v0.10.0") > version_tuple("v0.9.9")
+        assert version_tuple("0.4.2") > version_tuple("0.4.1")
+        # Pre-release suffixes are dropped, not ranked above the release.
+        assert version_tuple("v0.5.0-rc1") == version_tuple("v0.5.0")
+
+
+class TestCandidateNotesResolveToTheBaseVersion:
+    """`notes v0.6.0-rc1` must return v0.6.0's section.
+
+    A candidate ships the same code as the release it is a candidate for, and the
+    CHANGELOG only ever has the one section. Without this, the release workflow cut
+    v0.6.0-rc1, passed every gate, and then died extracting the body -- so the tag
+    existed but nothing was ever published. Caught in an rc, which is the entire
+    point of having them.
+    """
+
+    CHANGELOG = "# C\n\n## v0.6.0 — 2026-07-13\n\n### Added\n- the thing\n\n## v0.5.1 — 2026-07-13\n\n- older\n"
+
+    def test_an_rc_resolves_to_its_base_version(self):
+        body = extract_notes(self.CHANGELOG, "v0.6.0-rc1")
+        assert "the thing" in body
+        assert "older" not in body
+
+    def test_rc10_too(self):
+        assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0-rc10")
+
+    def test_the_plain_version_still_works(self):
+        assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0")
+
+    def test_a_genuinely_missing_section_still_raises(self):
+        with pytest.raises(KeyError):
+            extract_notes(self.CHANGELOG, "v9.9.9-rc1")
+
+
+class TestTheChangelogIsStructurallySound:
+    """The release body IS this file, so a mangled section ships to every user.
+
+    It has been mangled once: an edit matched the literal `## Unreleased` inside a
+    backticked phrase in a prose bullet and spliced a whole new section into the middle
+    of it, splitting the sentence in half.
+    """
+
+    def changelog(self):
+        with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh:
+            return fh.read()
+
+    def test_no_version_section_is_empty(self):
+        text = self.changelog()
+        for v in changelog_versions(text):
+            assert extract_notes(text, v).strip(), f"v{v} has an empty section"
+
+    def test_versions_are_in_descending_order(self):
+        from release_notes import version_tuple
+        versions = changelog_versions(self.changelog())
+        assert versions == sorted(versions, key=version_tuple, reverse=True), (
+            "CHANGELOG versions are out of order — a section was spliced in wrong"
+        )
+
+    def test_headings_are_at_the_start_of_a_line_and_not_inside_prose(self):
+        # A `### Fixed` that ends up indented under a bullet is a section nobody sees.
+        for i, line in enumerate(self.changelog().splitlines(), 1):
+            if line.lstrip().startswith(("## ", "### ")) and line != line.lstrip():
+                raise AssertionError(
+                    f"line {i}: heading is indented, so it is inside a list item "
+                    f"rather than being a section: {line!r}"
+                )
+
+    def test_every_bullet_that_opens_a_bold_phrase_closes_it(self):
+        # The splice cut `- **A stable release ... under \`## Unreleased` in half,
+        # leaving an unterminated ** and a dangling sentence.
+        #
+        # A bullet is the `- ` line plus everything up to the next top-level bullet or
+        # heading -- bold phrases routinely wrap across lines, so a per-line check
+        # would flag every long bullet in the file.
+        text = self.changelog()
+        bullets = re.split(r"^(?=- |#{2,3} )", text, flags=re.M)
+        bad = []
+        for b in bullets:
+            if not b.startswith("- "):
+                continue
+            # Code spans are not markup: `*args, **kwargs` is a literal, not a bold
+            # phrase, and counting its ** would flag a perfectly well-formed bullet.
+            prose = re.sub(r"`[^`]*`", "", b)
+            if prose.count("**") % 2:
+                bad.append(b.splitlines()[0][:70])
+        assert not bad, (
+            "unbalanced ** in a bullet — a section was probably spliced into the "
+            "middle of it:\n  " + "\n  ".join(bad)
+        )

tests/test_workflows.py

@@ -0,0 +1,174 @@
+"""Guards on the CI workflows themselves.
+
+The workflows run on TWO forges -- Gitea (canonical) and GitHub (mirror), because
+Gitea reads .github/workflows too -- and they hold tokens. A mistake here is not a
+failed build, it is a bug report nobody files or a command nobody meant to run.
+"""
+
+import os
+import re
+
+import pytest
+
+ROOT = os.path.join(os.path.dirname(__file__), "..")
+WORKFLOWS = os.path.join(ROOT, ".github", "workflows")
+
+
+def workflow_files():
+    return [
+        os.path.join(WORKFLOWS, f)
+        for f in sorted(os.listdir(WORKFLOWS))
+        if f.endswith((".yml", ".yaml"))
+    ]
+
+
+def run_bodies(path):
+    """Every `run:` block's text, with its line number."""
+    with open(path, encoding="utf-8") as fh:
+        lines = fh.readlines()
+
+    out = []
+    i = 0
+    while i < len(lines):
+        m = re.match(r"^(\s*)run:\s*\|", lines[i])
+        if not m:
+            i += 1
+            continue
+        indent = len(m.group(1))
+        start = i + 1
+        body = []
+        i += 1
+        while i < len(lines):
+            line = lines[i]
+            if line.strip() and (len(line) - len(line.lstrip())) <= indent:
+                break
+            body.append(line)
+            i += 1
+        out.append((start + 1, "".join(body)))
+    return out
+
+
+class TestNoExpressionInterpolationIntoShell:
+    """`${{ ... }}` inside a `run:` body is spliced into the SCRIPT TEXT.
+
+    This is not theoretical. `echo "${{ steps.report.outputs.body }}"` in the compat
+    workflow pasted the report -- which is full of backticks -- straight into bash,
+    which promptly ran `create-snapshot`, `def` and `async` as commands. And because
+    that report is built from iX's middleware source, anything landing in their tree
+    would have executed on our runner.
+
+    The rule: files for data, `env:` for scalars. `env:` is safe because the runner
+    sets the variable rather than pasting it into the script.
+    """
+
+    @pytest.mark.parametrize("path", workflow_files(), ids=os.path.basename)
+    def test_no_github_expression_in_a_run_body(self, path):
+        offenders = []
+        for lineno, body in run_bodies(path):
+            for m in re.finditer(r"\$\{\{[^}]*\}\}", body):
+                offenders.append(f"{os.path.basename(path)}:~{lineno}: {m.group(0)}")
+        assert not offenders, (
+            "GitHub/Gitea expressions interpolate into the shell script text, so "
+            "backticks and $() in the value EXECUTE. Pass data via a file, or a "
+            "scalar via `env:`.\n  " + "\n  ".join(offenders)
+        )
+
+
+class TestBothForges:
+    """Gitea is canonical; GitHub is a mirror. Both run these files."""
+
+    def test_release_publishes_on_each_forge_exactly_once(self):
+        with open(os.path.join(WORKFLOWS, "release.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        # One step gated ON github.com, one gated OFF it. Without the pair, a release
+        # either double-publishes or silently never publishes on the canonical host.
+        assert "if: ${{ contains(github.server_url, 'github.com') }}" in src
+        assert "if: ${{ !contains(github.server_url, 'github.com') }}" in src
+
+    def test_compat_files_its_report_through_ONE_implementation(self):
+        # It used to be two near-identical shell steps, one per forge. Two copies of
+        # "find the issue, decide whether to comment, post it" is two chances to drift,
+        # and the Gitea one duplicated an issue for real.
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        assert "tools/compat_publish.py" in src
+        assert "file a bug report (GitHub)" not in src
+        assert "file a bug report (Gitea)" not in src
+
+
+class TestTheBotDoesNotSpam:
+    """It left 11 identical 3,000-character comments on one issue in a single day.
+
+    A bot that repeats itself daily gets muted — and then the next REAL finding is
+    scrolled past, which defeats the entire reason for building it.
+    """
+
+    def publisher(self):
+        with open(os.path.join(ROOT, "tools", "compat_publish.py"), encoding="utf-8") as fh:
+            return fh.read()
+
+    def test_it_compares_a_fingerprint_before_saying_anything(self):
+        src = self.publisher()
+        assert "extract_fingerprint" in src
+        assert "staying quiet" in src
+
+    def test_the_body_is_edited_in_place_not_appended_to(self):
+        src = self.publisher()
+        assert '"PATCH"' in src, "the issue body must be updated, not commented onto"
+
+    def test_it_closes_the_issue_when_everything_is_fixed(self):
+        src = self.publisher()
+        assert '"state": "closed"' in src
+
+    def test_the_matrix_refresh_opens_a_PR_rather_than_pushing_to_main(self):
+        # An unattended push to main from CI is exactly what the release barrier exists
+        # to prevent: a bot that can move main can move it somewhere nobody looked.
+        #
+        # Checked against CODE, not comments — the step's own commentary explains what
+        # it replaced, and that mention must not read as the thing itself.
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        code = "\n".join(
+            ln for ln in src.splitlines() if not ln.lstrip().startswith("#")
+        )
+        assert "/pulls" in code, "the matrix refresh must open a PR"
+        assert "HEAD:main" not in code, "CI still pushes straight to main"
+
+    def test_the_matrix_PR_targets_the_CANONICAL_forge_not_the_mirror(self):
+        # GitHub is a one-way mirror: a PR merged there would be silently clobbered by
+        # the next `fleet-repos mirror` push from Gitea. A bot opening PRs against a
+        # mirror is a bot doing nothing, slowly.
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        i = src.index("refresh the README matrix")
+        step = src[i:i + 400]
+        assert "!contains(github.server_url, 'github.com')" in step, (
+            "the matrix PR must be opened on Gitea (canonical), not GitHub (mirror)"
+        )
+
+    def test_the_workflow_has_the_permissions_its_steps_actually_need(self):
+        # It shipped with `contents: read` while the step pushed a branch and opened a
+        # PR — it would have died with a 403 on the first scheduled run, and I would
+        # have had a bot that silently never worked.
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        perms = src[src.index("permissions:"):src.index("jobs:")]
+        assert "contents: write" in perms, "pushing a branch needs contents: write"
+        assert "pull-requests: write" in perms, "opening a PR needs pull-requests: write"
+        assert "issues: write" in perms
+
+
+class TestCompatCannotSilentlyPass:
+    def test_the_exit_code_is_captured_not_swallowed(self):
+        # Actions runs `bash -e`: `cmd > out` followed by `echo $?` never reaches the
+        # echo, so the "a shipped release is broken" signal would be lost and the job
+        # would go green while users were broken.
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        assert "|| rc=$?" in src
+        assert "shipped_broken=$rc" in src
+
+    def test_a_broken_shipped_release_fails_the_job(self):
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        assert "steps.check.outputs.shipped_broken != '0'" in src

tools/compat_publish.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""Keep ONE bug report in sync with what compat.py currently finds.
+
+WHY THIS IS NOT JUST "POST A COMMENT"
+-------------------------------------
+The first version commented on every run that found a break. In one day it left
+**11 identical 3,000-character comments** on the same issue. That is not a warning
+system; it is a mute button with extra steps. The next real finding would have been
+scrolled past, which defeats the entire point of building it.
+
+So:
+
+  * **The issue body is the current truth.** It is edited in place, never appended to.
+  * **Comments are a changelog of CHANGES.** A run whose findings are identical to the
+    last one says nothing at all -- no comment, no edit, no notification.
+  * A fingerprint of the findings (broken ref/module/problem triples only) is embedded
+    in the body. It deliberately ignores things that move on their own -- healthy rows,
+    the hardware-verified column, TrueNAS point releases -- so `TS-25.10.4` becoming
+    `TS-25.10.5` is not news, and does not wake anybody up.
+
+  * When everything is fixed, the issue is **closed** with a comment saying so.
+
+Works against GitHub and Gitea, which differ only in the auth header and the issue
+list URL. One implementation, so the two cannot drift.
+
+    python3 tools/compat_publish.py --api <url> --token <tok> --matrix /tmp/matrix.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.error
+import urllib.request
+
+sys.path.insert(0, __file__.rsplit("/", 1)[0])
+
+from compat import (  # noqa: E402
+    extract_fingerprint,
+    fingerprint,
+    is_broken,
+    render_issue,
+)
+
+TITLE = "TrueNAS compatibility: the patch's assumptions no longer hold"
+
+
+def _call(url, token, method="GET", data=None):
+    req = urllib.request.Request(
+        url, method=method,
+        headers={
+            # Gitea wants `token <t>`; GitHub accepts `Bearer <t>`. GitHub also
+            # accepts `token <t>`, so one header serves both.
+            "Authorization": f"token {token}",
+            "Content-Type": "application/json",
+            "Accept": "application/vnd.github+json",
+        },
+        data=json.dumps(data).encode() if data else None,
+    )
+    with urllib.request.urlopen(req) as r:  # noqa: S310
+        return json.load(r) if r.length != 0 else {}
+
+
+def _same(a, b):
+    """Is the issue body already what we would write?
+
+    Compared after normalising line endings and trailing space: forges are free to
+    round-trip `\r\n`, and a body that only "differs" by that would be rewritten on
+    every single run -- a silent edit, but a pointless one that churns `updated_at`
+    and makes the issue look freshly touched every morning.
+    """
+    def norm(s):
+        return "\n".join(line.rstrip() for line in (s or "").replace("\r\n", "\n").split("\n")).strip()
+    return norm(a) == norm(b)
+
+
+def find_issue(api, token, title):
+    """The LOWEST-numbered issue with this title, open or closed.
+
+    Lowest, not "whichever the API returns first": two issues with the same title
+    existed once (an earlier version put the ref list in the title, so the identity
+    changed whenever that set changed), and an order-dependent pick would alternate
+    between them -- reopening one while commenting on the other.
+
+    Both forges list PRs alongside issues, but they SAY SO DIFFERENTLY: GitHub omits
+    the `pull_request` key on a plain issue, Gitea sends it as `null`. Testing for the
+    KEY therefore discards every Gitea issue as if it were a PR -- so this returned
+    None on every Gitea run, and the bot filed a brand-new duplicate report each time
+    instead of editing the one it already had. Test the VALUE; it is the only form
+    that is true on both.
+    """
+    issues = _call(f"{api}/issues?state=all&per_page=100&limit=100", token)
+    mine = [
+        i for i in issues
+        if i.get("title") == title and not i.get("pull_request")
+    ]
+    return min(mine, key=lambda i: i["number"]) if mine else None
+
+
+def main(argv):
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("--api", required=True, help="…/repos/<owner>/<repo>")
+    ap.add_argument("--token", required=True)
+    ap.add_argument("--matrix", required=True, help="compat.py --matrix --json output")
+    args = ap.parse_args(argv[1:])
+
+    with open(args.matrix, encoding="utf-8") as fh:
+        rows = json.load(fh)
+
+    broken = [r for r in rows if any(is_broken(m) for m in r["modules"].values())]
+    issue = find_issue(args.api, args.token, TITLE)
+
+    # ── everything is healthy ────────────────────────────────────────────────
+    if not broken:
+        if issue and issue["state"] == "open":
+            _call(f"{args.api}/issues/{issue['number']}/comments", args.token, "POST",
+                  {"body": "All of the patch's assumptions hold again on every "
+                           "checked TrueNAS version. Closing."})
+            _call(f"{args.api}/issues/{issue['number']}", args.token, "PATCH",
+                  {"state": "closed"})
+            print(f"closed #{issue['number']} — nothing is broken any more")
+        else:
+            print("nothing broken; no open report to close")
+        return 0
+
+    body = render_issue(rows)
+    want = fingerprint(rows)
+
+    # ── nothing to file yet ──────────────────────────────────────────────────
+    if issue is None:
+        made = _call(f"{args.api}/issues", args.token, "POST",
+                     {"title": TITLE, "body": body})
+        print(f"filed #{made['number']}")
+        return 0
+
+    have = extract_fingerprint(issue.get("body") or "")
+    n = issue["number"]
+
+    # ── two different questions, and they were being answered with one answer ────
+    #
+    #   * IS THE BODY STILL TRUE?     -> if not, rewrite it. Editing an issue body
+    #     notifies NOBODY on either forge, so keeping it honest is free.
+    #   * HAVE THE FINDINGS CHANGED?  -> only then comment. Comments DO notify, and a
+    #     daily "still broken, same as yesterday" is what teaches everyone to ignore
+    #     the one that finally matters.
+    #
+    # Conflating them meant an unchanged FINGERPRINT froze the BODY. The fingerprint
+    # deliberately ignores everything that moves on its own -- healthy rows, the
+    # hardware-verified column, point releases, how a row is LABELLED -- so none of
+    # that could ever reach the report. Relabelling master `27-dev` (it is not the
+    # next release; a red row there was reading as "the version you are about to
+    # install is broken") would have shipped to the README and never to the issue
+    # anybody actually opens.
+    body_is_current = _same(issue.get("body"), body)
+
+    if have == want and issue["state"] == "open" and body_is_current:
+        print(f"#{n} is already current ({want}) — staying quiet")
+        return 0
+
+    _call(f"{args.api}/issues/{n}", args.token, "PATCH", {"body": body, "state": "open"})
+
+    if have != want:
+        refs = ", ".join(f"`{r['ref']}`" for r in broken)
+        note = (
+            "The findings changed — the report above has been updated.\n\n"
+            f"Currently broken on: {refs}."
+            if have else
+            "This report is now kept up to date automatically: the body above always "
+            "reflects the current findings, and a comment is only added when they "
+            "change."
+        )
+        _call(f"{args.api}/issues/{n}/comments", args.token, "POST", {"body": note})
+        print(f"updated #{n}: {have} -> {want}")
+    elif issue["state"] != "open":
+        print(f"reopened #{n}")
+    else:
+        # Same findings, new rendering. Silent by design: nothing has changed that
+        # anybody needs waking up for, but the report should not be telling lies.
+        print(f"#{n}: findings unchanged ({want}); body refreshed silently")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

tools/release_gate.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""The barrier: a stable release must have been a release candidate first.
+
+CHECKS
+------
+release_notes.py checks the *content* of the tree (versions agree, CHANGELOG has a
+non-empty section, nothing stranded under Unreleased). This module checks the
+*provenance* of the commit: was this exact code ever a release candidate, and did
+that candidate pass CI?
+
+WHY
+---
+This repo cut twelve releases in a single day, several of them "fix the thing the
+last release broke". With an update alert live on every user's box, that is not
+iteration, it is nagging -- and it teaches people to ignore the alert that will one
+day carry a real security fix.
+
+The rule that makes the bad path impossible:
+
+    A stable vX.Y.Z tag is only publishable if a vX.Y.Z-rcN tag points at the SAME
+    commit, and that candidate's CI run passed.
+
+Release candidates are invisible to users: update.sh and the alert source both take
+the newest plain vX.Y.Z tag, so an rc is never offered as an update. Debugging
+therefore happens across rc1, rc2, rc3 -- where it costs nobody anything -- instead
+of across v0.5.0, v0.5.1, v0.5.2, where it costs everybody an alert.
+
+The commit must be *identical*, not merely an ancestor. "The rc passed, then I
+pushed one more little fix" is exactly the habit this exists to break.
+
+    python3 tools/release_gate.py v0.6.0        # exit 1 if not promotable
+    python3 tools/release_gate.py v0.6.0 --next-rc   # -> the rc tag to cut next
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+import sys
+
+from release_notes import base_version, is_prerelease, normalise
+
+
+def _git(*args: str, cwd: str | None = None) -> str:
+    return subprocess.run(
+        ["git", *args],
+        cwd=cwd, capture_output=True, text=True, check=True,
+    ).stdout.strip()
+
+
+def rc_tags(version: str, cwd: str | None = None) -> list[str]:
+    """Every rc tag for this version, oldest first (rc2 sorts after rc1).
+
+    The glob is only a prefilter; the anchored regex decides. `v1.0.0-rc*` also
+    matches `v1.0.0-rc1-hotfix`, which _rc_number reads as 0 -- so a tag the
+    numbering logic does not understand could satisfy the barrier while never having
+    been a release candidate.
+    """
+    want = re.escape(normalise(base_version(version)))
+    exact = re.compile(rf"^v{want}-rc\d+$")
+
+    out = _git("tag", "--list", f"v{normalise(base_version(version))}-rc*", cwd=cwd)
+    tags = [t.strip() for t in out.splitlines() if exact.match(t.strip())]
+    return sorted(tags, key=_rc_number)
+
+
+def _rc_number(tag: str) -> int:
+    m = re.search(r"-rc(\d+)$", tag)
+    return int(m.group(1)) if m else 0
+
+
+def next_rc(version: str, cwd: str | None = None) -> str:
+    """The next rc tag to cut: v0.6.0-rc1, then -rc2, ..."""
+    existing = rc_tags(version, cwd=cwd)
+    n = max((_rc_number(t) for t in existing), default=0) + 1
+    return f"v{normalise(base_version(version))}-rc{n}"
+
+
+def commit_for(ref: str, cwd: str | None = None) -> str | None:
+    try:
+        return _git("rev-list", "-n", "1", ref, cwd=cwd)
+    except subprocess.CalledProcessError:
+        return None
+
+
+def check_promotable(version: str, cwd: str | None = None) -> list[str]:
+    """Every reason v<version> may not be cut as a stable release.
+
+    Empty list means the barrier is satisfied.
+
+    The commit under test is the tag's if it exists, and HEAD otherwise. Both are
+    real: CI runs this AFTER the tag is pushed, and release.sh runs it BEFORE
+    creating the tag -- which is the whole point, since refusing after the tag
+    exists is too late to be a gate. Requiring the tag unconditionally made
+    `release.sh --promote` impossible: it dies if the tag already exists, and the
+    gate died if it did not, so the only way through was to hand-tag and bypass
+    every check this file exists to enforce.
+    """
+    if is_prerelease(version):
+        return []  # candidates are what the barrier exists to encourage
+
+    want = normalise(version)
+    tag = f"v{want}"
+
+    target = commit_for(tag, cwd=cwd)
+    if target is None:
+        target = commit_for("HEAD", cwd=cwd)
+    if target is None:
+        return ["cannot resolve a commit to release (no HEAD?)"]
+
+    candidates = rc_tags(want, cwd=cwd)
+    if not candidates:
+        return [
+            f"{tag} was never a release candidate. Cut one first:\n"
+            f"    bash release.sh {want} --rc\n"
+            f"Candidates are invisible to users -- debug there, not in a release."
+        ]
+
+    matching = [c for c in candidates if commit_for(c, cwd=cwd) == target]
+    if not matching:
+        newest = candidates[-1]
+        return [
+            f"{tag} points at {target[:12]}, but no release candidate does.\n"
+            f"    Candidates: {', '.join(candidates)}\n"
+            f"    Newest ({newest}) is at "
+            f"{(commit_for(newest, cwd=cwd) or '?')[:12]}.\n"
+            f"Code changed after the last candidate. That change is untested as a\n"
+            f"release: cut {next_rc(want, cwd=cwd)} and promote THAT commit."
+        ]
+
+    return []
+
+
+def main(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("version", help="e.g. v0.6.0")
+    ap.add_argument("--next-rc", action="store_true",
+                    help="print the next rc tag to cut, and exit")
+    ap.add_argument("-C", dest="cwd", default=None, help="run git in this directory")
+    args = ap.parse_args(argv[1:])
+
+    if args.next_rc:
+        print(next_rc(args.version, cwd=args.cwd))
+        return 0
+
+    problems = check_promotable(args.version, cwd=args.cwd)
+    for p in problems:
+        print(f"::error::{p}")
+    if problems:
+        return 1
+
+    print(f"v{normalise(args.version)} was a release candidate and may be promoted")
+    return 0
+
+
+if __name__ == "__main__":
+    # Running `python3 tools/release_gate.py` already puts tools/ on sys.path[0],
+    # which is what makes the `release_notes` import above resolve.
+    sys.exit(main(sys.argv))

tools/release_notes.py

@@ -29,6 +29,7 @@ VERSIONED_FILES = [
     "recover.sh",
     os.path.join("patch", "apply.sh"),
     os.path.join("patch", "create_task.py"),   # exposes `--version` to users
+    "update.sh",
 ]
 
 # `VERSION="x"` (shell) or `__version__ = "x"` (python).
@@ -36,10 +37,53 @@ _VERSION_RE = re.compile(r'^(?:VERSION=|__version__\s*=\s*)"([^"]+)"', re.M)
 _HEADING_RE = re.compile(r"^##\s+v?(\d+\.\d+\.\d+[^\s]*)", re.M)
 
 
+#: Work in progress lives here until a release promotes it. Batching through this
+#: section is what stops "tag, find bug, tag again" from becoming twelve releases.
+UNRELEASED = "Unreleased"
+
+_UNRELEASED_RE = re.compile(r"^##\s+Unreleased\s*$", re.M | re.I)
+_RC_RE = re.compile(r"-(rc|beta|alpha)\d*$", re.I)
+
+
 def normalise(v: str) -> str:
     return v.strip().lstrip("v")
 
 
+def is_prerelease(tag: str) -> bool:
+    """True for v1.2.3-rc1 / -beta / -alpha. Those never reach users."""
+    return bool(_RC_RE.search(tag.strip()))
+
+
+def base_version(tag: str) -> str:
+    """v1.2.3-rc2 -> 1.2.3"""
+    return _RC_RE.sub("", normalise(tag))
+
+
+def unreleased_body(text: str) -> str:
+    """Content under `## Unreleased`, or "" if the section is absent/empty."""
+    m = _UNRELEASED_RE.search(text)
+    if not m:
+        return ""
+    rest = text[m.end():]
+    nxt = _HEADING_RE.search(rest)
+    return (rest[:nxt.start()] if nxt else rest).strip()
+
+
+def promote(text: str, version: str, date: str) -> str:
+    """Rename `## Unreleased` to `## vX.Y.Z — date`.
+
+    Refuses if the section is missing or empty: a release with nothing in it is a
+    release nobody needed, and cutting one only trains people to ignore alerts.
+    """
+    if not unreleased_body(text):
+        raise ValueError(
+            "CHANGELOG.md has no `## Unreleased` content — nothing to release. "
+            "Add your changes there first."
+        )
+    m = _UNRELEASED_RE.search(text)
+    return text[:m.start()] + f"## v{normalise(version)} — {date}" + text[m.end():]
+
+
 def script_versions(root: str = ROOT) -> dict[str, str]:
     """VERSION= as declared by each script."""
     found = {}
@@ -63,10 +107,15 @@ def changelog_versions(text: str) -> list[str]:
 def extract_notes(text: str, version: str) -> str:
     """The body of one version's section, without its heading.
 
+    A release candidate resolves to its BASE version: v0.6.0-rc2 ships the same code
+    as v0.6.0 and therefore the same notes, and the CHANGELOG only ever has the one
+    section. Without this, the release workflow cut the tag, passed every gate, and
+    then died extracting the body -- so the candidate existed but was never published.
+
     Raises KeyError if the version has no section -- a release with an empty or
     wrong body is worse than a failed release.
     """
-    want = normalise(version)
+    want = base_version(version)
     lines = text.splitlines()
 
     start = None
@@ -87,9 +136,67 @@ def extract_notes(text: str, version: str) -> str:
     return "\n".join(lines[start:end]).strip()
 
 
+# ── significance ──────────────────────────────────────────────────────────────
+# Used by the TrueNAS update alert to decide whether a release is worth bothering
+# anyone about. The CHANGELOG's own section headings are the signal: a release that
+# only has "### Docs" changed no code, and nobody should get an alert for a README.
+
+_SECTION_RE = re.compile(r"^###\s+(.+?)\s*$", re.M)
+
+#: Headings that mean "nothing about the running system changed".
+QUIET_SECTIONS = {"docs", "documentation"}
+
+
+def version_tuple(v: str) -> tuple:
+    """Sortable version. Pre-release suffixes are dropped, not ranked."""
+    return tuple(int(x) for x in normalise(v).split("-")[0].split("."))
+
+
+def section_headings(body: str) -> list[str]:
+    """The `### ...` headings inside one version's body, lowercased."""
+    return [h.strip().lower() for h in _SECTION_RE.findall(body)]
+
+
+def significance(text: str, current: str, latest: str):
+    """How much does upgrading `current` -> `latest` actually matter?
+
+    Returns ``(level, versions, headings)`` where level is one of:
+
+      "security"  a release in the range has a Security section  -> alert loudly
+      "notable"   something about the system changed             -> alert quietly
+      "docs"      only documentation changed                     -> DO NOT alert
+
+    Considers every release in the range, not just the newest: a docs-only v0.4.2
+    on top of a security-fixing v0.4.1 must still be reported as security.
+    """
+    cur, lat = version_tuple(current), version_tuple(latest)
+
+    versions = [
+        v for v in changelog_versions(text)
+        if cur < version_tuple(v) <= lat
+    ]
+
+    headings = []
+    for v in versions:
+        try:
+            headings.extend(section_headings(extract_notes(text, v)))
+        except KeyError:
+            continue
+
+    if any(h.startswith("security") for h in headings):
+        return "security", versions, headings
+    if [h for h in headings if h not in QUIET_SECTIONS]:
+        return "notable", versions, headings
+    return "docs", versions, headings
+
+
 def check(version: str, root: str = ROOT) -> list[str]:
-    """Every reason this version is not releasable. Empty list means it is."""
-    want = normalise(version)
+    """Every reason this version is not releasable. Empty list means it is.
+
+    `version` may be a release candidate (v1.2.3-rc2); the scripts and CHANGELOG
+    are checked against its BASE version, since an rc ships the same code.
+    """
+    want = base_version(version)
     problems = []
 
     versions = script_versions(root)
@@ -115,6 +222,15 @@ def check(version: str, root: str = ROOT) -> list[str]:
         if not body:
             problems.append(f"CHANGELOG.md section for v{want} is empty")
 
+    # A stable release must not leave work stranded under `## Unreleased`. If it is
+    # finished enough to ship, it belongs in the release; if it is not, the release
+    # is premature. (An rc may legitimately have more work queued behind it.)
+    if not is_prerelease(version) and unreleased_body(text):
+        problems.append(
+            "CHANGELOG.md still has content under `## Unreleased` — either include "
+            "it in this release, or do not cut the release yet"
+        )
+
     return problems
 
 
@@ -139,7 +255,11 @@ def main(argv):
     version = argv[2]
 
     if cmd == "notes":
-        with open(CHANGELOG, encoding="utf-8") as fh:
+        # An explicit path lets update.sh show the notes from the CHANGELOG of the
+        # version it is about to install (`git show <tag>:CHANGELOG.md`), not the
+        # one already checked out.
+        path = argv[3] if len(argv) > 3 else CHANGELOG
+        with open(path, encoding="utf-8") as fh:
             print(extract_notes(fh.read(), version))
         return 0
 

uninstall.sh

@@ -3,7 +3,7 @@
 
 set -euo pipefail
 
-VERSION="0.3.5"
+VERSION="0.7.0"
 
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 _HOOK_COMMENT='TrueCloud provider patch (S3/B2)'

update.sh

@@ -0,0 +1,305 @@
+#!/bin/bash
+# update.sh — fetch a newer release of truecloud-patch and apply it.
+#
+# ── RUN THIS BY HAND. NEVER FROM CRON OR A SYSTEMD TIMER. ─────────────────────
+#
+# This patch injects Python into middlewared and re-applies itself at every boot.
+# An unattended pull would let any bad upstream commit reach your box with no
+# human in the loop, and take effect on the next reboot. That is not theoretical:
+# v0.0.4 shipped a boot-time bug that took every app on the box down.
+#
+# The manual step IS the safety gate. Keep it.
+#
+# By default this updates to the newest RELEASE TAG, not to main. main can be
+# mid-refactor; a tag is the tested artifact. Use --main only if you know why.
+#
+#   bash update.sh              # to the newest release, with a confirmation
+#   bash update.sh --check      # show what would happen; change nothing
+#   bash update.sh --rollback   # undo the last update
+
+set -euo pipefail
+
+VERSION="0.7.0"
+
+PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
+_PREV_FILE="$PATCH_DIR/.update_previous"
+
+_target=""
+_use_main=0
+_assume_yes=0
+_check_only=0
+_rollback=0
+
+usage() {
+    cat <<USAGE
+Usage: bash update.sh [options]
+
+Options:
+  --to <ref>    Update to a specific tag or commit (default: newest release tag)
+  --main        Update to origin/main — UNRELEASED code, no guarantees
+  --check       Show what an update would do and exit; changes nothing
+  --rollback    Return to the revision recorded before the last update
+  --yes, -y     Skip the confirmation prompt
+  -h, --help    Show this help
+
+Updating preserves your nested-snapshot opt-in setting either way.
+USAGE
+}
+
+# An UNTRACKED file that the target tracks makes `git checkout` abort. The dirty-
+# tree check deliberately ignores untracked files, so this slips past it and the
+# checkout then dies mid-operation. Not hypothetical: a hand-copied
+# patch/wait_restart.sh blocked a pull on a real box exactly this way.
+#
+# Used by BOTH the update and the rollback path -- rolling back moves the tree too,
+# and would hit the identical failure.
+_abort_if_untracked_blockers() {
+    local ref="$1" blocking
+
+    # Set intersection of {untracked, not ignored} and {tracked by the target}. Two
+    # git calls, not one `ls-files --error-unmatch` per file in the target tree.
+    # --exclude-standard is deliberate: git silently overwrites *ignored* files on
+    # checkout, so those are not blockers — only untracked-and-not-ignored ones are.
+    blocking="$(comm -12 \
+        <(git ls-files --others --exclude-standard | sort) \
+        <(git ls-tree -r --name-only "$ref" | sort) \
+        | sed 's/^/    /')"
+
+    [ -n "$blocking" ] || return 0
+
+    echo "ERROR: these untracked files would be overwritten:" >&2
+    printf '%s\n\n' "$blocking" >&2
+    echo "  They exist here but git does not track them — most likely hand-copied" >&2
+    echo "  or scp'd in. Move or delete them, then re-run." >&2
+
+    # "Delete update.sh, then re-run update.sh" is impossible. If the script itself
+    # is a blocker, it was hand-copied in to bootstrap; the honest answer is to
+    # bootstrap with git instead, which installs it properly.
+    case "$blocking" in
+        *update.sh*)
+            echo "" >&2
+            echo "  update.sh itself is untracked here — you copied it in to bootstrap." >&2
+            echo "  Do that with git instead, once; it installs update.sh properly:" >&2
+            echo "" >&2
+            echo "    rm -f $PATCH_DIR/update.sh" >&2
+            echo "    git -C $PATCH_DIR checkout $ref" >&2
+            echo "    bash $PATCH_DIR/install.sh" >&2
+            echo "" >&2
+            echo "  Every later update is then just: bash update.sh" >&2
+            ;;
+    esac
+    exit 1
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --to)
+            if [ -z "${2:-}" ]; then
+                echo "ERROR: --to needs a tag, branch, or commit." >&2
+                exit 1
+            fi
+            _target="$2"; shift ;;
+        --main)     _use_main=1 ;;
+        --check)    _check_only=1 ;;
+        --rollback) _rollback=1 ;;
+        --yes|-y)   _assume_yes=1 ;;
+        -h|--help)  usage; exit 0 ;;
+        *) echo "ERROR: unknown option: $1" >&2; echo "" >&2; usage >&2; exit 1 ;;
+    esac
+    shift
+done
+
+echo "=== TrueNAS TrueCloud Provider Patch — Update (v${VERSION}) ==="
+echo ""
+
+# ── Preflight ─────────────────────────────────────────────────────────────────
+
+if [ "$(id -u)" -ne 0 ]; then
+    echo "ERROR: must be run as root (install.sh needs it)." >&2
+    exit 1
+fi
+
+cd "$PATCH_DIR"
+
+if ! git rev-parse --git-dir >/dev/null 2>&1; then
+    echo "ERROR: $PATCH_DIR is not a git clone — nothing to update." >&2
+    echo "  Re-clone from https://github.com/sudolulo/truenas-truecloud-patch" >&2
+    exit 1
+fi
+
+# Past `sudo git pull`s can leave root-owned objects in .git that then break any
+# non-root git command. We run as root, so we would only make that worse.
+_owner="$(stat -c '%U' "$PATCH_DIR")"
+if [ -n "$_owner" ] && [ "$_owner" != "root" ]; then
+    chown -R "$_owner" "$PATCH_DIR/.git" 2>/dev/null || true
+fi
+
+# A dirty tree means someone edited or scp'd files in place; merging over that
+# silently loses their changes, or conflicts halfway through.
+#
+# CONTENT changes only. A mode-only change (100644 -> 100755) is not somebody's work
+# and must not block an update -- and it is not hypothetical: install.sh chmod +x's
+# these very scripts, so on any version where git recorded one as 100644, INSTALLING
+# dirtied the checkout and update.sh then refused to run. Install once, and updating
+# was blocked forever, with an error telling the user to `git checkout -- .` (which
+# merely undoes the exec bit so the next install can re-dirty it). A real box sat on
+# an old version for exactly this reason.
+#
+# `git diff --numstat` reports "0  0  file" for a mode-only change, so anything with a
+# nonzero insert or delete count is a genuine edit.
+_dirty=$(git diff --numstat HEAD -- . | awk '$1 != 0 || $2 != 0 { print $3 }')
+if [ -n "$_dirty" ]; then
+    echo "ERROR: the working tree has uncommitted changes:" >&2
+    printf '  M %s\n' $_dirty >&2
+    echo "" >&2
+    echo "  Refusing to update over them. Commit, stash, or discard them first:" >&2
+    echo "    git -C $PATCH_DIR checkout -- ." >&2
+    exit 1
+fi
+
+# ── Rollback ──────────────────────────────────────────────────────────────────
+
+if [ "$_rollback" -eq 1 ]; then
+    if [ ! -f "$_PREV_FILE" ]; then
+        echo "ERROR: no previous revision recorded — nothing to roll back to." >&2
+        exit 1
+    fi
+    _prev="$(cat "$_PREV_FILE")"
+    if ! git rev-parse --verify --quiet "${_prev}^{commit}" >/dev/null; then
+        echo "ERROR: recorded revision '$_prev' is not a valid commit." >&2
+        echo "  The history may have been rewritten. Pick a target explicitly:" >&2
+        echo "    bash update.sh --to <tag>" >&2
+        exit 1
+    fi
+    _abort_if_untracked_blockers "$_prev"
+    echo "Rolling back to $_prev ..."
+    git checkout -q --detach "$_prev"
+    echo "Reverted. Re-applying ..."
+    echo ""
+    bash "$PATCH_DIR/install.sh"
+    exit 0
+fi
+
+# ── Work out where we are and where we are going ──────────────────────────────
+
+echo "Fetching ..."
+git fetch --quiet --tags --prune origin
+
+_current="$(git rev-parse HEAD)"
+_current_desc="$(git describe --tags --always 2>/dev/null || echo "$_current")"
+
+if [ -n "$_target" ]; then
+    :
+elif [ "$_use_main" -eq 1 ]; then
+    _target="origin/main"
+else
+    # Newest release tag by VERSION order, not by tag date. Date order is only
+    # correct while tags are created in ascending version order; it breaks the
+    # moment a hotfix is tagged out of band (a v0.3.6 released after v0.4.0 would
+    # sort as "newest" by date and silently downgrade the box).
+    #
+    # Filter to PLAIN vX.Y.Z: git's version sort ranks `v0.5.0-rc1` ABOVE `v0.5.0`
+    # (verified), so without this a release candidate would be installed as though
+    # it were the newest release. The release workflow deliberately supports
+    # rc/beta/alpha tags, so they will exist.
+    _target="$(git tag -l 'v*' --sort=-version:refname \
+                 | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1)"
+    if [ -z "$_target" ]; then
+        echo "ERROR: no release tags found; use --main to track unreleased code." >&2
+        exit 1
+    fi
+fi
+
+if ! _target_sha="$(git rev-parse --verify --quiet "${_target}^{commit}")"; then
+    echo "ERROR: '$_target' is not a valid tag, branch, or commit." >&2
+    exit 1
+fi
+
+echo "  current: $_current_desc"
+echo "  target:  $_target ($(git rev-parse --short "$_target_sha"))"
+echo ""
+
+if [ "$_current" = "$_target_sha" ]; then
+    echo "Already up to date. Nothing to do."
+    exit 0
+fi
+
+_abort_if_untracked_blockers "$_target_sha"
+
+# ── Show what is coming ───────────────────────────────────────────────────────
+
+echo "Commits you do not have yet:"
+git log --oneline --no-decorate "$_current..$_target_sha" | sed 's/^/  /' || true
+echo ""
+
+# Reuse tools/release_notes.py rather than re-implementing the extractor here —
+# a second copy would be the untested one. Read the CHANGELOG *of the target*, so
+# the notes describe what you are about to install.
+if [ -f "$PATCH_DIR/tools/release_notes.py" ] && [ "$_use_main" -eq 0 ] \
+       && [ -z "${_target##v*}" ]; then
+    _cl="$(mktemp)"
+    if git show "$_target_sha:CHANGELOG.md" > "$_cl" 2>/dev/null && [ -s "$_cl" ]; then
+        echo "Release notes for $_target:"
+        python3 "$PATCH_DIR/tools/release_notes.py" notes "$_target" "$_cl" \
+            2>/dev/null | sed 's/^/  /' || echo "  (no notes for $_target)"
+        echo ""
+    fi
+    rm -f "$_cl"
+fi
+
+if [ "$_use_main" -eq 1 ]; then
+    echo "NOTE: --main tracks UNRELEASED code. It has passed CI, but it is not a"
+    echo "      tested release, and apply.sh runs at every boot."
+    echo ""
+fi
+
+if [ "$_check_only" -eq 1 ]; then
+    echo "--check given; nothing changed."
+    exit 0
+fi
+
+# ── Confirm ───────────────────────────────────────────────────────────────────
+
+if [ "$_assume_yes" -eq 0 ]; then
+    printf "Apply this update and restart middlewared? [y/N] "
+    read -r _answer </dev/tty || _answer=""
+    case "$_answer" in
+        y|Y|yes|YES) ;;
+        *) echo "Aborted. Nothing changed."; exit 0 ;;
+    esac
+    echo ""
+fi
+
+# ── Apply ─────────────────────────────────────────────────────────────────────
+
+# Record where we were BEFORE moving, so --rollback works even if install.sh dies.
+echo "$_current" > "$_PREV_FILE"
+
+echo "Checking out $_target ..."
+git checkout -q --detach "$_target_sha"
+echo "  now at $(git describe --tags --always)"
+echo ""
+
+echo "Applying (this preserves your nested-snapshot setting) ..."
+echo ""
+if ! bash "$PATCH_DIR/install.sh"; then
+    echo ""
+    echo "ERROR: install.sh failed after updating." >&2
+    echo "  Roll back with:  bash $PATCH_DIR/update.sh --rollback" >&2
+    echo "  Or disable the patch entirely:  bash $PATCH_DIR/recover.sh" >&2
+    exit 1
+fi
+
+echo ""
+echo "=== Update complete ==="
+echo "  $_current_desc  ->  $(git describe --tags --always)"
+echo ""
+if ! git symbolic-ref -q HEAD >/dev/null; then
+    echo "NOTE: the checkout is now pinned to a release tag (detached HEAD), which is"
+    echo "      what you want for a deployment. Plain \`git pull\` will not work here —"
+    echo "      use \`bash update.sh\` from now on."
+    echo ""
+fi
+echo "If anything looks wrong:"
+echo "  bash $PATCH_DIR/update.sh --rollback    # back to $_current_desc"
+echo "  bash $PATCH_DIR/recover.sh              # kill switch + restart"

uv.lock

@@ -0,0 +1,3 @@
+version = 1
+revision = 3
+requires-python = ">=3.12"