docs: user-facing URLs point at GitHub, the user-facing repo

Gitea is canonical for development; GitHub is where users clone from and where the
box's read-only checkout points. The install instructions, the re-clone hint and the
'file an issue' link are all read by users, so they name GitHub. docs/releasing.md
still names Gitea, because that is a contributor doc about where the code is pushed.

.github/workflows/ci.yml

@@ -0,0 +1,62 @@
+name: CI
+
+on:
+  push:
+    branches: [main, "feat/**", "fix/**"]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  shell:
+    name: shell (shellcheck + syntax)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: bash syntax check
+        run: |
+          fail=0
+          while IFS= read -r f; do
+            bash -n "$f" || { echo "::error file=$f::bash syntax error"; fail=1; }
+          done < <(find . -name '*.sh' -not -path './.git/*')
+          exit $fail
+
+      # Pinned to a release tag, not @master: a third-party action on a moving
+      # branch runs whatever that branch contains at the time CI fires.
+      - name: shellcheck
+        uses: ludeeus/action-shellcheck@2.0.0
+        env:
+          SHELLCHECK_OPTS: -S warning -e SC1091
+
+  python:
+    name: python ${{ matrix.python }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # TrueNAS SCALE middleware runs 3.11+; keep the patch importable across
+        # the versions it may be injected into.
+        python: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: install dev deps
+        run: python -m pip install --upgrade pip pytest ruff
+
+      - name: ruff
+        run: ruff check patch tests tools
+
+      - name: pytest
+        run: pytest tests -v
+
+      - name: verify injected middleware blocks compile
+        # Belt-and-braces: the *_BLOCK strings are appended into live middlewared
+        # modules. A syntax error there would break the box at boot.
+        run: pytest tests/test_apply_blocks.py -v

.github/workflows/compat.yml

@@ -0,0 +1,228 @@
+name: TrueNAS compatibility
+
+# Find out that iX broke us BEFORE their release ships, not after a user's backup
+# fails.
+#
+# This patch appends code to middlewared's internal modules. There is no stability
+# contract: TrueNAS 26 rewrote the whole cloud_backup path from async to sync, and
+# every block the nested module injects is an `async def` wrapping an `await`ed
+# original. Nobody would have found out until a restore did not work.
+#
+# tools/compat.py records what the patch assumes and checks it against iX's actual
+# source at every release line -- including master and the current BETA/RC, which is
+# where a break shows up first. When an UNRELEASED line breaks, this opens a bug
+# report so there is time to fix it before that version reaches anyone.
+#
+# Runs on both forges: Gitea (canonical) and GitHub (mirror). Only the "file an
+# issue" call differs.
+
+on:
+  schedule:
+    - cron: "17 6 * * *"     # daily, off the hour: everyone crons on the hour
+  workflow_dispatch:
+  push:
+    paths:
+      # The manifest itself changed -- re-check immediately rather than waiting a day.
+      - "tools/compat.py"
+      - ".github/workflows/compat.yml"
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  compat:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      # ONE pass over the network. Every other step renders from this JSON -- calling
+      # compat.py three times would re-fetch every file from every release line three
+      # times, and could even disagree with itself if iX pushed mid-run.
+      #
+      # The exit code is CAPTURED, not allowed to abort the step: a nonzero exit means
+      # "a shipped release is broken", which is a result to report, not a reason to
+      # die before reporting it. (Actions runs `bash -e`, so `cmd > out` followed by
+      # `echo $?` never reaches the echo.) It becomes a job failure at the end, after
+      # the bug report has been filed.
+      - name: check every TrueNAS release line
+        id: check
+        run: |
+          rc=0
+          python3 tools/compat.py --matrix --json > /tmp/matrix.json || rc=$?
+          echo "shipped_broken=$rc" >> "$GITHUB_OUTPUT"
+
+      # The report body is written to a FILE, and never becomes a step output.
+      #
+      # An earlier version did `echo "${{ steps.report.outputs.body }}"`, which
+      # splices the text into the shell script itself -- and the report is full of
+      # backticks, so bash ran `create-snapshot`, `def` and `async` as commands. It
+      # is also an injection vector: the report is built from iX's source, so
+      # anything that lands in middleware would execute on the runner.
+      #
+      # The rule that avoids the whole class: never interpolate ${{ }} into a `run:`
+      # body. Files for data, `env:` for scalars (the runner sets those, rather than
+      # pasting them into the script).
+      - name: build the report
+        id: report
+        run: |
+          python3 - <<'PY' >> "$GITHUB_OUTPUT"
+          import json, sys
+
+          sys.path.insert(0, "tools")
+          import compat
+
+          with open("/tmp/matrix.json") as fh:
+              rows = json.load(fh)
+
+          with open("/tmp/matrix.md", "w") as fh:
+              fh.write(compat.render_markdown(rows))
+
+          broken = [
+              r for r in rows
+              if any(compat.is_broken(m) for m in r["modules"].values())
+          ]
+          native = [
+              (r["ref"], mod)
+              for r in rows
+              for mod, m in sorted(r["modules"].items())
+              if m["native"] and not compat.is_broken(m)
+          ]
+
+          lines = [
+              "`tools/compat.py` found that the patch's assumptions about "
+              "middlewared no longer hold.",
+              "",
+              compat.render_markdown(rows),
+              "",
+          ]
+          for r in broken:
+              lines.append(f"### {r['ref']}")
+              lines.append("")
+              for mod, m in sorted(r["modules"].items()):
+                  if not compat.is_broken(m):
+                      continue
+                  lines.append(f"**{mod}** — the patch will not apply:")
+                  lines.append("")
+                  for p in m["problems"]:
+                      lines.append(f"- `{p['id']}`: {p['detail']}")
+                      lines.append(f"  - why it matters: {p['why']}")
+                  lines.append("")
+          for ref, mod in native:
+              lines.append(
+                  f"- `{ref}`: **{mod}** appears to be NATIVE now — retire the "
+                  f"module rather than fixing it."
+              )
+          lines += ["", "_Filed automatically by `.github/workflows/compat.yml`._"]
+
+          with open("/tmp/issue.md", "w") as fh:
+              fh.write("\n".join(lines))
+
+          # Scalars only. The body stays in the file.
+          print(f"broken={'1' if broken else '0'}")
+          print(f"refs={','.join(r['ref'] for r in broken)}")
+          PY
+
+      - name: matrix
+        run: cat /tmp/matrix.md
+
+      # Keep the README's table true. A support matrix that quietly goes stale is not
+      # a stale doc -- it is a false promise to somebody deciding whether to trust
+      # this with their backups.
+      #
+      # Only ever touches the block between the COMPAT MATRIX markers, and only on
+      # the canonical host (Gitea) so the two forges cannot race each other. The
+      # `paths:` trigger above does not include README.md, so this cannot re-trigger
+      # itself; and a README change is documentation-only, which by design raises no
+      # update alert on anyone's box.
+      - name: refresh the README matrix
+        if: ${{ github.event_name == 'schedule' && !contains(github.server_url, 'github.com') }}
+        run: |
+          python3 - <<'PY'
+          import json, sys
+          sys.path.insert(0, "tools")
+          import compat
+          with open("/tmp/matrix.json") as fh:
+              rows = json.load(fh)
+          print("changed" if compat.update_readme(rows) else "unchanged")
+          PY
+
+          if ! git diff --quiet -- README.md; then
+            git config user.name  "truecloud-patch bot"
+            git config user.email "bot@onetick.ninja"
+            git add README.md
+            git commit -m "docs: refresh the TrueNAS compatibility matrix"
+            git push origin HEAD:main
+          fi
+
+      # A broken SHIPPED release is an outage: users are on it right now.
+      - name: fail if a shipped release is broken
+        if: ${{ steps.check.outputs.shipped_broken != '0' }}
+        run: |
+          echo "::error::The patch is broken on a SHIPPED TrueNAS release."
+          exit 1
+
+      - name: file a bug report (GitHub)
+        if: ${{ steps.report.outputs.broken == '1' && contains(github.server_url, 'github.com') }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TITLE: "Incompatible with upcoming TrueNAS: ${{ steps.report.outputs.refs }}"
+        run: |
+          # One issue per set of broken refs, reopened/updated rather than duplicated
+          # daily -- a bot that files the same issue every morning gets muted, and
+          # then it is not a warning system any more.
+          existing="$(gh issue list --state all --search "$TITLE" \
+                        --json number,title \
+                        --jq '.[] | select(.title == env.TITLE) | .number' | head -1)"
+
+          if [ -n "$existing" ]; then
+            gh issue comment "$existing" --body-file /tmp/issue.md
+            gh issue reopen "$existing" 2>/dev/null || true
+          else
+            gh issue create --title "$TITLE" --body-file /tmp/issue.md
+          fi
+
+      - name: file a bug report (Gitea)
+        if: ${{ steps.report.outputs.broken == '1' && !contains(github.server_url, 'github.com') }}
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+          TITLE: "Incompatible with upcoming TrueNAS: ${{ steps.report.outputs.refs }}"
+        run: |
+          # python3, not jq: jq is not guaranteed on a self-hosted runner, and a bug
+          # report that dies on a missing tool is a warning system that does not warn.
+          python3 - <<'PY'
+          import json, os, urllib.error, urllib.request
+
+          api, token, title = os.environ["API"], os.environ["TOKEN"], os.environ["TITLE"]
+          with open("/tmp/issue.md", encoding="utf-8") as fh:
+              body = fh.read()
+          headers = {"Authorization": f"token {token}",
+                     "Content-Type": "application/json"}
+
+          def call(url, method, data=None):
+              req = urllib.request.Request(
+                  url, method=method, headers=headers,
+                  data=json.dumps(data).encode() if data else None)
+              with urllib.request.urlopen(req) as r:  # noqa: S310
+                  return json.load(r) if r.length != 0 else {}
+
+          # Same title => same issue. Comment on it rather than filing a new one every
+          # morning: a bot that duplicates itself daily gets muted, and then it is not
+          # a warning system any more.
+          issues = call(f"{api}/issues?state=all&type=issues", "GET")
+          match = next((i for i in issues if i["title"] == title), None)
+
+          if match:
+              n = match["number"]
+              call(f"{api}/issues/{n}/comments", "POST", {"body": body})
+              call(f"{api}/issues/{n}", "PATCH", {"state": "open"})
+              print(f"commented on and reopened issue #{n}")
+          else:
+              made = call(f"{api}/issues", "POST", {"title": title, "body": body})
+              print(f"filed issue #{made['number']}")
+          PY

.github/workflows/release.yml

@@ -0,0 +1,208 @@
+name: Release
+
+# Push a tag, get a release. The body always comes from CHANGELOG.md, so there is
+# no second place to write release notes and therefore no second place for them to
+# go stale.
+#
+#   git tag -a v0.4.0 -m "v0.4.0" && git push origin v0.4.0
+#
+# workflow_dispatch re-cuts (or updates) the release for a tag that already
+# exists, since re-pushing an existing tag triggers nothing.
+#
+# It checks out the TAG, because the tagged code is what people install and it has
+# to pass its own tests. That means it only works for tags that actually contain
+# this tooling (>= v0.3.0). Tags older than that were backfilled by hand.
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Existing tag to create a release for (e.g. v0.2.1)"
+        required: true
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      # Via `env:`, never spliced into the script. `inputs.tag` is attacker-chosen on
+      # a workflow_dispatch, and a ${{ }} in a `run:` body is pasted into the shell
+      # TEXT -- a tag of `$(...)` would simply execute. env: is safe: the runner sets
+      # the variable instead of rewriting the script.
+      - name: Resolve tag
+        id: tag
+        env:
+          EVENT: ${{ github.event_name }}
+          INPUT_TAG: ${{ inputs.tag }}
+        run: |
+          if [ "$EVENT" = "workflow_dispatch" ]; then
+            tag="$INPUT_TAG"
+          else
+            tag="${GITHUB_REF#refs/tags/}"
+          fi
+
+          # Whatever it came from, it has to look like a tag we cut.
+          case "$tag" in
+            v[0-9]*.[0-9]*.[0-9]*) ;;
+            *) echo "::error::refusing to release a tag that is not vX.Y.Z[-rcN]: $tag"; exit 1 ;;
+          esac
+
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.tag.outputs.tag }}
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      # Never publish a release for code that does not pass its own tests. A
+      # tagged commit is what people install; it has to be at least as good as
+      # main.
+      - name: install dev deps
+        run: python -m pip install --upgrade pip pytest ruff
+
+      - name: ruff
+        run: ruff check patch tests tools
+
+      - name: pytest
+        run: pytest tests -q
+
+      - name: shell syntax
+        run: |
+          fail=0
+          while IFS= read -r f; do
+            bash -n "$f" || { echo "::error file=$f::bash syntax error"; fail=1; }
+          done < <(find . -name '*.sh' -not -path './.git/*')
+          exit $fail
+
+      # Catches the failure mode this repo actually had: VERSION= drifted to
+      # three different values across the scripts, and nothing noticed.
+      - name: "gate: version matches tag, CHANGELOG complete, nothing stranded"
+        env:
+          TAG: ${{ steps.tag.outputs.tag }}
+        run: python3 tools/release_notes.py check "$TAG"
+
+      # THE BARRIER. A stable release must have been a release candidate on this
+      # exact commit. Candidates are invisible to users (update.sh and the alert
+      # source both take the newest plain vX.Y.Z), so debugging happens across
+      # rc1/rc2/rc3 at no cost to anyone -- instead of across v0.5.0/v0.5.1/v0.5.2,
+      # which alerts every installed box every time.
+      #
+      # Same code release.sh runs locally, so this should never be the first place
+      # you find out. It is here because this is the only place that cannot be
+      # bypassed: it holds the token that publishes.
+      - name: "gate: this commit was a release candidate"
+        env:
+          TAG: ${{ steps.tag.outputs.tag }}
+        run: python3 tools/release_gate.py "$TAG" -C .
+
+      # There is deliberately NO "did the candidate's CI run pass?" gate here.
+      #
+      # It would have to query the forge's run history, which is the one thing that
+      # differs between GitHub and Gitea -- and it adds nothing: the steps above
+      # re-run ruff, pytest and the shell checks against the TAGGED COMMIT, and
+      # release_gate.py has already proved a candidate points at that same commit.
+      # If the code passes now, it passed then; they are the same code.
+      #
+      # What a candidate really buys is the thing no CI can check: that a human
+      # installed it on a real box and exercised it. The barrier makes room for
+      # that; it cannot verify it.
+
+      - name: extract release notes from CHANGELOG
+        env:
+          TAG: ${{ steps.tag.outputs.tag }}
+        run: |
+          python3 tools/release_notes.py notes "$TAG" > /tmp/notes.md
+          echo "--- release body ---"
+          cat /tmp/notes.md
+
+      # This repo is canonically hosted on Gitea (git.onetick.ninja) and mirrored to
+      # GitHub, and BOTH run this workflow -- Gitea reads .github/workflows too. So
+      # the publish step has to work on whichever forge it lands on. Everything
+      # above is forge-agnostic; only the "create a release" API differs.
+
+      - name: publish the release (GitHub)
+        if: ${{ contains(github.server_url, 'github.com') }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ steps.tag.outputs.tag }}
+        run: |
+          # Lowercased: release_gate/release_notes match the suffix case-INsensitively
+          # (`is_prerelease` uses re.I), so a `v0.6.0-RC1` skipped the barrier as a
+          # candidate and then landed here as a case-sensitive MISS -- published as the
+          # forge's "Latest release" on a commit that was never a candidate.
+          prerelease=""
+          case "$(printf '%s' "$TAG" | tr '[:upper:]' '[:lower:]')" in
+            *-rc*|*-beta*|*-alpha*) prerelease="--prerelease" ;;
+          esac
+
+          if gh release view "$TAG" >/dev/null 2>&1; then
+            echo "Release $TAG exists — updating notes."
+            gh release edit "$TAG" --notes-file /tmp/notes.md
+          else
+            # shellcheck disable=SC2086
+            gh release create "$TAG" --title "$TAG" --notes-file /tmp/notes.md $prerelease
+          fi
+
+      - name: publish the release (Gitea)
+        if: ${{ !contains(github.server_url, 'github.com') }}
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
+          TAG: ${{ steps.tag.outputs.tag }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+        run: |
+          prerelease=false
+          case "$(printf '%s' "$TAG" | tr '[:upper:]' '[:lower:]')" in
+            *-rc*|*-beta*|*-alpha*) prerelease=true ;;
+          esac
+
+          # python3, not jq. The changelog is full of quotes, backticks and newlines,
+          # so the body must be properly JSON-encoded -- but `jq` is not guaranteed on
+          # a self-hosted Gitea runner, and a publish step that dies on a missing tool
+          # leaves a tag with no release behind it. python3 is guaranteed: setup-python
+          # ran above.
+          python3 - "$TAG" "$API" "$TOKEN" "$prerelease" <<'PY'
+          import json, sys, urllib.error, urllib.request
+
+          tag, api, token, prerelease = sys.argv[1:5]
+          with open("/tmp/notes.md", encoding="utf-8") as fh:
+              body = fh.read()
+
+          payload = {
+              "tag_name": tag, "name": tag, "body": body,
+              "prerelease": prerelease == "true",
+          }
+          headers = {
+              "Authorization": f"token {token}",
+              "Content-Type": "application/json",
+          }
+
+          def call(url, method, data=None):
+              req = urllib.request.Request(
+                  url, method=method, headers=headers,
+                  data=json.dumps(data).encode() if data else None)
+              with urllib.request.urlopen(req) as r:  # noqa: S310
+                  return r.status, json.load(r) if r.length != 0 else {}
+
+          try:
+              _, existing = call(f"{api}/releases/tags/{tag}", "GET")
+          except urllib.error.HTTPError as e:
+              if e.code != 404:
+                  raise
+              existing = None
+
+          if existing:
+              status, _ = call(f"{api}/releases/{existing['id']}", "PATCH", payload)
+              print(f"updated release {tag} -> {status}")
+          else:
+              status, _ = call(f"{api}/releases", "POST", payload)
+              print(f"created release {tag} (prerelease={payload['prerelease']}) -> {status}")
+          PY

.gitignore

@@ -4,3 +4,16 @@
 /apply.log.2
 /hook_status.json
 /disabled
+/nested_snapshots_enabled
+# Written by apply.sh when a module's assumptions no longer fit the installed
+# middlewared (see tools/compat.py). Evidence for the alert; not source.
+/incompatible.json
+
+# Python
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+.venv/
+venv/
+/update_alerts_disabled

CHANGELOG.md

@@ -1,5 +1,735 @@
 # Changelog
 
+Work lands under **Unreleased** and stays there until a release promotes it. That
+is deliberate: see [Releasing](docs/releasing.md). Twelve releases were cut on
+2026-07-13, several of them fixing the release before — and with the update alert
+live, every one of those interrupts every user. An alert people learn to ignore is
+worse than no alert, because one day it carries a security fix.
+
+## v0.6.0 — 2026-07-13
+### Added
+
+- **`release.sh` — a two-stage release process, and a barrier that enforces it.**
+  A stable `vX.Y.Z` tag is now only publishable if a `vX.Y.Z-rcN` tag points at the
+  **same commit**, and the release job re-runs the entire suite against that tagged
+  commit before publishing. Candidates are invisible to users — `update.sh` and the
+  update alert both take the newest plain `vX.Y.Z` tag — so debugging happens across
+  rc1, rc2, rc3 at nobody's expense, instead of across v0.5.0, v0.5.1, v0.5.2 at
+  everybody's.
+
+      bash release.sh 0.6.0 --rc        # candidate. Invisible to users.
+      bash release.sh 0.6.0 --promote   # stable. Refused unless an rc passed HERE.
+
+  The rule is enforced in `tools/release_gate.py`, which `release.sh` runs locally
+  (so you fail in 200 ms) and `.github/workflows/release.yml` runs again where it
+  cannot be bypassed (so failing locally is not optional). "The candidate passed,
+  then I pushed one more little fix" is refused by name — that is precisely how
+  v0.5.1 happened.
+
+- **TrueNAS compatibility is now checked, not hoped for.**
+  [`tools/compat.py`](tools/compat.py) is a written-down record of everything each
+  module assumes about middlewared, checked in two places:
+
+  - **CI, daily** — against iXsystems' source at every release line *including
+    `master` and the current BETA/RC*. When an unreleased TrueNAS breaks the patch
+    it files a bug report automatically, so there is time to fix it before that
+    version reaches anyone. It also refreshes the README's support matrix, so the
+    table cannot quietly become a false promise.
+  - **`apply.sh`, at every boot** — against the middleware actually installed on the
+    box. **A module whose assumptions no longer hold is not applied.** Stock TrueNAS
+    without a feature beats TrueNAS with a broken backup.
+
+  It immediately found two real breaks: TrueNAS 26 (below), and a nested-snapshot bug
+  that had been shipping for two releases (below).
+
+- **The compatibility check now covers the middlewared methods the patch _calls_,**
+  not only the symbols it wraps — and that gap was hiding a catastrophe.
+
+  TrueNAS 26 **deletes `plugins/zfs_/dataset.py` and `plugins/zfs_/snapshot.py`
+  outright**, taking `zfs.dataset.query`, `zfs.snapshot.query` and
+  `zfs.snapshot.delete` with them (26 uses `filesystem.statfs` and `zfs.resource.*`).
+  Nothing about the five `cloud_backup` files reveals that, so every other check went
+  green. The patch would have applied perfectly and then **failed on the first
+  backup** — or, far worse, snapshotted successfully and failed to *delete*,
+  orphaning one snapshot per descendant dataset (**250 on a real pool**) on every
+  single run, forever.
+
+  This is now an assumption class of its own, so a method disappearing is a BROKEN
+  verdict rather than a silent time bomb.
+
+- **Groundwork for TrueNAS 26** (async→sync and the deleted helper — see below).
+  **26 is still reported BROKEN and the nested module will not apply there**, because
+  the ZFS API rewrite above is not yet ported. Porting it needs a real 26 box to
+  verify against, and shipping a port nobody has run is exactly the failure this
+  project exists to avoid. On 26, TrueNAS is left stock: B2/S3 keeps working, nested
+  datasets are simply not covered.
+
+### Fixed
+
+- **Nested snapshots were broken on TrueNAS 24.10 and 25.04, and had been all
+  along.** `SYNC_BLOCK`'s wrapper spelled out the stock signature and forwarded five
+  arguments — but those releases declare `restic_backup(middleware, job,
+  cloud_backup, dry_run)`; `rate_limit` only arrived in 25.10. Every nested backup on
+  24.10/25.04 raised `TypeError: restic_backup() takes 4 positional arguments but 5
+  were given`. The wrapper now takes `*args, **kwargs` and forwards whatever it is
+  handed, so a trailing parameter appearing or disappearing is a non-event.
+
+  Found by the new compatibility check, not by a user — which is the whole argument
+  for having it. The check it replaced only asked whether the parameter *names* still
+  appeared somewhere in the signature, so it happily passed a call that could never
+  work.
+
+- **The nested module is now one synchronous implementation behind two thin
+  wrappers.** TrueNAS 26 rewrites `cloud_backup` from async to **synchronous** and
+  separately **deletes `get_dataset_recursive()`**, which an injected block called out
+  of the host module's namespace. Either alone is a broken backup found at restore
+  time: an `async def` wrapper hands `sync.py` a coroutine where it unpacks a tuple,
+  and the vanished helper is a straight `NameError`.
+
+  The module now talks to middlewared through `call_sync`, and `apply.sh` reads which
+  flavour the installed middleware declares and injects the matching wrapper —
+  TrueNAS ≤ 25.10 reaches it via `await middleware.run_in_thread(...)`; a synchronous
+  TrueNAS, already in a worker thread, calls it directly. The logic that owns the
+  snapshots, the bind mounts and the failure modes exists **once**; an async twin
+  would mean every future fix had to land twice, and the one that got missed would be
+  the one that eats a backup. A middleware whose three wrapped functions **disagree**
+  about async-ness is refused outright rather than guessed at, and
+  `get_dataset_recursive` is carried as our own copy — removing the dependency on both
+  versions instead of asserting it.
+
+- **The patch no longer reaches into CloudSync tasks it has no business touching.**
+  `create_snapshot` is module-global in `plugins/cloud/snapshot.py` and is imported by
+  **`cloud_sync.py` as well as `cloud_backup/sync.py`** — so the wrapper sat in the
+  path of every rclone/Storj **CloudSync** task with `snapshot=true`, and issued a
+  `zfs.dataset.query` before deciding it had nothing to do. That added a brand-new
+  failure mode to jobs that worked fine before this patch was installed, and worse: a
+  CloudSync task that ever *did* get staged would **never be torn down**, because the
+  teardown is wired into `cloud_backup`'s `restic_backup` and `CRUD_BLOCK`
+  deliberately leaves CloudSync's nesting guard intact — the bind mounts would pin the
+  ZFS snapshot forever. The staging path now bails out immediately unless the snapshot
+  is named `cloud_backup-*`, before any middleware call.
+
+- **Teardown warnings are no longer silently swallowed on TrueNAS ≤ 25.10.** The async
+  wrapper's `finally` dropped the `logger=` kwarg that the sync one passes, so a
+  cleanup that failed to unmount a bind mount *or* to delete a snapshot tree logged
+  **nothing at all** — on the only platform anyone actually runs. `run_in_thread`
+  forwards `**kwargs` via `functools.partial`; it was a regression, not a limitation.
+
+- **`do_delete` is recognised as `delete`.** TrueNAS 24.10 and 25.04 declare
+  `do_delete` (the `CRUDService` convention); 25.10 renamed it to `delete`. Both
+  answer to `zfs.snapshot.delete`. Accepting only the literal name reported both older
+  releases as BROKEN — a false verdict that would have switched nested snapshots off
+  on boxes where they work perfectly.
+
+- **An incompatible TrueNAS no longer sets the permanent kill switch.** `apply.sh`
+  reused a "nothing left to do" exit that touches `disabled`, which suppresses
+  patching on every future boot and is cleared only by `install.sh` — never by
+  `update.sh`. On TrueNAS 26 (providers-compatible, nested opt-out by default) that
+  branch would have fired, and the very release that fixed 26 could not have
+  re-enabled itself: the user would run `bash update.sh`, exactly as the update alert
+  tells them to, and the patch would stay dead with their B2 backups off.
+  Incompatibility now means "apply nothing this boot, try again next boot".
+  Retirement and incompatibility are opposite situations and no longer share an exit.
+
+- **The compatibility check itself could be fooled**, in ways that each had teeth: a
+  reordered, keyword-only, or newly-required parameter now reads as broken (the patch
+  calls these positionally); a **re-exported or conditionally-defined** symbol reads
+  as *unknown* rather than broken, so an innocent upstream refactor cannot make a
+  working module decline to apply; an **unreadable** source (rate limit, DNS, timeout)
+  is *unknown* rather than "iXsystems deleted this file", so a network blip cannot
+  file a bug report, fail CI, and repaint the published support matrix; and `native`
+  no longer masks `BROKEN`, which used to render a TrueNAS that both reworded the
+  nesting guard *and* reshaped the functions as good news.
+
+- **`compat.py --tree` no longer reads the patch's own code as native support.**
+  `B2_BLOCK` writes `B2RcloneRemote.restic = True` into `b2.py` — exactly the string
+  the providers native-probe looks for — so the one command the docs recommend for
+  checking a live box said "retire the providers module" on every *patched* machine.
+  It now reads only the part of the file iXsystems wrote.
+
+- **`release.sh --promote` could never succeed.** It refused to run if the stable tag
+  existed, and the gate refused if it did not — mutually exclusive, so the only way to
+  cut a stable release was to hand-tag and bypass every gate this work exists to
+  enforce. The gate now resolves the tag's commit if it exists and `HEAD` otherwise.
+  The tests hid it by always tagging first.
+
+### Changed
+
+- **The minimum supported TrueNAS is stated, and enforced: 24.10.** TrueCloud Backup
+  does not exist before it — `plugins/cloud_backup/` is simply absent — so the patch
+  had nothing to attach to and would have done nothing at all, silently, while the
+  user believed their backups were configured. `install.sh` now reads
+  `system.version` and refuses, naming the reason. A version it cannot *parse* is a
+  warning, not a refusal: declining to install over a string we failed to read would
+  be a worse failure than the one being prevented.
+
+- **A stable release may not leave work stranded under `## Unreleased`.** Either it
+  is finished and belongs in the release, or the release is premature. Candidates
+  are exempt: an rc may legitimately have work queued behind it.
+
+- **`release.sh` refuses to run on an installed box.** The whole repo is cloned onto
+  every box, so this file is there too; `update.sh` pins the checkout to a tag in
+  detached HEAD, and `release.sh` now recognises that and says so, rather than
+  emitting a confusing branch error.
+
+- **Gitea (`git.onetick.ninja/flan/truenas-truecloud-patch`) is now canonical**, with
+  GitHub as a mirror. Both forges run the same workflows and publish the same
+  releases. The update alert now **derives the changelog URL from the `origin`
+  remote** instead of hard-coding GitHub — which matters more than it sounds: when
+  the changelog cannot be read, the alert deliberately fires *anyway* rather than risk
+  hiding a security fix, so a stale URL would not have disabled the alert, it would
+  have made it nag on every release, including documentation-only ones.
+
+### Security
+
+- **Workflow expressions are no longer interpolated into shell.**
+  `echo "${{ steps.report.outputs.body }}"` pasted the compatibility report into the
+  script text, and the report is full of backticks — bash ran `create-snapshot`,
+  `def` and `async` as commands. Since that report is built from iXsystems' source,
+  anything landing in their tree would have executed on the runner. `inputs.tag` on
+  `workflow_dispatch` had the same shape, and that one is attacker-chosen. Data now
+  moves through files and scalars through `env:`; a test enforces it across every
+  workflow.
+
+### Internal
+
+- Static-analysis annotations in `patch/alert_source.py` (`# noqa` placement). No
+  runtime change.
+
+## v0.5.1 — 2026-07-13
+
+### Fixed
+
+- **The update alert could have broken middlewared at startup.** middlewared's
+  `alert.load()` imports every file in `alert/source/` with **no try/except**, and
+  it runs during setup — so a module that raises on import takes middlewared down
+  with it. `apply.sh` now **compiles the substituted alert source and refuses to
+  write it** if it does not parse. An uninstalled alert is a missing convenience;
+  a broken one is a broken box.
+
+- **`@PATCH_DIR@` is substituted with `repr()`**, so a repository path containing
+  a quote or a backslash produces a valid Python literal instead of a syntax error
+  in the installed module.
+
+- **The alert source no longer mutates `sys.path`.** It loaded
+  `tools/release_notes.py` via `sys.path.insert(0, …)`, which shadows the stdlib
+  for that interpreter — and `ThreadedAlertSource` runs in middlewared's thread
+  pool, so mutating `sys.path` is a race. It now loads the module by file path with
+  `importlib`.
+
+### Notes
+
+Timing, for the record: `process_alerts` is `@periodic(60)` and
+`alert_source_last_run` is in-memory, so the check runs **within 60 seconds of any
+middlewared restart** (which this patch performs at every boot) and otherwise
+**within 24 hours** of a release.
+
+## v0.5.0 — 2026-07-13
+
+### Added
+
+- **A TrueNAS alert when an update is available** — the bell in the UI, not a log
+  line nobody reads. On by default, checked once a day.
+  `install.sh --no-update-alerts` turns it off.
+
+  **It does not nag.** A release whose CHANGELOG contains only a `### Docs`
+  section changed no code and raises nothing. Anything else raises INFO; a
+  `### Security` section raises WARNING. The CHANGELOG's own section headings are
+  the signal, and a security fix anywhere in the range escalates the whole span —
+  so a docs-only release sitting on top of a security fix still reports as
+  security, rather than hiding it.
+
+  **Why an AlertSource and not `midclt`:** TrueNAS cannot raise an alert from the
+  CLI. `midclt` exposes only `alert.dismiss`, `alert.list`, `alert.list_categories`,
+  `alert.list_policies` and `alert.restore` — alert *creation* is internal to
+  middlewared, and none of its ~60 one-shot classes is generic enough to reuse. So
+  registering an `AlertSource` is the only way, and it is also the least invasive
+  thing this patch does: it **adds one file and modifies none**, where the
+  providers and nested modules both append code to stock middleware files. It is
+  the native mechanism, and TrueNAS polls it itself — no cron, no systemd timer.
+
+  - Fail-safe: every error path returns `None`; it cannot take middlewared down.
+  - Read-only: `git ls-remote` plus an HTTPS fetch of the CHANGELOG. It never
+    writes to `.git`, so it cannot leave root-owned objects behind the way a
+    `git fetch` from middlewared (running as root) would.
+  - Removed by `uninstall.sh`.
+  - It only *tells* you; it never updates anything.
+
+## v0.4.2 — 2026-07-13
+
+### Docs
+
+- **The Updating section never said how to *get* `update.sh`.** It ships inside the
+  patch, so a clone older than v0.4.0 doesn't have it — the docs told you to run a
+  script you didn't have. There is now an explicit bootstrap step (`git pull &&
+  bash install.sh`, once), including the fix for the *"insufficient permission for
+  adding an object to repository database"* failure that past `sudo git pull`s
+  cause.
+
+- **`After a TrueNAS update` rewritten.** It didn't explain that the patch
+  re-applies itself at every boot (so you never reinstall), and it didn't say what
+  each failure actually costs you. "Fail-safe" means *the box stays up* — not that
+  your backups keep running. A `[FAIL] providers` is a **broken backup**, and the
+  docs now say so rather than implying everything degrades gracefully.
+
+- Added a repo map. `patch/mw_patch.py` and `tools/release_notes.py` were
+  documented nowhere.
+
+- `Development` told you to run `ruff check patch tests`, which misses `tools/`.
+
+- Every command and file path in the README is now verified to exist and run.
+
+## v0.4.1 — 2026-07-13
+
+### Fixed
+
+- **`update.sh` would have picked a release candidate as "the newest release".**
+  Git's version sort ranks `v0.5.0-rc1` *above* `v0.5.0` (verified), and the
+  release workflow deliberately supports rc/beta tags — so an RC would have been
+  installed as though it were the latest stable. Tag selection is now filtered to
+  plain `vX.Y.Z`.
+
+- **`update.sh` would have died mid-update on an untracked file.** The dirty-tree
+  guard uses `--untracked-files=no`, so an untracked file that the *target* tracks
+  slipped past it — and `git checkout` then aborts. Under `set -e` the script died
+  with a raw git error, *after* recording the rollback point. This is exactly what
+  blocked a pull on a real box (a hand-copied `patch/wait_restart.sh`). It now
+  detects the collision up front and names the files. Gitignored files are
+  correctly *not* treated as blockers — git overwrites those silently.
+
+  Special case: if `update.sh` *itself* is the blocker, you hand-copied it in to
+  bootstrap — and "delete `update.sh`, then re-run `update.sh`" is impossible. It
+  now says so and prints the git commands that bootstrap it properly.
+
+- **`--rollback` skipped that check entirely**, so it would have hit the identical
+  failure. The check is now a shared function used by both paths, and rollback also
+  validates that the recorded revision still exists (history can be rewritten).
+
+- `install.sh`'s `chmod` aborted under `set -e` if any listed file was missing. The
+  file set changes between versions, so `update.sh --rollback` to an older revision
+  must not be killed by a filename this version happens to know about.
+
+- `--to` with no value was silently ignored and fell back to the default target.
+
+## v0.4.0 — 2026-07-13
+
+### Added
+
+- **`update.sh`** — fetch a newer release and apply it, preserving your
+  nested-snapshot opt-in setting.
+
+  ```bash
+  bash update.sh              # to the newest release, with a confirmation
+  bash update.sh --check      # show what would happen; change nothing
+  bash update.sh --rollback   # undo the last update
+  ```
+
+  **Run it by hand. Never from cron or a systemd timer.** This patch injects
+  Python into middlewared and re-applies itself at every boot, so an unattended
+  pull would let any bad upstream commit reach your box with no human in the loop
+  and take effect on the next reboot. v0.0.4 shipped exactly such a bug and took
+  every app on the box down. The manual step *is* the safety gate.
+
+  Design:
+
+  - **Defaults to the newest release tag, not `main`.** `main` can be mid-refactor;
+    a tag is the tested artifact. `--main` exists but says so loudly.
+  - Tags are ordered by **version**, not by date — date order silently downgrades
+    the box the first time a hotfix is tagged out of band (a v0.3.6 released after
+    v0.4.0 would sort as "newest").
+  - **Refuses to run over a dirty working tree** rather than merging across
+    hand-edited or scp'd files.
+  - Shows the commits you don't have and the target's release notes (read from the
+    *target's* CHANGELOG, via `tools/release_notes.py` — not a second copy of the
+    extractor), then asks before doing anything.
+  - **Records the previous revision before moving**, so `--rollback` works even if
+    `install.sh` dies halfway.
+  - Repairs `.git` ownership, which past `sudo git pull`s leave root-owned and
+    which then breaks every later non-root git command.
+
+- `update.sh` is covered by the version-drift check, so it cannot quietly go stale
+  the way `create_task.py.__version__` did.
+
+## v0.3.5 — 2026-07-13
+
+### Changed
+
+- `delete_snapshot_tree` swallowed the error from its recursive-delete fast path.
+  That failure is *usually* just "parent already gone" — stock's `finally` winning
+  the race once our mounts are released, which the by-name sweep then handles. But
+  if the cause were anything else, this was the only place it was visible, and it
+  went straight to `/dev/null`. It is now logged before falling through.
+
+- Annotated the two remaining static-analysis findings as considered-and-accepted
+  rather than leaving them to be re-litigated: `subprocess` is always called in
+  list form (no shell, so ZFS dataset names cannot inject), and the partial
+  `systemctl` path is moot in a script that only runs as root.
+
+## v0.3.4 — 2026-07-13
+
+### Changed
+
+- **One implementation of apply/revert (`patch/mw_patch.py`).** The "strip the
+  `TRUECLOUD_PATCH` block" logic existed twice — in `apply.sh`'s heredoc and in an
+  inline heredoc in `uninstall.sh` — and the uninstall copy was the untested one.
+  That is exactly how the two could have drifted apart, with `apply.sh` reverting
+  one set of files and `uninstall.sh` another. Both now call the same tested
+  module (17 new tests, including that `revert_nested` never touches `restic.py`,
+  which belongs to the providers module and whose removal would silently break B2
+  backups).
+
+  `apply.sh` imports it fail-safe: if it cannot, the backend patch is skipped and
+  middlewared starts stock, which is this script's whole design principle. The
+  import uses `sys.path.append`, never `insert(0)` — prepending would give
+  `patch/` precedence over the stdlib for that interpreter, so a future
+  `patch/json.py` would shadow the real `json` and break the boot.
+
+### Docs
+
+- The README's `create_task.py` example still taught `--password <secret>`, which
+  is how a security fix quietly fails to land. It now shows `--password-stdin`.
+
+## v0.3.3 — 2026-07-13
+
+### Security
+
+- **The restic repository password no longer passes through a process's argv.**
+  `create_task.py` shelled out to `midclt call cloud_backup.create '<json>'`, and
+  that JSON contains the repo password — so it appeared in the process's argv,
+  which is world-readable via `ps`, for the duration of the call. That password is
+  the encryption key for the entire cloud backup repository.
+
+  It now talks to the middleware through `truenas_api_client` (the library that
+  backs `midclt` itself), so the password never leaves the process's memory.
+
+- **`--password` no longer required.** Passing a secret as a CLI argument writes it
+  to shell history permanently. `--password-stdin` reads it from stdin, and with
+  neither flag the tool prompts via `getpass`. `--password` still works but now
+  warns.
+
+### Fixed
+
+- **`uninstall.sh` could leave every patch installed.** It reverted by unmounting
+  the overlay — but `apply.sh` only mounts one when the target directory is
+  read-only. On a writable `/usr` it patches the real files in place, and uninstall
+  would remove the boot hook, report success, and leave the patch applied. It now
+  strips the appended blocks from the middleware files explicitly.
+
+- **`create_task.py.__version__` had been stuck at `0.2.0`** for three releases.
+  The version-drift check added in v0.3.1 only looked at `VERSION=` in shell
+  scripts, so it missed the one file that actually shows a version to users
+  (`--version`). The check now covers `__version__` too — and caught this
+  immediately.
+
+## v0.3.2 — 2026-07-13
+
+### Fixed
+
+- **`install.sh --disable-nested-snapshots` did not actually disable anything
+  until the next reboot.** `apply.sh` only ever *added* patches — there was no
+  revert path. Disabling removed the opt-in marker and then merely *skipped*
+  re-applying, but the overlay persists for the whole boot, so the previously
+  patched `plugins/cloud/{snapshot,crud}.py`, `plugins/cloud_backup/sync.py` and
+  `_truecloud_nested.py` were all still sitting there — and middlewared
+  re-imported them on the restart `install.sh` performs.
+
+  It printed *"DISABLED (stock guard restored)"* while the feature kept running.
+  Someone turning it off *because they were worried about it* would have believed
+  it was off.
+
+  `apply.sh` now actively reverts: it removes the module first (every injected
+  block is guarded by `if _tc_nested is not None`, so the stock guard is restored
+  even if a later step fails), then strips its appended blocks from the three
+  patched files. `restic.py` also carries a `TRUECLOUD_PATCH` block but belongs to
+  the *providers* module and is deliberately left alone — reverting it would break
+  B2 backups. `install.sh --disable` also tears down any staging tree first, since
+  those bind mounts pin ZFS snapshots that could otherwise never be destroyed.
+
+  Updating **without** the flag was always correct and is unchanged: the nested
+  module is never installed into middleware unless it is explicitly enabled.
+
+## v0.3.1 — 2026-07-13
+
+### Added
+
+- **Automated releases.** Pushing a `v*` tag runs the full test suite and then
+  cuts a GitHub release whose body is the matching `CHANGELOG.md` section — so
+  release notes have exactly one source of truth, and no second place to go stale.
+  The workflow refuses to publish if the tests fail, if the tag does not match the
+  `VERSION=` declared by every script, or if the CHANGELOG has no section for it.
+
+- **Version-drift check.** `VERSION=` had silently diverged to three different
+  values across `install.sh`, `uninstall.sh`, `recover.sh`, and `patch/apply.sh`,
+  and nothing noticed. CI now asserts every script agrees with the others and with
+  the newest CHANGELOG entry.
+
+### Note
+
+- Releases for `v0.2.0` and `v0.2.1` were backfilled — they had been tagged but
+  never released, so the releases page jumped v0.1.0 → v0.3.0 and hid the fix for
+  the boot race that took every app down.
+
+## v0.3.0 — 2026-07-13
+
+### Added
+
+- **`snapshot = true` now works on datasets that have child datasets** —
+  **opt-in, off by default** (`install.sh --enable-nested-snapshots` /
+  `--disable-nested-snapshots`). It changes how backups read their source data,
+  so it is never enabled implicitly; with neither flag `install.sh` preserves
+  the existing setting, so a `git pull && bash install.sh` cannot silently flip
+  it. When disabled, `apply.sh` skips the patch entirely and the stock guard
+  remains. `uninstall.sh` tears down any staging mounts and removes the marker.
+  Stock TrueNAS refuses this with *"This option is only available for datasets
+  that have no further nesting"*, which makes the snapshot option unusable for
+  the single most common case on any box running Apps — every app is its own
+  dataset, often with `config`/`pgdata` children of its own. Without it, the
+  backup reads **live** files: databases are captured mid-write, and a busy app
+  rewriting its files can stall a backup indefinitely as restic chases a moving
+  target.
+
+  The stock guard is **correct, and it is not an arbitrary limit.**
+  `plugins/cloud/snapshot.py` already takes a *recursive* ZFS snapshot, but it
+  then points the backup tool at the **parent** dataset's
+  `.zfs/snapshot/<snap>/` directory — and ZFS does not expose child datasets
+  through a parent's snapshot directory:
+
+  ```
+  /mnt/Tap/.zfs/snapshot/<snap>/apps/               -> 0 entries (children invisible)
+  /mnt/Tap/apps/lidarr/config/.zfs/snapshot/<snap>/ -> the real data
+  ```
+
+  So without the guard the backup tool would walk a near-empty tree, report
+  SUCCESS, and upload almost nothing. iX gate the config rather than ship a
+  backup that lies about succeeding.
+
+  This release implements the missing half. After the (already recursive)
+  snapshot is taken, every descendant dataset's own `.zfs/snapshot/<snap>` is
+  bind-mounted into a **staging tree** mirroring the original layout, and the
+  backup tool is pointed at the staging root — a complete, consistent,
+  point-in-time view of the whole subtree. Only then is the guard relaxed.
+
+  Safety properties, in order of importance:
+
+  - **Staging failure is loud.** If any descendant cannot be staged, the backup
+    fails. A silently-incomplete backup is the exact outcome the stock guard
+    exists to prevent, and it would be worse than not having the feature.
+  - **A post-mount verification pass** asserts every planned target is really a
+    mountpoint and the staging root is non-empty, so this can never regress into
+    the empty-backup failure it is meant to fix.
+  - **The guard is relaxed last.** `apply.sh` installs the traversal, patches
+    `snapshot.py`, then `sync.py`, and only then `crud.py`. A partial failure
+    leaves the guard intact and the option merely unavailable — never
+    "guard removed, traversal missing".
+  - **The patch owns the whole snapshot lifecycle.** `zfs.snapshot.delete`
+    defaults to `recursive=False` and stock `restic_backup()` calls it with no
+    options. Stock gets away with that only because its validation means
+    `recursive` is never True in the field — but enabling nested datasets makes
+    recursive snapshots real, so the parent now has one child snapshot per
+    descendant dataset (160+ on a typical Apps pool). Relying on stock's delete
+    would therefore orphan every child snapshot **on every successful run**.
+    This patch sweeps the parent *and* all children, is idempotent against
+    stock's `finally` winning the race, records the snapshot in a sidecar file
+    (so a middlewared restart mid-backup cannot orphan it), reclaims the tree
+    left by a crashed run, and deletes the tree when staging fails — where
+    sync.py's own `finally` would otherwise delete nothing at all, because its
+    `snapshot` local never gets assigned.
+  - **The dataset list is enumerated *after* the snapshot, never before.** A
+    list read beforehand can miss a dataset created in the gap: the recursive
+    snapshot would capture it but the staging plan would not, silently omitting
+    its data. Read afterwards, an unsnapshotted dataset trips the staging check
+    and fails the run loudly instead.
+  - **Every injected block no-ops** if `_truecloud_nested` is absent.
+  - Datasets that cannot contribute to a file tree (`mountpoint=none|legacy`,
+    unmounted/locked, encrypted-and-locked) are skipped and **reported** —
+    never dropped silently.
+  - Scoped to `cloud_backup` only. Cloud Sync (rclone) shares the same
+    validation mixin but has no staging teardown wired in, so its guard is left
+    in place deliberately.
+
+  Side benefit: the staging root is a **stable** path per task, so restic can
+  find its parent snapshot between runs. Stock's
+  `.zfs/snapshot/<name>-<timestamp>/` path changes every run, which defeats
+  restic's parent detection and forces a full re-scan each time.
+
+- **CI** (GitHub Actions): shellcheck + `bash -n` on every script, ruff, and
+  pytest on Python 3.11/3.12/3.13. Includes tests that `compile()` the
+  `*_BLOCK` strings — they are Python source appended to live middlewared
+  modules, so a syntax error there would break the box at boot, and nothing
+  previously checked them.
+
+### Changed
+
+- **The patch is now two independent modules, and each retires on its own.**
+  Previously the native-support check looked only for native B2 restic support
+  and, on finding it, set the kill switch and disabled *everything*. With a
+  second capability in the patch that would silently take a still-needed module
+  down with the superseded one — TrueNAS is likely to ship one of these long
+  before the other.
+
+  `apply.sh` now detects each separately (`providers`: does `B2RcloneRemote`
+  carry a real `get_restic_config()`; `nested`: is the *"no further nesting"*
+  validation still in `plugins/cloud/crud.py`), skips just the superseded one,
+  and only sets the kill switch once **both** are done. The UI patch belongs to
+  `providers` and is skipped with it. The deferred middlewared restart now fires
+  when *any* still-needed module landed — keying it off `providers` alone would
+  have left a freshly-patched `nested` module on disk and never loaded on a
+  native-B2 box. `hook_status.json` reports each module with an `active` flag and
+  a reason.
+
+- README rewritten to be less alarmist: dropped the warning boxes and the
+  disclaimer's fear-bulleting in favour of plain statements, and documented the
+  two-module design. The one caveat kept as a plain sentence: the `mount --bind`
+  staging step has not yet been exercised by a live backup run.
+
+- Version strings in `install.sh`, `uninstall.sh`, and `recover.sh` were stale
+  at `0.0.4`; all scripts now report the same version.
+- `patch_ui.py`: replaced a `try`/`except`/`pass` with `contextlib.suppress`
+  (no behaviour change; satisfies the new lint gate).
+
+### Removed
+
+- `patch/__pycache__/create_task.cpython-314.pyc` was committed to the
+  repository; it is now untracked and `__pycache__/` is gitignored.
+
+### Fixed (post-merge audit)
+
+- **`create_task.py verify` failed on a default install.** `hook_status.json`
+  emitted a per-file entry for the nested module with `ok: false` whenever the
+  feature was switched off — which is the default — so `verify` printed `[FAIL]`
+  and exited 1 right after the README told users to run it. Status is now
+  reported per *module* with an `active` flag, and `verify` renders an inactive
+  module as `[SKIP]` rather than a failure.
+- **A partial apply suppressed the middlewared restart.** The exit code
+  conflated "nothing applied" with "one module applied, one failed", so a failing
+  providers patch would prevent the restart that a freshly-applied nested patch
+  needs — leaving it on disk and never loaded. Exit 2 now means partial, and the
+  restart still fires.
+- **The native-nested probe could never fire.** It scanned `crud.py` for the
+  guard message, but our own injected block *quotes* that message, so once
+  applied the probe would always conclude the guard was still present. It now
+  reads only the stock portion of the file.
+- `recover.sh` did not unmount staging trees, so an emergency recovery left bind
+  mounts pinning ZFS snapshots that could then never be destroyed.
+- `uninstall.sh` deleted sidecar files without reading them. A sidecar is the
+  only record that an interrupted run's snapshot tree is still on disk; both
+  scripts now name the snapshot (`zfs destroy -r ...`) before clearing it.
+
+- **The native-nested probe could never detect the guard, silently disabling the
+  whole module.** Stock splits the message across adjacent string literals:
+
+  ```python
+  verrors.add(f"{name}.snapshot", "This option is only available for datasets that have no further "
+                                  "nesting")
+  ```
+
+  Python concatenates those at runtime — so the *errmsg* is contiguous and the
+  runtime filter works — but the **source never contains the whole phrase**. The
+  probe's substring search found nothing, concluded iX had removed the guard, and
+  skipped the nested module as "already native". `apply.log` would report
+  *"TrueNAS now handles nesting natively"* and the feature would never work.
+  It fails safe (the stock guard stays, so no data is at risk) but the module was
+  100% dead. The probe now strips whitespace and quotes before matching, which is
+  robust to any wrapping style. Caught only by running the probe against real
+  middlewared; there is now a regression test that executes apply.sh's own probe
+  code against the real wrapped source.
+
+### Changed (production audit)
+
+- **`delete_snapshot_tree` now uses a single recursive delete.** It previously
+  removed the parent and each child snapshot one at a time — 252 sequential
+  middleware calls on a real pool. That is slow, but the real problem is that it
+  is **not atomic**: a run killed part-way through the sweep leaves exactly the
+  orphaned snapshots the function exists to prevent. It now issues one
+  `zfs.snapshot.delete(..., {"recursive": True})` and falls back to the
+  name-by-name sweep only when that fails (e.g. stock's `finally` already removed
+  the parent, which leaves the children behind).
+
+### Refactored
+
+- Staging teardown had been copy-pasted into `uninstall.sh` and `recover.sh` —
+  two untested shell copies of the fiddly depth-ordering and lazy-umount logic.
+  Both now call `python3 patch/truecloud_nested.py cleanup`, so there is one
+  implementation and it is the one under test.
+- Dropped the in-memory `ACTIVE` dict. The sidecar file was already the source of
+  truth; a second in-process record could only desync — and it is the
+  middlewared-restart case (which empties it) that must not orphan a snapshot
+  tree. One record, on disk, or none.
+
+### Validated in production
+
+An unattended scheduled backup of a live 252-dataset pool (`/mnt/Tap`, TrueNAS
+25.10) ran through the staging tree end to end:
+
+- 252 datasets recursively snapshotted, 173 bind mounts built and verified
+- completed in **18m14s**, `SUCCESS` — the same task previously stalled at 74%
+  for over 12 hours reading live files
+- **zero** orphaned ZFS snapshots and **zero** stale mounts afterwards, which is
+  the failure mode that would otherwise have accumulated 251 snapshots per run
+
+### Known issues
+
+- Stock `restic_backup()` deletes the ZFS snapshot in its own `finally`, which
+  fails with `EBUSY` while the staging bind mounts pin it. It logs one benign
+  `Error deleting snapshot ...` warning per run; the patch then unmounts and
+  deletes the snapshot for real. The warning is expected and harmless.
+
+## v0.2.1 — 2026-07-09
+
+### Fixed
+
+- **Deferred restart raced the rest of boot, leaving all apps and dashboard
+  stats down.** The `truecloud-mw-restart` unit introduced in v0.0.4 relied
+  on systemd ordering (`After=multi-user.target`, `After=ix-postinit.service`),
+  which cannot see middlewared's *internal* boot work. Observed on 25.10.4:
+  the restart fired two seconds into `ix-reporting.service`'s
+  `midclt call reporting.start_service` and before the docker/apps startup
+  task (created on middlewared's system-ready event) had run. Both were
+  killed, and nothing retries them until the next boot — every app stayed
+  down (`docker.status` FAILED, the apps dataset never mounted), netdata
+  never started (no dashboard hardware stats), and the SMB middleware
+  backend was left uninitialized.
+
+  The transient unit now runs `patch/wait_restart.sh` instead of restarting
+  directly: it waits for the systemd boot job queue to drain
+  (`systemctl is-system-running --wait`, covering in-flight `ix-*` oneshots
+  such as ix-reporting), then polls `midclt call docker.status` until the
+  docker state machine leaves its transitional states, then allows a short
+  grace period for middleware-internal tasks with no queryable state before
+  issuing `systemctl try-restart middlewared`. The unit no longer sets
+  `Type=oneshot` — a oneshot's start job stays in the very queue the script
+  waits on and would deadlock on itself. All waits are bounded and fail
+  open: worst case the restart still happens, just later.
+
+  Recovery on a boot that already hit this (without rebooting):
+  `midclt call reporting.start_service` and
+  `midclt call docker.state.start_service true`.
+
+## v0.2.0 — 2026-07-08
+
+### Changed
+
+- **`create_task.py` now uses the TrueNAS middleware via `midclt` instead of the
+  deprecated `/api/v2.0` REST API**, which is removed in TrueNAS 26.04. Practical
+  effects:
+  - Run the script **on the TrueNAS host** — it uses the local middleware socket, so
+    it no longer needs a host address or API key.
+  - `--host`, `--api-key`, and `--insecure` are accepted but **ignored** (a deprecation
+    note is printed); they will be removed in a future release.
+  - `list-credentials` → `cloudsync.credentials.query`, `list-tasks` →
+    `cloud_backup.query`, `create` → `cloud_backup.create`.
+- Dropped the `ssl`/`urllib` HTTP client; no TLS certificate handling is needed anymore.
+
+## v0.1.0 — 2026-07-08
+
+### Added
+
+- `create --cache-path PATH` — sets the restic cache directory on the task.
+  Without a cache path, TrueNAS runs restic with `--no-cache`, which re-reads all
+  repository metadata from the provider on every run and is glacially slow on
+  large repos (a 564 GB dataset estimated **55 days** to a first backup). Tasks
+  created without `--cache-path` now print a warning explaining the consequence.
+
 ## v0.0.4 — 2026-07-06
 
 ### Fixed

README.md

@@ -1,185 +1,163 @@
 # truenas-truecloud-patch
 
-Extends TrueNAS SCALE's **TrueCloud Backup** feature to work with S3-compatible
-providers and native Backblaze B2, instead of Storj only.
+Extends TrueNAS SCALE's **TrueCloud Backup** to:
 
----
+- back up to **Backblaze B2 and any S3-compatible provider**, not just Storj;
+- snapshot **datasets that have child datasets** — which is every box running Apps.
 
-## Why this exists
+**Requires TrueNAS SCALE 24.10 or newer.** TrueCloud Backup does not exist before
+that, and `install.sh` will refuse.
 
-In 2026, Storj raised the price of their TrueNAS-integrated storage tier from
-**$5/month to $50/month** — a 10× increase. For many home lab and small-office
-users, the TrueCloud Backup feature became unaffordable overnight.
-
-TrueCloud Backup is the only native TrueNAS mechanism that provides:
-- Integrated ZFS snapshot support before each backup
-- Restic-based incremental deduplication
-- Scheduled tasks with progress and log tracking in the UI
-- Dataset lock integration
-
-Running restic manually is possible but loses all of the above.
-This patch restores access to the TrueCloud Backup feature for users who
-need a provider other than Storj, with storage they already pay for or that
-costs a fraction of the new Storj price.
-
----
-
-## ⚠ Disclaimer — please read before installing
-
-**This project is unofficial, unsupported, and not affiliated with iXsystems
-or the TrueNAS project in any way.**
-
-By installing this patch you accept the following:
-
-- **Unsupported configuration.** TrueNAS support staff are not obligated to
-  help with any issue on a system running this patch. If you file a bug report,
-  remove the patch first and reproduce the issue on an unmodified system.
-
-- **May break on TrueNAS updates.** The patch targets internal middleware APIs
-  that are not part of any public contract. They can change at any time. When
-  they do, the patch silently degrades to Storj-only behaviour rather than
-  breaking TrueNAS — but you should check the log after each update.
-
-- **Your backups are your responsibility.** Verify that your backup jobs
-  complete successfully and that restores work before relying on them for
-  disaster recovery.
-
-- **No warranty.** This software is provided as-is. See the LICENSE file.
-
-If TrueNAS adds native B2 or S3 support to TrueCloud Backup, the patch
-detects it at boot, disables itself, and tells you to run `uninstall.sh` —
-see [Native support](#if-truenas-adds-native-support) below.
-
----
-
-## What is actually patched
-
-**Nothing in TrueNAS's persistent database or configuration is modified**
-(other than the boot-hook entry itself). On every boot, `patch/apply.sh` runs
-as a PREINIT script. It mounts a writable
-[overlayfs](https://docs.kernel.org/filesystems/overlayfs.html) over the
-relevant directories in `/usr/` (upper layer in `/run` tmpfs), then patches
-`b2.py` and `restic.py` inside that overlay. The overlay is volatile — it
-exists only for the current boot — but the PREINIT script recreates it
-automatically on every subsequent boot. Nothing in `/usr/` is written to
-directly.
-
-PREINIT scripts are executed *by* middlewared, which by then has already
-imported the stock modules — so after patching, `apply.sh` schedules a single
-detached middlewared restart (transient systemd unit `truecloud-mw-restart`,
-ordered after `multi-user.target`) that loads the patched modules once boot
-completes. Expect one middlewared restart shortly after every boot; the UI
-and API are briefly unavailable while it happens, and running services are
-not affected.
-
-| Layer | What changes | Technique |
-|---|---|---|
-| **Backend** | `B2RcloneRemote` gains `get_restic_config()` — skipped automatically if TrueNAS already provides one on the class. `restic.py` URL builder is fixed: strips the stray leading slash and converts the slash separator to a colon (`b2:bucket:path`), which is the format restic 0.16.x expects. URL wrapper is a no-op if the URL is already correctly formed. | File patch applied inside the overlayfs upper layer |
-| **UI** | The Angular bundle's `filterByProviders` binding is widened from `["STORJ_IX"]` to `["STORJ_IX","S3","B2"]` | In-place text replacement in the compiled JS chunk; original is backed up before patching |
-
-Both changes are **fail-safe**: if a patch cannot be applied (e.g. TrueNAS
-restructured the relevant code), middlewared starts normally with Storj-only
-support and the reason is logged to `apply.log` in your repo root.
-
-## Supported providers after patching
-
-| Provider | Credential type in TrueNAS |
-|---|---|
-| Backblaze B2 (native B2 API) | `B2` |
-| AWS S3, Wasabi, Cloudflare R2, MinIO, and any S3-compatible endpoint | `S3` |
-| Storj (unchanged) | `STORJ_IX` |
-
-## How persistence works
-
-TrueNAS SCALE updates replace `/usr/` entirely. The patch survives by keeping
-this repository on a **persistent ZFS pool** (your data pool, not `/tmp` or a
-system path) and registering a **PREINIT initshutdownscript** in the TrueNAS
-database — the one piece of state that survives both reboots and OS updates.
-On every boot, `patch/apply.sh` runs (executed by middlewared after pools are
-imported), mounts a writable
-[overlayfs](https://docs.kernel.org/filesystems/overlayfs.html) over the
-relevant directories (upper layer in `/run`, recreated each boot), patches
-`b2.py` and `restic.py` directly in that overlay, re-patches the UI bundle,
-and schedules the one-time deferred middlewared restart that loads the
-patched backend. No extra configuration is needed.
+> Storj raised the price of their TrueNAS-integrated tier from **$5/month to
+> $50/month** in 2026. TrueCloud Backup is the only native TrueNAS feature that gives
+> you pre-backup ZFS snapshots, restic dedup, scheduled tasks with UI progress, and
+> dataset-lock integration. Running restic by hand loses all of it. This gets the
+> feature back with storage you already pay for.
 
 ---
 
 ## Install
 
-Clone the repository to a **persistent ZFS pool** so it survives OS updates,
-then run `install.sh` from there:
+Clone it onto a **pool** (not the boot device — that is wiped on TrueNAS upgrades),
+then run `install.sh` as root:
 
 ```bash
-# Replace /mnt/tank with your pool name
 git clone https://github.com/sudolulo/truenas-truecloud-patch.git \
-    /mnt/tank/truenas-truecloud-patch
+    /mnt/tank/truenas-truecloud-patch          # replace `tank` with your pool
 cd /mnt/tank/truenas-truecloud-patch
-bash install.sh
+sudo bash install.sh
 ```
 
-The directory you clone into becomes the **permanent install location**. The
-PREINIT boot hook is registered with the exact path you chose, and TrueNAS will
-call that path on every boot.
+That registers a PREINIT boot hook, patches middleware in a volatile overlay, and
+restarts middlewared. **It survives TrueNAS updates** — the patch is re-applied at
+every boot, never written to the system dataset.
 
-> **Do not delete or move the repository after install.**
-> If you need to relocate it, run `bash uninstall.sh` first, move the directory,
-> then run `bash install.sh` again from the new location. Deleting the repo
-> without uninstalling leaves a dangling PREINIT hook in the TrueNAS database —
-> if that happens, see [Emergency recovery](#emergency-recovery) below.
-
-Refresh your browser. S3 and B2 credentials now appear in the
-**Data Protection → TrueCloud Backup → Add** credential dropdown.
-
-## Updating
-
-To update to a new version of the patch:
+Nested-dataset snapshots are **opt-in**:
 
 ```bash
-cd /mnt/tank/truenas-truecloud-patch
-
-# If install.sh was previously run as root, the .git directory may be owned
-# by root. Fix it first, or just pull as root:
-sudo git pull          # easiest option
-# — or —
-sudo chown -R $(whoami) .git && git pull
-
-bash install.sh
+sudo bash install.sh --enable-nested-snapshots
 ```
 
-`install.sh` clears any stale kill switch, re-applies the updated patches,
-and restarts middlewared. Run `python3 patch/create_task.py verify` afterwards
-to confirm the patches loaded successfully.
+Then create a TrueCloud Backup task in the UI with a B2 or S3 credential, or from
+[the CLI](docs/cli.md).
 
-Check [CHANGELOG.md](CHANGELOG.md) to see what changed between versions.
+**Check it worked:**
+
+```bash
+sudo python3 patch/create_task.py verify
+```
+
+If something is wrong, the reason is in `apply.log` — start at
+[Recovery](docs/recovery.md).
 
 ---
 
-## Creating a task via CLI
+## TrueNAS compatibility
 
-If the UI still shows only Storj after refreshing (e.g. the JS bundle pattern
-changed in a new TrueNAS version), create tasks directly via the REST API:
+<!-- BEGIN COMPAT MATRIX (generated by tools/compat.py --matrix --markdown) -->
+| TrueNAS | B2/S3 providers | Nested snapshots | Hardware-verified |
+| --- | --- | --- | --- |
+| 24.10.2.4 | ok | ok | — |
+| 25.04.2.6 | ok | ok | — |
+| 25.10.4 | ok | ok | nested + providers; 252-snapshot recursive backup of /mnt/Tap, 18m |
+| 26.0.0-BETA.3 _(unreleased)_ | ok | **BROKEN** | — |
+| master _(unreleased)_ | **BROKEN** | **BROKEN** | — |
+
+| verdict | meaning |
+| --- | --- |
+| **ok** | Every assumption the patch makes about middleware still holds. |
+| **BROKEN** | middleware changed underneath the patch. `apply.sh` **refuses to apply that module** on this version and leaves TrueNAS stock, so backups keep working — without the module's feature. |
+| **native** | TrueNAS does this itself now. The module retires; it is not a failure. |
+
+"ok" means *the patch's assumptions hold*, checked automatically against iX's
+source. It does not mean a human ran a backup on it — that is the
+**Hardware-verified** column, which is filled in by hand and only by doing it.
+<!-- END COMPAT MATRIX -->
+
+The table is **regenerated daily by CI** against iXsystems' actual middleware source
+— it is not a claim somebody typed once and forgot.
+
+**TrueNAS 26: nested snapshots are not supported yet, and upgrading will not break
+you.** 26 rewrites `cloud_backup` and deletes the ZFS methods this module calls. On
+26 `apply.sh` finds that the assumptions no longer hold and **does not apply the
+module**: TrueNAS is left stock, B2/S3 keeps working, nested datasets are simply not
+covered, and the reason is named in `apply.log`. A broken backup is worse than a
+missing feature. Details: [How it works](docs/how-it-works.md#truenas-26).
+
+---
+
+## Updating
 
 ```bash
-# Replace /mnt/tank/truenas-truecloud-patch with your clone path
-
-# List your cloud credentials to find the right ID
-python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py \
-    --host 192.168.1.1 --api-key <key> list-credentials
-
-# Create a task with a B2 credential (id=3)
-python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py \
-    --host 192.168.1.1 --api-key <key> create \
-    --name "tank-to-b2" \
-    --path /mnt/tank/data \
-    --credential 3 \
-    --bucket my-bucket \
-    --folder backups/tank \
-    --password "restic-repo-password" \
-    --keep-last 14
+cd /mnt/tank/truenas-truecloud-patch
+sudo bash update.sh              # newest release
+sudo bash update.sh --check      # what would change?
+sudo bash update.sh --rollback   # back to the previous version
 ```
 
-Get an API key from **System → API Keys → Add**.
+`update.sh` refuses to run on a dirty checkout, pins you to a release tag, and keeps
+the previous revision so a rollback is one command. **There is no auto-update**: this
+patches system internals as root, and a bad commit reaching your box unattended would
+detonate on the next reboot — v0.0.4 shipped exactly such a bug and took 54 apps
+down. The manual step *is* the safety gate.
+
+---
+
+## Update alerts
+
+When a newer release exists, the patch raises a **TrueNAS alert** (the bell in the
+UI) telling you so. It's on by default and checks once a day.
+
+```bash
+bash install.sh --no-update-alerts   # turn it off
+bash install.sh --update-alerts      # turn it back on
+```
+
+**It will not nag you about a README.** A release whose CHANGELOG contains only a
+`### Docs` section changed no code, and raises nothing. Anything that touched the
+system raises an INFO alert; a release with a `### Security` section raises a
+WARNING. The CHANGELOG's own section headings are the signal, and a security fix
+anywhere in the range escalates the whole span — a docs-only release on top of a
+security fix still reports as security.
+
+**Release candidates never alert.** They are invisible to `update.sh` and to the
+alert, which both take the newest plain `vX.Y.Z` tag. That is what lets debugging
+happen in `-rc` tags instead of in your notification bell — see
+[Releasing](docs/releasing.md).
+
+The changelog is read from whichever forge `origin` points at, derived from the
+remote rather than hard-coded. That is not cosmetic: when the changelog cannot be
+read, the alert deliberately fires **anyway** rather than risk hiding a security
+fix — so a wrong URL would not silence the alert, it would make it fire on *every*
+release, including the documentation-only ones this section promises to suppress.
+
+### How it works, and why it's built this way
+
+TrueNAS **cannot raise an alert from the CLI** — `midclt` exposes only
+`alert.dismiss`, `alert.list`, `alert.list_categories`, `alert.list_policies` and
+`alert.restore`. Alert *creation* is internal to middlewared, and none of its ~60
+one-shot alert classes is generic enough to reuse. So the only way to get a real
+alert is to register an `AlertSource`, which is what `patch/alert_source.py` does.
+
+That is also the **least invasive** thing this patch does:
+
+| | |
+|---|---|
+| providers module | **modifies** stock files (appends code to `b2.py`, `restic.py`) |
+| nested module | **modifies** stock files (3 middleware modules) |
+| **update alert** | **adds one file. Modifies nothing.** |
+
+It's the native mechanism — the same one every built-in TrueNAS alert uses — and
+TrueNAS polls it itself, so there is no cron job and no systemd timer.
+
+- **Fail-safe.** Every error path returns `None`. It cannot take middlewared down.
+- **Read-only.** `git ls-remote` plus an HTTPS fetch of the CHANGELOG. It never
+  writes to `.git`, so it cannot leave root-owned objects behind the way a
+  `git fetch` from middlewared (which runs as root) would.
+- **Removed by `uninstall.sh`.**
+
+It only *tells* you. It never updates anything — see
+[Updating](#updating).
 
 ---
 
@@ -195,183 +173,39 @@ and restores the original UI bundle from backup.
 
 ---
 
-## If TrueNAS adds native support
+## Documentation
 
-`apply.sh` checks at every boot whether TrueNAS has shipped native B2 restic
-support (by inspecting `B2RcloneRemote.__dict__`). If it has:
+| | |
+|---|---|
+| [Nested-dataset snapshots](docs/nested-snapshots.md) | Why stock refuses, what this does instead, and **how to verify your backups actually contain the data** |
+| [How it works](docs/how-it-works.md) | What is patched, how it survives updates, the boot sequence, and what happens when TrueNAS goes native |
+| [Recovery](docs/recovery.md) | middlewared won't start, blank web UI, `verify` shows FAIL |
+| [CLI](docs/cli.md) | Creating tasks with `create_task.py` |
+| [Development](docs/releasing.md) | Tests, CI, and the release process |
 
-1. The kill switch (`disabled` file) is set — no patching on any future boot.
-2. Any active overlays are unmounted immediately.
-3. The following message is written to `apply.log`:
+## What's in the repo
 
-```
-NOTICE: TrueNAS now provides native B2 restic support — truecloud-patch is no longer needed.
-NOTICE: Setting kill switch; patching will be skipped on all future boots.
-NOTICE: Run the following to fully remove the patch:
-NOTICE:   bash /mnt/tank/truenas-truecloud-patch/uninstall.sh
-```
+| Path | What it is |
+|---|---|
+| `install.sh` | Register the boot hook, patch, restart middlewared. Also `--enable/--disable-nested-snapshots`. |
+| `update.sh` | Fetch and apply a newer release. `--check`, `--rollback`, `--to`, `--main`. |
+| `uninstall.sh` | Remove everything. |
+| `recover.sh` | Emergency: kill switch + restart against stock files. |
+| `patch/apply.sh` | The PREINIT script. Runs at **every boot**. |
+| `patch/truecloud_nested.py` | Nested-dataset staging: plan, mount, verify, tear down, sweep snapshots. |
+| `patch/create_task.py` | Create tasks with S3/B2 credentials; `verify` the patch state. |
+| `tools/compat.py` | What the patch assumes about middlewared — and the checker. Run daily by CI *and* at every boot. |
+| `release.sh` | Cut a release. Two stages, and the second is refused without the first. |
 
-Check the log after any TrueNAS update:
-```bash
-cat /mnt/tank/truenas-truecloud-patch/apply.log | tail -20
-```
+## Before you install
 
-**Scenarios where the auto-detect may not fire** (manual check needed):
+- This is **unofficial** and not affiliated with iXsystems.
+- It patches **internal middleware APIs** with no stability contract. Every patch is
+  fail-safe: if it cannot apply, middlewared starts normally and the reason is logged.
+- **Test your restores.** True of any backup; more so here. See [Verifying it
+  works](docs/nested-snapshots.md#verifying-it-works).
+- Filing a TrueNAS bug? **Remove the patch first** and reproduce on a stock system.
+- Provided as-is, no warranty. See LICENSE.
 
-| Scenario | What happens | Action |
-|---|---|---|
-| B2 support added to a **base class** (not `B2RcloneRemote` directly) | `__dict__` check misses it; our method shadows native | Uninstall manually |
-| B2 **credential schema changed** (e.g. `provider["account"]` renamed) | `KeyError` on first backup | Uninstall or update the patch |
-| **URL builder** fixed but B2 class unchanged | URL wrapper becomes a no-op; no harm, but patch is dead weight | Uninstall at your convenience |
-
----
-
-## After a TrueNAS update
-
-1. Check the log: `cat /mnt/tank/truenas-truecloud-patch/apply.log | tail -30`
-2. If you see "WARNING: … pattern not found", the UI patch needs updating.
-   [Open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues)
-   with your TrueNAS version number.
-3. The backend patch (B2 support + URL fix) is more stable — check that a
-   B2 backup job still completes successfully after any update.
-
----
-
-## Emergency recovery
-
-### middlewared won't start
-
-Run this from the TrueNAS shell (local console, SSH, or the debug shell in
-the UI):
-
-```bash
-bash /mnt/tank/truenas-truecloud-patch/recover.sh
-```
-
-Replace the path with your clone location. This creates a kill-switch file
-(`disabled`) in the repo root, unmounts the overlay so the original files are
-visible immediately, then restarts middlewared. No reboot required.
-
-If you cannot run a script and only have a bare shell prompt:
-
-```bash
-touch /mnt/tank/truenas-truecloud-patch/disabled
-systemctl restart middlewared
-```
-
-If you don't remember where you cloned the repo (midclt won't work while middlewared is
-down), find the path two ways:
-
-```bash
-# Option 1 — search the filesystem:
-find /mnt -name "recover.sh" -path "*/truenas-truecloud-patch/*" 2>/dev/null
-
-# Option 2 — query the TrueNAS database directly:
-sqlite3 /data/freenas-v1.db \
-    "SELECT script FROM initshutdownscript WHERE comment = 'TrueCloud provider patch (S3/B2)';"
-```
-
-The `script` column shows the full path to `patch/apply.sh`; your clone root is one
-level up (strip `/patch/apply.sh` from the end). Then run the `touch` command above
-with that path.
-
-If middlewared **still** won't start after the kill switch is set, the problem
-is unrelated to this patch. Check:
-
-```bash
-journalctl -u middlewared -n 50
-```
-
-To re-enable the patch once you have investigated:
-
-```bash
-rm /mnt/tank/truenas-truecloud-patch/disabled
-bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh
-systemctl restart middlewared   # manual apply.sh runs never restart for you
-```
-
----
-
-### Web UI is blank or broken
-
-If the TrueNAS web interface loads blank or shows JavaScript errors, the
-Angular bundle may have been interrupted mid-write (e.g. power cut during
-boot). The original bundle is always backed up before patching, so recovery
-is straightforward:
-
-```bash
-# Find the backup (the path varies by TrueNAS version):
-find /usr/share/truenas /usr/share/truenas-ui /var/www/truenas -name "*.js.pre-truecloud-patch" 2>/dev/null
-
-# Restore it — substitute the actual path from the find output:
-mv /usr/share/truenas/webui/main.XXXXXXXX.js.pre-truecloud-patch \
-   /usr/share/truenas/webui/main.XXXXXXXX.js
-```
-
-Refresh your browser. The UI will return to normal (Storj-only until the
-patch re-runs at next reboot, or you run
-`bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh` manually).
-
----
-
-### Backend verify shows FAIL
-
-```bash
-python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
-```
-
-If one or more entries show `[FAIL]`:
-
-1. **Check the apply log** for errors during the last boot:
-   ```bash
-   cat /mnt/tank/truenas-truecloud-patch/apply.log | tail -40
-   ```
-2. **Check middlewared's own log** for Python tracebacks:
-   ```bash
-   grep -i "truecloud\|traceback\|error" /var/log/middlewared.log 2>/dev/null | tail -30
-   journalctl -u middlewared -n 50
-   ```
-3. **A FAIL is non-fatal.** middlewared runs normally; the affected provider
-   falls back to Storj-only. Your existing backups are not at risk.
-4. **If the detail says the module doesn't exist**, a TrueNAS update renamed
-   or restructured the internal API.
-   [Open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues)
-   with your TrueNAS version number and the full verify output.
-
----
-
-## Troubleshooting
-
-**Apply log** (check after each reboot or install):
-```bash
-cat /mnt/tank/truenas-truecloud-patch/apply.log
-```
-
-**Verify backend patch is loaded** (while middlewared is running):
-```bash
-python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
-```
-Reads `hook_status.json` written by `apply.sh` at boot **and** checks that the
-running middlewared process started *after* the patches were applied — an
-on-disk patch that middlewared has not loaded yet is reported as FAIL with
-instructions. Does not require `--host` or `--api-key`.
-
-**Middlewared log:**
-```bash
-grep truecloud-patch /var/log/middlewared.log 2>/dev/null | tail -20
-journalctl -u middlewared -n 100 2>/dev/null | grep truecloud-patch
-```
-
-**Verify the UI patch** (should print your TrueNAS version):
-```bash
-grep -c 'STORJ_IX.*S3.*B2' \
-    $(find /usr/share/truenas -name '*.js' 2>/dev/null) 2>/dev/null \
-    | grep -v ':0'
-```
-
-**`create_task.py` SSL error connecting to TrueNAS**
-`create_task.py` talks to the **TrueNAS API**, not your S3 endpoint, and
-verifies its TLS certificate. If your NAS uses a self-signed certificate,
-pass `--insecure` — but be aware this disables certificate verification for
-the API call that transmits your TrueNAS API key. Adding your NAS certificate
-to your system's trust store is safer.
+Parts of this project were written with AI assistance (Claude); all of it is reviewed
+and tested before release. Bugs are mine.

docs/cli.md

@@ -0,0 +1,45 @@
+# Creating a task from the CLI
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## Creating a task via CLI
+
+If the UI still shows only Storj after refreshing (e.g. the JS bundle pattern
+changed in a new TrueNAS version), create tasks directly. Run this **on the
+TrueNAS host** — it talks to the local middleware via `midclt`, so it needs no
+host address or API key:
+
+```bash
+# Replace /mnt/tank/truenas-truecloud-patch with your clone path
+
+# List your cloud credentials to find the right ID
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py list-credentials
+
+# Create a task with a B2 credential (id=3).
+# The restic repo password is read from stdin, so it never lands in your shell
+# history — nor in any process's argv, where `ps` would expose it.
+printf '%s' 'restic-repo-password' | \
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py create \
+    --name "tank-to-b2" \
+    --path /mnt/tank/data \
+    --credential 3 \
+    --bucket my-bucket \
+    --folder backups/tank \
+    --password-stdin \
+    --cache-path /mnt/tank/.restic-cache \
+    --keep-last 14
+```
+
+Omit `--password-stdin` and you'll be prompted for the password instead. `--password
+<secret>` still works but warns: that password is the encryption key for the whole
+repository, and a CLI argument persists in your shell history forever.
+
+> **Always pass `--cache-path`.** Without it TrueNAS runs restic with `--no-cache`,
+> which re-fetches all repo metadata from the provider every run — glacially slow
+> on large repos. Point it at a writable dir on a pool with free space.
+
+> Versions ≤ 0.1.0 used the `/api/v2.0` REST API with `--host`/`--api-key`; those
+> flags are now accepted-but-ignored (REST is removed in TrueNAS 26.04).
+
+---
+

docs/how-it-works.md

@@ -0,0 +1,209 @@
+# How it works
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## What is actually patched
+
+**Nothing in TrueNAS's persistent database or configuration is modified**
+(other than the boot-hook entry itself). On every boot, `patch/apply.sh` runs
+as a PREINIT script. It mounts a writable
+[overlayfs](https://docs.kernel.org/filesystems/overlayfs.html) over the
+relevant directories in `/usr/` (upper layer in `/run` tmpfs), then patches
+`b2.py` and `restic.py` inside that overlay. The overlay is volatile — it
+exists only for the current boot — but the PREINIT script recreates it
+automatically on every subsequent boot. Nothing in `/usr/` is written to
+directly.
+
+PREINIT scripts are executed *by* middlewared, which by then has already
+imported the stock modules — so after patching, `apply.sh` schedules a single
+detached middlewared restart (transient systemd unit `truecloud-mw-restart`
+running `patch/wait_restart.sh`) that loads the patched modules once boot has
+*actually* settled: the script waits for the systemd boot job queue to drain
+and for the docker/apps state machine to reach a terminal state before
+restarting. Expect one middlewared restart shortly after every boot; the UI
+and API are briefly unavailable while it happens, and running services are
+not affected.
+
+| Module | What changes | Technique |
+|---|---|---|
+| **providers** | `B2RcloneRemote` gains `get_restic_config()` — skipped automatically if TrueNAS already provides one on the class. `restic.py` URL builder is fixed: strips the stray leading slash and converts the slash separator to a colon (`b2:bucket:path`), which is the format restic 0.16.x expects. URL wrapper is a no-op if the URL is already correctly formed. | File patch applied inside the overlayfs upper layer |
+| **providers** (UI) | The Angular bundle's `filterByProviders` binding is widened from `["STORJ_IX"]` to `["STORJ_IX","S3","B2"]` | In-place text replacement in the compiled JS chunk; original is backed up before patching |
+| **nested** (opt-in) | `_truecloud_nested.py` is installed into `plugins/cloud/`, and `plugins/cloud/{snapshot,crud}.py` + `plugins/cloud_backup/sync.py` are patched so `snapshot = true` works on a dataset that has child datasets. See [below](nested-snapshots.md). | New module + file patches inside the overlayfs upper layer |
+
+All changes are **fail-safe**: if a patch cannot be applied (e.g. TrueNAS
+restructured the relevant code), middlewared starts normally, the affected module
+is simply inactive, and the reason is logged to `apply.log` in your repo root. The
+two modules are independent — one failing or going native does not disable the
+other.
+
+
+## How persistence works
+
+Two different things must survive two different events:
+
+| Event | What would be lost | What makes it survive |
+|---|---|---|
+| **Reboot** | The overlay holding the patched files lives in `/run` (tmpfs) and vanishes | The PREINIT hook re-runs `apply.sh` on every boot and schedules one middlewared restart to load the result |
+| **TrueNAS update** | `/usr/` is replaced entirely; custom files in `/etc/` are wiped with the new boot environment | This repo lives on your **data pool**, and the hook registration lives in the **TrueNAS config database** — both survive updates. The first boot after an update is just a normal boot |
+
+### What happens on every boot
+
+1. **middlewared starts** with the stock (unpatched) modules. This is
+   unavoidable: PREINIT scripts are executed *by* middlewared
+   (`ix-preinit.service` → `midclt call initshutdownscript.execute_init_tasks`),
+   so nothing registered there can run before it.
+2. **Pools import** (`ix-zfs.service`), making `/mnt/<pool>` — and this
+   repository — available.
+3. **`apply.sh` runs** (`ix-preinit.service`). Before it patches anything it runs
+   the **compatibility preflight** ([`tools/compat.py`](../tools/compat.py)) against
+   the middlewared that is *actually installed*, and **any module whose assumptions
+   no longer hold is not applied** — see [TrueNAS
+   compatibility](../README.md#truenas-compatibility). What survives that check gets applied:
+   it mounts the writable overlay (upper layer in `/run`), patches `b2.py` and
+   `restic.py` on disk inside it, patches the UI bundle, and writes `apply.log` and
+   `hook_status.json`.
+
+   An incompatible module is skipped **for this boot only**. It is not the kill
+   switch: install a release that supports your TrueNAS and the patch re-applies
+   itself on the next boot, with no manual step. (The kill switch is permanent and
+   is set only when TrueNAS has made the patch *unnecessary* — a different
+   situation, and the opposite conclusion.)
+4. **A deferred restart is scheduled.** The middlewared that is running
+   imported the stock modules in step 1 and never re-imports, so the on-disk
+   patch alone is not enough. `apply.sh` detects it was invoked by middlewared
+   and creates a transient systemd unit (`truecloud-mw-restart`, via
+   `systemd-run --no-block`) running `patch/wait_restart.sh` — detached so it
+   cannot disrupt the remainder of the boot sequence.
+5. **Once boot has settled, middlewared restarts once** and imports the
+   patched modules from the overlay. `wait_restart.sh` holds the restart until
+   the systemd boot job queue has drained (so in-flight `ix-*` units like
+   `ix-reporting` finish first) *and* middlewared's docker/apps startup has
+   reached a terminal state — plain unit ordering cannot see either, and
+   restarting middlewared while they run kills apps and dashboard reporting
+   for the whole boot. S3/B2 backup support is then active until the next
+   reboot, when the cycle repeats.
+
+What you will observe: one middlewared restart shortly after every boot (a
+brief web UI/API blip; running services are unaffected). Between steps 3
+and 5 there is a short window — typically well under a minute — where the UI
+already shows S3/B2 (the JS bundle is read from disk per request) but the
+backend is still stock. A backup job that fires inside that window fails once
+with `NotImplementedError` and succeeds on its next run; see
+[Troubleshooting](recovery.md) if it persists beyond boot.
+
+Manual runs of `bash patch/apply.sh` never trigger the restart — that only
+happens in boot context. `install.sh` and `recover.sh` perform their own
+explicit restarts instead, which is why a manual re-apply must be followed by
+`systemctl restart middlewared`.
+
+---
+
+
+## If TrueNAS adds native support
+
+The patch is **two independent modules**, and each retires on its own — TrueNAS
+is likely to ship one of these natively long before the other, and a module
+going native must not take the other one down with it.
+
+| Module | What it does | Detected as native when |
+|---|---|---|
+| **providers** | B2/S3 credentials for TrueCloud Backup (`b2.py`, `restic.py`, UI dropdown) | `B2RcloneRemote` carries a real `get_restic_config()` |
+| **nested** | Snapshots on datasets with child datasets (`plugins/cloud/*`) | the *"no further nesting"* validation is gone from `plugins/cloud/crud.py` |
+
+At every boot `apply.sh` checks both:
+
+- **One module goes native** → that module is skipped and logged; the other keeps
+  working, and the patch stays installed.
+- **Both are done** (native, or nested was never enabled) → the kill switch
+  (`disabled` file) is set, overlays are unmounted, and `apply.log` tells you to
+  run `uninstall.sh`.
+
+So on a box using only the provider patch, native B2 support retires the whole
+thing as before. On a box that also uses nested snapshots, native B2 support
+retires *just* that half.
+
+Check the log after any TrueNAS update:
+```bash
+tail -20 /mnt/tank/truenas-truecloud-patch/apply.log
+```
+
+`hook_status.json` reports each module separately (`module.providers`,
+`module.nested_snapshots`) with an `active` flag and a reason.
+
+**Scenarios where the auto-detect may not fire** (manual check needed):
+
+| Scenario | What happens | Action |
+|---|---|---|
+| B2 support added to a **base class** (not `B2RcloneRemote` directly) | `__dict__` check misses it; our method shadows native | Uninstall manually |
+| B2 **credential schema changed** (e.g. `provider["account"]` renamed) | `KeyError` on first backup | Uninstall or update the patch |
+| **URL builder** fixed but B2 class unchanged | URL wrapper becomes a no-op; no harm, but patch is dead weight | Uninstall at your convenience |
+
+---
+
+## After a TrueNAS update
+
+A TrueNAS update replaces `/usr/` wholesale, wiping the patch. You do **not** need
+to reinstall: `patch/apply.sh` runs at every boot and re-applies itself from your
+clone. But it targets internal APIs with no stability contract, so an update *can*
+break it — and the failure is quiet by design (middlewared starts fine; the patch
+just doesn't).
+
+**Check the log after any TrueNAS update:**
+
+```bash
+tail -30 /mnt/tank/truenas-truecloud-patch/apply.log
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
+```
+
+| What you see | What it means |
+|---|---|
+| `[OK] providers`, `[OK]`/`[SKIP] nested_snapshots` | Fine. Nothing to do. |
+| `WARNING: … pattern not found` (UI) | The Angular bundle changed. The UI dropdown reverts to Storj-only, but **backups keep working** — create tasks with `create_task.py` meanwhile, and [open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues) with your TrueNAS version. |
+| `WARNING: truecloud-patch is NOT COMPATIBLE with this TrueNAS version` | This TrueNAS changed middleware underneath the patch, and the named module was **deliberately not applied** — see `incompatible.json` for exactly which assumption broke. TrueNAS is left stock, so nothing is half-patched. Check [TrueNAS compatibility](../README.md#truenas-compatibility), then `bash update.sh` once a release supports your version; it re-applies itself on the next boot. This is **not** the kill switch and needs no manual reset. |
+| `[FAIL] providers` | **Your B2/S3 backups will not run.** middlewared is fine, but the credential/URL handling is gone. Open an issue with your version. |
+| `[FAIL] nested_snapshots` | The stock guard is back, so tasks with `snapshot = true` on a nested dataset will fail validation. Turn the option off on those tasks until it's fixed. |
+
+"Fail-safe" means *the box stays up* — not that your backups keep running. A
+`[FAIL] providers` is a broken backup, so check the log rather than assume.
+
+Then update the patch itself if a newer release fixes it:
+
+```bash
+bash /mnt/tank/truenas-truecloud-patch/update.sh
+```
+
+---
+
+
+
+## TrueNAS 26
+
+TrueNAS 26 changes three things underneath the nested module. **Each one alone is
+backup-breaking**, and none of them is visible from the `cloud_backup` files:
+
+| what changed | what it would do |
+| --- | --- |
+| `cloud_backup` rewritten **async → synchronous** | an `async def` wrapper hands `sync.py` a coroutine where it unpacks a tuple |
+| `get_dataset_recursive()` **deleted** from `plugins/cloud/snapshot.py` | `NameError` — the injected block called it out of the host module's namespace |
+| `plugins/zfs_/dataset.py` and `zfs_/snapshot.py` **deleted** | `zfs.dataset.query`, `zfs.snapshot.query` and `zfs.snapshot.delete` all vanish. 26 uses `filesystem.statfs` and `zfs.resource.*` |
+
+The first two are fixed: the patch reads which flavour of `cloud_backup` your box
+declares and injects the wrapper that matches (one implementation of the real logic,
+two thin wrappers), and it carries its own copy of the deleted helper.
+
+The third is **not** fixed, and is why 26 reports BROKEN. Porting it means rewriting
+the module's ZFS calls onto 26's new API, and no single API spans 24.10 through 26 —
+so it needs a real 26 box to verify against, not a plausible-looking diff. Shipping a
+port nobody has run is exactly the failure this project exists to avoid.
+
+It is also the row that would have hurt most. `zfs.snapshot.delete` is what sweeps the
+recursive snapshot; without it, **every run would orphan one snapshot per descendant
+dataset — 250 on a real pool — forever.** The compatibility check caught it only
+because it now asserts the middleware *methods the patch calls*, not just the symbols
+it wraps.
+
+`master` (development after 26) reports BROKEN too: iXsystems are still reshaping
+these functions there, renaming `middleware` → `context` and `cloud_backup` → `entry`
+and adding a required `credentials` parameter. That is a moving target and is
+deliberately not chased; the check keeps reporting it until it settles into a beta,
+which is when it becomes worth fixing.

docs/nested-snapshots.md

@@ -0,0 +1,160 @@
+# Nested-dataset snapshots
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## Nested-dataset snapshots
+
+**Opt-in, off by default.** It changes how backups read their source data, so it
+is never enabled implicitly:
+
+```bash
+bash install.sh --enable-nested-snapshots
+bash install.sh --disable-nested-snapshots
+```
+
+With neither flag `install.sh` leaves the setting alone, so `git pull && bash
+install.sh` won't flip it. The providers module is unaffected either way.
+
+Validated end to end on a live 252-dataset pool: an unattended scheduled backup
+of `/mnt/Tap` built a 173-mount staging tree, completed in **18m14s**, and left
+**zero** orphaned snapshots and **zero** stale mounts behind. The same backup
+previously stalled at 74% for over 12 hours reading live files.
+
+Still: verify your own first run actually contains child-dataset data before you
+rely on it — see [Verifying it works](#verifying-it-works). That advice is not
+boilerplate; it is the specific thing this feature exists to make true.
+
+TrueCloud Backup's **Take Snapshot** option makes restic read from a frozen ZFS
+snapshot instead of live files. Without it the backup reads data *while apps are
+writing to it* — databases get captured mid-write, and an app that rewrites its
+files continuously can stall a backup indefinitely as restic chases a moving
+target.
+
+Stock TrueNAS refuses to enable it on most real-world paths:
+
+```
+[EINVAL] cloud_backup_update.snapshot:
+  This option is only available for datasets that have no further nesting
+```
+
+That rules out **any pool running Apps** — every app is its own dataset, usually
+with `config`/`pgdata` children of its own. On a typical box that is 100+ nested
+datasets, so the feature is effectively unusable exactly where it matters most.
+
+### Why stock refuses
+
+The guard is **correct**. `plugins/cloud/snapshot.py` already takes a *recursive*
+ZFS snapshot — but it then points restic at the **parent** dataset's
+`.zfs/snapshot/<snap>/` directory, and ZFS does not expose child datasets
+through a parent's snapshot directory:
+
+```
+/mnt/Tap/.zfs/snapshot/<snap>/apps/                 ->  0 entries  (children invisible)
+/mnt/Tap/apps/lidarr/config/.zfs/snapshot/<snap>/   ->  the real data
+```
+
+So if you just remove the validation, restic walks a near-empty tree, reports
+SUCCESS, and uploads almost nothing — a green backup job protecting no data. iX
+gate the config rather than ship a backup that lies about succeeding.
+
+That is worth spelling out, because deleting those four lines in
+`plugins/cloud/crud.py` is the obvious "fix" and it is the wrong one. The guard
+is load-bearing: it has to be *replaced* with a working traversal, not removed.
+
+### What this patch does instead
+
+After the (already recursive) snapshot is taken, every descendant dataset's own
+`.zfs/snapshot/<snap>` is bind-mounted into a **staging tree** that mirrors the
+original layout, and restic is pointed at the staging root — a complete,
+consistent, point-in-time view of the whole subtree. Only then is the guard
+relaxed.
+
+Safety properties, in order of importance:
+
+- **Staging failure is loud.** If any descendant cannot be staged, the backup
+  *fails*. A silently-incomplete backup is precisely what the stock guard exists
+  to prevent, and it would be worse than not having the feature at all.
+- **Post-mount verification** asserts every planned target really is a mountpoint
+  and the staging root is non-empty — so this cannot regress into the empty
+  backup it exists to fix.
+- **The guard is relaxed last.** `apply.sh` installs the traversal, patches
+  `snapshot.py`, then `sync.py`, and only then `crud.py`. Any partial failure
+  leaves the guard intact and the option merely unavailable — never
+  "guard removed, traversal missing".
+- Datasets that cannot contribute to a file tree (`mountpoint=none|legacy`,
+  unmounted, locked/encrypted) are skipped and **reported to the log** — never
+  dropped silently.
+- Scoped to **cloud_backup only**. Cloud Sync (rclone) shares the same
+  validation mixin but has no staging teardown wired in, so its guard is
+  deliberately left in place.
+
+Side benefit: the staging root is a **stable path per task**, so restic can find
+its parent snapshot between runs. Stock's `.zfs/snapshot/<name>-<timestamp>/`
+path changes every run, defeating restic's parent detection and forcing a full
+re-scan each time.
+
+### Snapshot lifecycle
+
+`zfs.snapshot.delete` defaults to **`recursive=False`**, and stock
+`restic_backup()` calls it with no options. Stock is safe only because its
+validation means a *recursive* snapshot never actually happens in the field.
+Enabling nested datasets makes them real: on a 250-dataset pool,
+`zfs snapshot -r` creates **250 snapshots**, and stock's delete removes only the
+parent — orphaning **249 on every successful run** (measured, not theorised).
+
+So the patch owns the whole lifecycle:
+
+- **Sweeps the parent and every child**, and is idempotent against stock's
+  `finally` winning the race once the mounts are released.
+- **Records the snapshot in a sidecar file before mounting anything**, so a
+  middlewared restart mid-backup cannot orphan the tree (this patch *schedules*
+  a restart at boot, so that is not hypothetical).
+- **Reclaims the tree left by a crashed run** instead of overwriting the record.
+- **Deletes the tree when staging fails** — sync.py's own `finally` deletes
+  *nothing* in that case, because its `snapshot` local never gets assigned.
+- **Enumerates datasets *after* the snapshot, never before.** A list read
+  beforehand can miss a dataset created in the gap, which the recursive snapshot
+  *would* capture but the staging plan would not — a silent omission.
+
+**Expected log noise:** stock's delete fails with `EBUSY` while the staging
+mounts pin the snapshot. You will see one benign `Error deleting snapshot ...`
+warning per run; the patch then unmounts and deletes the tree for real.
+
+### Verifying it works
+
+This feature exists because a backup can report SUCCESS while containing
+nothing, so check the contents rather than the exit status:
+
+```bash
+# 1. Does the restic snapshot actually contain child-dataset data?
+#    Pick a path that lives in a CHILD dataset (e.g. an app's config).
+midclt call cloud_backup.list_snapshots <task_id> | head
+
+# 2. List a child-dataset path inside the newest restic snapshot.
+#    If this is empty, the staging tree did not work and you are backing up NOTHING.
+midclt call cloud_backup.list_snapshot_directory <task_id> "<snapshot_id>" "/apps/lidarr/config"
+```
+
+You should see the app's real files (`lidarr.db`, `config.xml`, …). An empty
+listing means the child datasets were not staged; disable the feature and open an
+issue.
+
+```bash
+# 3. No snapshots may be left behind after a run.
+zfs list -t snapshot -r <pool> | grep -c cloud_backup-   # expect 0 between runs
+
+# 4. No staging mounts may be left behind.
+mount | grep truecloud-nested                            # expect no output
+```
+
+### Troubleshooting
+
+| Symptom | Cause |
+|---|---|
+| `This option is only available for datasets that have no further nesting` | Feature not enabled. Run `install.sh --enable-nested-snapshots`, then restart middlewared. |
+| Backup fails: `dataset '…' has no snapshot '…'; refusing to back up an incomplete tree` | Working as designed — a descendant dataset was not covered by the snapshot. The backup is refused rather than silently omitting that data. |
+| Backup fails: `snapshot '…' cannot be read (Permission denied)` | The snapshot exists but is unreadable. Middleware runs as root, so this indicates a real permissions problem, not a missing snapshot. |
+| `cloud_backup-*` snapshots accumulating | The sweep is not running. Check `apply.log` for the nested patch applying, and confirm `sync.py` carries the `TRUECLOUD_PATCH` block. |
+| Web UI blank after a patch | A bad pattern unbalanced the bundle. `apply.sh` now refuses to write in that case, but if you hit it on an older version: restore `chunk-*.js.pre-truecloud-patch` over the live chunk, then re-run `install.sh`. (`MARKER` makes an already-patched file skip, so the patch cannot heal a corrupted bundle by itself.) |
+| Stale mounts under `/run/truecloud-nested` | A crashed run. The next backup tears them down. To clear them now: `python3 patch/truecloud_nested.py cleanup` (also run by `uninstall.sh` and `recover.sh`). It names any ZFS snapshot an interrupted run left pinned. |
+

docs/recovery.md

@@ -0,0 +1,179 @@
+# Recovery and troubleshooting
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## Emergency recovery
+
+### middlewared won't start
+
+Run this from the TrueNAS shell (local console, SSH, or the debug shell in
+the UI):
+
+```bash
+bash /mnt/tank/truenas-truecloud-patch/recover.sh
+```
+
+Replace the path with your clone location. This creates a kill-switch file
+(`disabled`) in the repo root, unmounts the overlay so the original files are
+visible immediately, then restarts middlewared. No reboot required.
+
+If you cannot run a script and only have a bare shell prompt:
+
+```bash
+touch /mnt/tank/truenas-truecloud-patch/disabled
+systemctl restart middlewared
+```
+
+If you don't remember where you cloned the repo (midclt won't work while middlewared is
+down), find the path two ways:
+
+```bash
+# Option 1 — search the filesystem:
+find /mnt -name "recover.sh" -path "*/truenas-truecloud-patch/*" 2>/dev/null
+
+# Option 2 — query the TrueNAS database directly:
+sqlite3 /data/freenas-v1.db \
+    "SELECT script FROM initshutdownscript WHERE comment = 'TrueCloud provider patch (S3/B2)';"
+```
+
+The `script` column shows the full path to `patch/apply.sh`; your clone root is one
+level up (strip `/patch/apply.sh` from the end). Then run the `touch` command above
+with that path.
+
+If middlewared **still** won't start after the kill switch is set, the problem
+is unrelated to this patch. Check:
+
+```bash
+journalctl -u middlewared -n 50
+```
+
+To re-enable the patch once you have investigated:
+
+```bash
+rm /mnt/tank/truenas-truecloud-patch/disabled
+bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh
+systemctl restart middlewared   # manual apply.sh runs never restart for you
+```
+
+---
+
+### Web UI is blank or broken
+
+If the TrueNAS web interface loads blank or shows JavaScript errors, the
+Angular bundle may have been interrupted mid-write (e.g. power cut during
+boot). The original bundle is always backed up before patching, so recovery
+is straightforward:
+
+```bash
+# Find the backup (the path varies by TrueNAS version):
+find /usr/share/truenas /usr/share/truenas-ui /var/www/truenas -name "*.js.pre-truecloud-patch" 2>/dev/null
+
+# Restore it — substitute the actual path from the find output:
+mv /usr/share/truenas/webui/main.XXXXXXXX.js.pre-truecloud-patch \
+   /usr/share/truenas/webui/main.XXXXXXXX.js
+```
+
+Refresh your browser. The UI will return to normal (Storj-only until the
+patch re-runs at next reboot, or you run
+`bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh` manually).
+
+---
+
+### Backend verify shows FAIL
+
+```bash
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
+```
+
+`verify` reports one line per module:
+
+| Label | Meaning |
+|---|---|
+| `[OK  ]` | Module is active and applied. |
+| `[SKIP]` | Module is inactive — either TrueNAS now does it natively, or it is opt-in and switched off. **Not a failure.** `nested_snapshots` shows SKIP on a default install. |
+| `[FAIL]` | Module is needed but did not apply. |
+
+If a module shows `[FAIL]`:
+
+1. **Check the apply log** for errors during the last boot:
+   ```bash
+   tail -40 /mnt/tank/truenas-truecloud-patch/apply.log
+   ```
+2. **Check middlewared's own log** for Python tracebacks:
+   ```bash
+   grep -i "truecloud\|traceback\|error" /var/log/middlewared.log 2>/dev/null | tail -30
+   journalctl -u middlewared -n 50
+   ```
+3. **A FAIL is non-fatal.** middlewared runs normally and the other module is
+   unaffected; the failed one is simply inactive. Existing backups are not at
+   risk.
+4. **If the detail says the module doesn't exist**, a TrueNAS update renamed
+   or restructured the internal API.
+   [Open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues)
+   with your TrueNAS version number and the full verify output.
+
+---
+
+## Troubleshooting
+
+**Backups fail with `NotImplementedError` after a reboot**
+
+The traceback ends in `rclone/base.py` → `raise NotImplementedError` and
+contains no `_tc_` frames: the running middlewared is executing stock code.
+Either the deferred restart never fired, or the patch never landed on disk
+this boot. Diagnose in this order:
+
+```bash
+# Did apply.sh run this boot, at which version, and did it schedule the restart?
+tail -40 /mnt/tank/truenas-truecloud-patch/apply.log
+
+# Full check — compares the running process against the patch timestamp
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
+
+# Did the deferred restart unit run, fail, or never get created?
+systemctl status truecloud-mw-restart.service
+journalctl -u truecloud-mw-restart.service --no-pager | tail -20
+```
+
+- `verify` reports the process started **before** the patch → the restart
+  didn't happen. `systemctl restart middlewared` fixes it immediately; the
+  journal output above tells you why it was missed.
+- `apply.log` shows the kill switch is active → `rm .../disabled`, then
+  `bash install.sh`.
+- `apply.log` has no entry for this boot → the hook didn't run; re-run
+  `bash install.sh` to re-register it.
+- `apply.log` header shows `[v0.0.3]` or older → update:
+  `git pull && bash install.sh` (v0.0.4 fixed patches not loading after
+  reboot).
+
+**Apply log** (check after each reboot or install):
+```bash
+cat /mnt/tank/truenas-truecloud-patch/apply.log
+```
+
+**Verify backend patch is loaded** (while middlewared is running):
+```bash
+python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
+```
+Reads `hook_status.json` written by `apply.sh` at boot **and** checks that the
+running middlewared process started *after* the patches were applied — an
+on-disk patch that middlewared has not loaded yet is reported as FAIL with
+instructions. Does not require `--host` or `--api-key`.
+
+**Middlewared log:**
+```bash
+grep truecloud-patch /var/log/middlewared.log 2>/dev/null | tail -20
+journalctl -u middlewared -n 100 2>/dev/null | grep truecloud-patch
+```
+
+**Verify the UI patch** (should print your TrueNAS version):
+```bash
+grep -c 'STORJ_IX.*S3.*B2' \
+    $(find /usr/share/truenas -name '*.js' 2>/dev/null) 2>/dev/null \
+    | grep -v ':0'
+```
+
+**`create_task.py` — "midclt not found" or permission errors**
+`create_task.py` now talks to the local middleware via `midclt`, so run it **on
+the TrueNAS host** (not remotely) as a user with middleware access (root). There
+is no HTTPS/API-key call anymore, so there is no TLS certificate to configure.

docs/releasing.md

@@ -0,0 +1,98 @@
+# Development and releasing
+
+> Part of [truenas-truecloud-patch](../README.md).
+
+## Development
+
+Parts of this project were written with AI assistance (Claude). All of it is
+reviewed and tested before release; the test suite and CI exist in large part to
+make that review meaningful. Bugs are mine.
+
+```bash
+pip install pytest ruff
+ruff check patch tests tools
+pytest tests
+```
+
+CI runs shellcheck, `bash -n`, ruff, and pytest on Python 3.11–3.13. Two checks
+are worth calling out, because nothing else would catch what they catch:
+
+- The tests **`compile()` the `*_BLOCK` strings** in `patch/apply.sh`. Those are
+  Python source appended into live `middlewared` modules — a syntax error there
+  breaks the box at boot, and they're string literals, so nothing else type-checks
+  them.
+- CI asserts **every script declares the same version**, and that it matches the
+  newest CHANGELOG entry. `VERSION=` had silently drifted to three different
+  values across the scripts before anything checked.
+
+The project is hosted on **Gitea** (`git.onetick.ninja/flan/truenas-truecloud-patch`)
+and mirrored to GitHub. Both run the same workflows — Gitea reads
+`.github/workflows/` too — so a change is checked twice, on two independent runners.
+
+---
+
+## Releasing
+
+**Every release interrupts every user.** An update alert fires on each installed
+box (see [Update alerts](../README.md#update-alerts)), so a release that exists only to fix the
+last release teaches people to dismiss the alert — and one day that alert will be
+carrying a security fix. This project cut twelve releases in a single day once.
+Never again, and not by good intentions: by a gate.
+
+### The rule
+
+> A stable `vX.Y.Z` may only be published if a `vX.Y.Z-rcN` tag points at the
+> **same commit**.
+
+Release candidates are **invisible to users**: `update.sh` and the update alert both
+take the newest plain `vX.Y.Z` tag, so an `-rc` is never offered as an update. All
+the debugging therefore happens across `rc1`, `rc2`, `rc3` — at nobody's expense —
+instead of across `v0.5.0`, `v0.5.1`, `v0.5.2`, at everybody's.
+
+"The candidate passed, then I pushed one more little fix" is refused **by name**.
+That is not hypothetical; it is exactly how v0.5.1 happened.
+
+### Day to day
+
+You don't touch the release machinery. Write your changes under `## Unreleased` in
+`CHANGELOG.md` and push to `main`. `main` is a work surface — it is allowed to be
+mid-thought. Releasing is a separate, deliberate act.
+
+### Cutting a release
+
+```bash
+bash release.sh 0.6.0 --check      # what would ship? what is the next rc?
+
+bash release.sh 0.6.0 --rc         # promotes `## Unreleased` -> v0.6.0, stamps every
+                                   # VERSION=, tags v0.6.0-rc1, pushes. Users see nothing.
+
+#   ... install it on a real box. Exercise it. Break it. ...
+#   Found a bug? Fix it on main, then `bash release.sh 0.6.0 --rc` again -> rc2.
+
+bash release.sh 0.6.0 --promote    # publishes v0.6.0. REFUSED unless an rc points here.
+```
+
+### The gates, and where they live
+
+The logic is Python so it can be unit-tested; CI is the enforcement boundary
+because it is the only actor holding the token that publishes. `release.sh` runs the
+**same** code locally so you fail in 200 ms instead of after a push.
+
+| gate | enforces | where |
+| --- | --- | --- |
+| [`release_notes.py check`](../tools/release_notes.py) | every script's `VERSION=` matches the tag; the CHANGELOG section exists and is non-empty; **nothing is stranded under `## Unreleased`** | `release.sh` + CI |
+| [`release_gate.py`](../tools/release_gate.py) | **an rc points at this exact commit** | `release.sh` + CI |
+| the full suite | ruff, pytest, shellcheck, `bash -n` — re-run against the *tagged* commit | CI |
+
+A release's body **is** its `CHANGELOG.md` section — there is no second place to
+write release notes, and therefore no second place for them to go stale. Releases
+are published on both forges.
+
+### Why the alert doesn't nag
+
+A release whose CHANGELOG contains only a `### Docs` section changed no code, and
+raises **no alert**. Candidates raise no alert either. So the only thing that ever
+interrupts a user is a real, complete change — which is the entire point.
+
+---
+

install.sh

@@ -18,11 +18,56 @@
 
 set -euo pipefail
 
-VERSION="0.0.4"
+VERSION="0.6.0"
 
 # The directory containing install.sh is the permanent install location.
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 _HOOK_COMMENT='TrueCloud provider patch (S3/B2)'
+_NESTED_MARKER="$PATCH_DIR/nested_snapshots_enabled"
+
+# ── Options ───────────────────────────────────────────────────────────────────
+# Nested-dataset snapshot support is OPT-IN and off by default. It changes how
+# backups read their source data, so an unattended re-run (e.g. after a
+# `git pull`) must never flip it on or off by itself: with neither flag given,
+# whatever was chosen previously is preserved.
+_nested_choice=""
+_alert_choice=""
+_ALERT_MARKER="$PATCH_DIR/update_alerts_disabled"
+
+usage() {
+    cat <<USAGE
+Usage: bash install.sh [options]
+
+Options:
+  --enable-nested-snapshots   Allow the "Take Snapshot" option on datasets that
+                              have child datasets (every pool running Apps).
+                              Stock TrueNAS refuses this; see README. Off by
+                              default because it changes how backups read data.
+  --disable-nested-snapshots  Turn it back off; the stock guard is restored.
+  --no-update-alerts          Do not raise a TrueNAS alert when an update exists.
+  --update-alerts             Re-enable those alerts (they are on by default).
+  -h, --help                  Show this help.
+
+With neither flag, the current setting is left unchanged.
+USAGE
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --enable-nested-snapshots)  _nested_choice="on" ;;
+        --disable-nested-snapshots) _nested_choice="off" ;;
+        --no-update-alerts)         _alert_choice="off" ;;
+        --update-alerts)            _alert_choice="on" ;;
+        -h|--help)                  usage; exit 0 ;;
+        *)
+            echo "ERROR: unknown option: $1" >&2
+            echo "" >&2
+            usage >&2
+            exit 1
+            ;;
+    esac
+    shift
+done
 
 if [ ! -f "$PATCH_DIR/patch/apply.sh" ]; then
     echo "ERROR: patch files not found at $PATCH_DIR/patch/" >&2
@@ -43,6 +88,44 @@ if [ "$(id -u)" -ne 0 ]; then
     exit 1
 fi
 
+# ── Minimum TrueNAS version ───────────────────────────────────────────────────
+#
+# TrueCloud Backup -- the feature this whole project extends -- was introduced in
+# 24.10. On anything older, `plugins/cloud_backup/` does not exist at all: there is
+# no restic, no cloud_backup task type, and nothing for the patch to attach to. It
+# would not break the box, it would simply do nothing, silently, while the user
+# believed their backups were configured. Say no clearly instead.
+#
+# A version we cannot PARSE is not a version we may refuse on: warn and continue.
+# Refusing to install over a string we failed to read would be a worse failure than
+# the one being prevented.
+MIN_TRUENAS="24.10"
+
+_tc_version_raw=""
+if command -v midclt &>/dev/null; then
+    _tc_version_raw=$(midclt call system.version 2>/dev/null || true)   # TrueNAS-25.10.4
+fi
+# Both spellings are real: modern releases report `TrueNAS-25.10.4`, older ones
+# `TrueNAS-SCALE-24.04.2` -- and the SCALE- form is used by exactly the versions
+# this gate exists to turn away, so failing to parse it would let them through.
+_tc_version=$(printf '%s' "$_tc_version_raw" \
+    | sed -n 's/^TrueNAS-\(SCALE-\)\{0,1\}\([0-9]\{1,\}\.[0-9]\{1,\}\).*/\2/p')
+
+if [ -z "$_tc_version" ]; then
+    echo "WARNING: could not determine the TrueNAS version" \
+         "${_tc_version_raw:+(got '${_tc_version_raw}')}."
+    echo "WARNING: this patch requires TrueNAS SCALE ${MIN_TRUENAS} or newer. Continuing anyway."
+    echo ""
+elif [ "$(printf '%s\n%s\n' "$MIN_TRUENAS" "$_tc_version" | sort -V | head -1)" != "$MIN_TRUENAS" ]; then
+    echo "ERROR: TrueNAS ${_tc_version} is too old — this patch requires ${MIN_TRUENAS} or newer." >&2
+    echo "" >&2
+    echo "  TrueCloud Backup does not exist before ${MIN_TRUENAS}, so there is nothing" >&2
+    echo "  here for the patch to extend. Upgrade TrueNAS first." >&2
+    exit 1
+else
+    echo "TrueNAS ${_tc_version} (minimum ${MIN_TRUENAS}) — ok"
+fi
+
 if ! command -v midclt &>/dev/null; then
     echo "ERROR: midclt not found. Run this script on TrueNAS SCALE." >&2
     exit 1
@@ -56,8 +139,14 @@ fi
 # ── Set permissions ───────────────────────────────────────────────────────────
 
 echo "Setting permissions ..."
-chmod +x "$PATCH_DIR/patch/apply.sh" "$PATCH_DIR/patch/create_task.py" \
-          "$PATCH_DIR/recover.sh" "$PATCH_DIR/uninstall.sh"
+# Guard each path: under `set -e` a chmod on a missing file aborts the install.
+# The file set changes between versions, so `update.sh --rollback` to an older
+# revision must not be killed by a name this version happens to know about.
+for _exe in patch/apply.sh patch/create_task.py recover.sh uninstall.sh update.sh; do
+    if [ -f "$PATCH_DIR/$_exe" ]; then
+        chmod +x "$PATCH_DIR/$_exe"
+    fi
+done
 echo "Done."
 echo ""
 
@@ -104,6 +193,77 @@ if [ -f "$PATCH_DIR/disabled" ]; then
     echo ""
 fi
 
+# ── Nested-dataset snapshot support (opt-in) ──────────────────────────────────
+
+case "$_nested_choice" in
+    on)
+        touch "$_NESTED_MARKER"
+        echo "Nested-dataset snapshots: ENABLED"
+        echo "  The \"Take Snapshot\" option will be allowed on datasets that have"
+        echo "  child datasets. Backups then read from a frozen, complete staging"
+        echo "  tree instead of live files."
+        echo ""
+        echo "  This changes how your backups read their source data. Verify that a"
+        echo "  backup completes AND that its restic snapshot actually contains"
+        echo "  child-dataset data before you rely on it."
+        ;;
+    off)
+        if [ -f "$_NESTED_MARKER" ]; then
+            rm -f "$_NESTED_MARKER"
+            # Tear down any staging tree first: those bind mounts PIN their ZFS
+            # snapshots, so leaving them would block those snapshots from ever
+            # being destroyed. apply.sh (below) then reverts the patched files.
+            python3 "$PATCH_DIR/patch/truecloud_nested.py" cleanup || \
+                echo "  WARNING: staging mounts remain; unmount them manually."
+            echo "Nested-dataset snapshots: DISABLED."
+            echo "  apply.sh will revert the patched middleware files and the stock"
+            echo "  guard is restored when middlewared restarts (this script does that)."
+            echo "  Any task that already has snapshot=true on a nested dataset will"
+            echo "  fail validation on its next edit. Turn the option off on those"
+            echo "  tasks first, or re-run with --enable-nested-snapshots."
+        else
+            echo "Nested-dataset snapshots: already disabled."
+        fi
+        ;;
+    *)
+        if [ -f "$_NESTED_MARKER" ]; then
+            echo "Nested-dataset snapshots: enabled (unchanged)."
+        else
+            echo "Nested-dataset snapshots: disabled (default)."
+            echo "  Enable with: bash install.sh --enable-nested-snapshots"
+        fi
+        ;;
+esac
+echo ""
+
+# ── Update alerts (on by default) ─────────────────────────────────────────────
+# TrueNAS cannot raise an alert from the CLI (midclt exposes only dismiss/list/
+# restore), so this installs an AlertSource into middlewared/alert/source/ — the
+# same mechanism every built-in TrueNAS alert uses. It ADDS a file and modifies
+# none, which makes it the least invasive thing this patch does.
+#
+# It only alerts for releases that changed something: a documentation-only release
+# raises nothing.
+
+case "$_alert_choice" in
+    off)
+        touch "$_ALERT_MARKER"
+        echo "Update alerts: DISABLED (apply.sh will remove the alert source)."
+        ;;
+    on)
+        rm -f "$_ALERT_MARKER"
+        echo "Update alerts: enabled."
+        ;;
+    *)
+        if [ -f "$_ALERT_MARKER" ]; then
+            echo "Update alerts: disabled (unchanged)."
+        else
+            echo "Update alerts: enabled (checks daily; docs-only releases are ignored)."
+        fi
+        ;;
+esac
+echo ""
+
 # ── Apply now ─────────────────────────────────────────────────────────────────
 
 echo "Applying patches ..."

patch/alert_source.py

@@ -0,0 +1,258 @@
+"""TrueNAS alert: a truecloud-patch update is available.
+
+Installed by patch/apply.sh into middlewared/alert/source/, where middlewared
+discovers and polls it natively — no cron job, no systemd timer.
+
+@PATCH_DIR@ is substituted at install time.
+
+Two rules govern this file:
+
+1. **It must never break middlewared.** It runs inside the alert framework on a
+   timer. Every failure path returns None (no alert) rather than raising.
+
+2. **It must not nag.** A release whose CHANGELOG only has a "### Docs" section
+   changed no code, and nobody wants an alert because a README was reworded. The
+   CHANGELOG's own section headings are the signal — see tools/release_notes.py.
+
+It also never writes to the repository. `git ls-remote` is read-only and the
+CHANGELOG is fetched over HTTPS, so this cannot leave root-owned objects in .git
+the way a `git fetch` from middlewared (running as root) would.
+"""
+
+import datetime
+import importlib.util
+import logging
+import os
+import re
+import subprocess
+import urllib.request
+
+from middlewared.alert.base import (
+    Alert,
+    AlertCategory,
+    AlertClass,
+    AlertLevel,
+    ThreadedAlertSource,
+)
+from middlewared.alert.schedule import IntervalSchedule
+
+logger = logging.getLogger(__name__)
+
+PATCH_DIR = "@PATCH_DIR@"
+DISABLED_MARKER = os.path.join(PATCH_DIR, "update_alerts_disabled")
+
+_TAG_RE = re.compile(r"^v\d+\.\d+\.\d+$")
+_VERSION_RE = re.compile(r'^VERSION="([^"]+)"', re.M)
+
+#: owner/repo out of any of:
+#:   git@github.com:sudolulo/repo.git
+#:   https://github.com/sudolulo/repo.git
+#:   ssh://git@git.onetick.ninja:55214/flan/repo.git
+#:   https://git.onetick.ninja/flan/repo.git
+#: The SSH port is deliberately not captured: it is not the web port.
+_REMOTE_RE = re.compile(
+    r"^(?:\w+://)?(?:[^@/]+@)?([^:/]+)(?::\d+)?[:/]([^/]+)/([^/]+?)(?:\.git)?/?$"
+)
+
+_TIMEOUT = 20
+
+
+class TrueCloudPatchUpdateAlertClass(AlertClass):
+    category = AlertCategory.SYSTEM
+    level = AlertLevel.INFO
+    title = "truecloud-patch update available"
+    text = (
+        "truecloud-patch %(current)s is installed; %(latest)s is available.%(summary)s "
+        "Update with:  bash %(dir)s/update.sh"
+    )
+
+
+class TrueCloudPatchSecurityUpdateAlertClass(AlertClass):
+    category = AlertCategory.SYSTEM
+    level = AlertLevel.WARNING
+    title = "truecloud-patch security update available"
+    text = (
+        "truecloud-patch %(current)s is installed; %(latest)s contains a SECURITY "
+        "fix.%(summary)s Update with:  bash %(dir)s/update.sh"
+    )
+
+
+class TrueCloudPatchUpdateAlertSource(ThreadedAlertSource):
+    schedule = IntervalSchedule(datetime.timedelta(hours=24))
+    run_on_backup_node = False
+
+    def check_sync(self):
+        try:
+            return self._check()
+        except Exception:
+            # An alert source must never take middlewared down with it.
+            logger.debug("truecloud-patch update check failed", exc_info=True)
+            return None
+
+    # ── internals ────────────────────────────────────────────────────────────
+
+    def _git(self, *args):
+        # List form, never shell=True, and every `args` value is a literal from
+        # this file -- nothing user-supplied reaches the command line. The partial
+        # `git` path is moot: this runs as root inside middlewared, so anyone who
+        # can poison PATH already has root.
+        return subprocess.run(  # noqa: S603
+            ["git", "-C", PATCH_DIR, *args],  # noqa: S607
+            capture_output=True, text=True, timeout=_TIMEOUT, check=True,
+        ).stdout
+
+    def _check(self):
+        if os.path.exists(DISABLED_MARKER):
+            return None
+        if not os.path.isdir(os.path.join(PATCH_DIR, ".git")):
+            return None
+
+        current = self._installed_version()
+        if not current:
+            return None
+
+        latest = self._latest_release_tag()
+        if not latest:
+            return None
+
+        rn = self._release_notes()
+        if rn is None:
+            return None
+        significance, version_tuple = rn.significance, rn.version_tuple
+
+        if version_tuple(latest) <= version_tuple(current):
+            return None
+
+        level, versions, summary = self._classify(
+            current, latest, significance
+        )
+
+        # Documentation-only releases are not worth an alert. This is the whole
+        # point: nobody should get a notification because a README was reworded.
+        if level == "docs":
+            logger.debug(
+                "truecloud-patch %s -> %s is documentation-only; not alerting",
+                current, latest,
+            )
+            return None
+
+        args = {
+            "current": f"v{current}",
+            "latest": latest,
+            "summary": summary,
+            "dir": PATCH_DIR,
+        }
+        klass = (
+            TrueCloudPatchSecurityUpdateAlertClass if level == "security"
+            else TrueCloudPatchUpdateAlertClass
+        )
+        return Alert(klass, args, key=[current, latest])
+
+    def _release_notes(self):
+        """Load tools/release_notes.py by path.
+
+        NOT via sys.path: prepending would shadow the stdlib for this interpreter,
+        and this runs in middlewared's thread pool, so mutating sys.path is a race.
+        """
+        path = os.path.join(PATCH_DIR, "tools", "release_notes.py")
+        try:
+            spec = importlib.util.spec_from_file_location("_tc_release_notes", path)
+            mod = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(mod)
+            return mod
+        except Exception:
+            logger.debug("could not load release_notes", exc_info=True)
+            return None
+
+    def _installed_version(self):
+        """The version of the patch actually checked out here."""
+        try:
+            with open(os.path.join(PATCH_DIR, "patch", "apply.sh"), encoding="utf-8") as fh:
+                m = _VERSION_RE.search(fh.read())
+        except OSError:
+            return None
+        return m.group(1) if m else None
+
+    def _latest_release_tag(self):
+        """Newest plain vX.Y.Z tag on the remote. Read-only: no .git writes.
+
+        Pre-release tags (-rc, -beta) are excluded: git's version sort ranks
+        v0.5.0-rc1 above v0.5.0, so including them would advertise a release
+        candidate as the latest stable.
+        """
+        try:
+            out = self._git("ls-remote", "--tags", "--refs", "origin")
+        except Exception:
+            return None
+
+        tags = []
+        for line in out.splitlines():
+            parts = line.split("refs/tags/")
+            if len(parts) == 2 and _TAG_RE.match(parts[1].strip()):
+                tags.append(parts[1].strip())
+        if not tags:
+            return None
+
+        return max(tags, key=lambda t: tuple(int(x) for x in t.lstrip("v").split(".")))
+
+    def _classify(self, current, latest, significance):
+        """(level, versions, one-line summary). Falls back to alerting."""
+        text = self._remote_changelog(latest)
+        if text is None:
+            # Cannot tell whether it matters. Alert rather than risk hiding a
+            # security fix -- but say that we could not tell.
+            return "notable", [], " (could not read the changelog)"
+
+        level, versions, headings = significance(text, current, latest)
+        if level == "docs":
+            return level, versions, ""
+
+        seen, ordered = set(), []
+        for h in headings:
+            if h not in seen:
+                seen.add(h)
+                ordered.append(h.capitalize())
+        detail = ", ".join(ordered)
+        return level, versions, f" Changes: {detail}." if detail else ""
+
+    def _changelog_url(self, tag):
+        """Where to read CHANGELOG.md at `tag`, derived from the origin remote.
+
+        Forge-agnostic on purpose. This project is canonically hosted on Gitea and
+        mirrored to GitHub, and hard-coding either one has a nastier failure than it
+        looks: when the changelog cannot be read, _classify() falls back to
+        "notable" and alerts ANYWAY, because the alternative is silently hiding a
+        security fix. So a stale URL does not disable the alert -- it makes the
+        alert fire on every release including documentation-only ones, which is
+        precisely the nagging this whole mechanism exists to prevent.
+        """
+        try:
+            remote = self._git("remote", "get-url", "origin").strip()
+        except Exception:
+            return None
+
+        m = _REMOTE_RE.match(remote)
+        if not m:
+            return None
+
+        host, owner, repo = m.group(1), m.group(2), m.group(3)
+
+        if host.endswith("github.com"):
+            return f"https://raw.githubusercontent.com/{owner}/{repo}/{tag}/CHANGELOG.md"
+
+        # Gitea and Forgejo both serve /{owner}/{repo}/raw/tag/{tag}/{path} over the
+        # web port, which is not the SSH port the remote may name.
+        return f"https://{host}/{owner}/{repo}/raw/tag/{tag}/CHANGELOG.md"
+
+    def _remote_changelog(self, tag):
+        """CHANGELOG.md at `tag`, over HTTPS. None if it cannot be read."""
+        url = self._changelog_url(tag)
+        if not url:
+            return None
+        try:
+            with urllib.request.urlopen(url, timeout=_TIMEOUT) as resp:  # noqa: S310
+                if resp.status != 200:
+                    return None
+                return resp.read().decode("utf-8", "replace")
+        except Exception:
+            return None

patch/apply.sh

@@ -7,13 +7,21 @@
 # is already up and has already imported the stock modules — the on-disk
 # patch alone cannot reach the running process.
 #
-# TrueNAS updates replace /usr/ entirely; this script re-applies two patches:
+# TrueNAS updates replace /usr/ entirely; this script re-applies three patches:
 #
 #   1. Backend — b2.py and restic.py are patched directly in the overlay.
 #      On a boot run, a single detached middlewared restart is scheduled
 #      (Step 3) so the patched modules actually get loaded.
 #
-#   2. Angular JS bundle — Widens the TrueCloud Backup credential dropdown
+#   2. Nested-dataset snapshots — installs _truecloud_nested.py and patches
+#      plugins/cloud/{snapshot,crud}.py + plugins/cloud_backup/sync.py so the
+#      "Take Snapshot" option works on a dataset that has child datasets.
+#      Stock middleware refuses that config, because it points the backup tool
+#      at the PARENT's .zfs/snapshot/ where children are invisible — it would
+#      silently back up a near-empty tree. We stage a complete tree of
+#      per-dataset bind mounts and only then relax the guard.
+#
+#   3. Angular JS bundle — Widens the TrueCloud Backup credential dropdown
 #                          from Storj-only to include S3 and B2. Served from
 #                          disk per request, so no restart is needed for it.
 #
@@ -24,7 +32,7 @@
 # Derive PATCH_DIR from this script's location (parent of the patch/ directory).
 PATCH_DIR="$(cd "$(dirname "$0")/.." && pwd)"
 LOG="$PATCH_DIR/apply.log"
-VERSION="0.0.4"
+VERSION="0.6.0"
 
 # Rotate log at 512 KB to avoid unbounded growth on a system volume.
 # Keep two prior generations (.1 and .2) so the last three boots are always available.
@@ -101,10 +109,20 @@ echo "Using Python: $PYTHON"
 # Combines what were previously four separate Python invocations into one to
 # avoid repeated interpreter startup overhead under the PREINIT timeout budget.
 
+# The patch has two independent modules, and each retires on its own:
+#
+#   providers  — B2/S3 credentials for TrueCloud Backup (b2.py, restic.py, UI)
+#   nested     — snapshots on datasets that have child datasets (cloud/*.py)
+#
+# TrueNAS may well ship one natively long before the other, so a single
+# all-or-nothing kill switch would silently take a still-needed module down with
+# the superseded one. Each module is detected separately and skipped on its own;
+# the global kill switch fires only once BOTH are native.
+
 _tc_info=$("$PYTHON" -c "
 import inspect, os, sys
 
-result = {'native': 'no', 'site_pkg': '', 'mw_dir': ''}
+result = {'native_b2': 'no', 'native_nested': 'no', 'site_pkg': '', 'mw_dir': ''}
 
 try:
     import middlewared
@@ -118,6 +136,7 @@ except ImportError:
     except Exception:
         pass
 
+# providers: does B2RcloneRemote already carry a real get_restic_config()?
 try:
     import middlewared.rclone.remote.b2 as _b2_mod
     from middlewared.rclone.remote.b2 import B2RcloneRemote
@@ -129,41 +148,246 @@ try:
             except (OSError, TypeError):
                 method_src = ''
             if 'NotImplementedError' not in method_src:
-                result['native'] = 'yes'
+                result['native_b2'] = 'yes'
 except Exception:
     pass
 
-print(result['native'])
+# nested: stock gates nested datasets with a validation in plugins/cloud/crud.py.
+# If that guard is gone, iX removed it, which means they implemented the traversal.
+#
+# Only look at the STOCK part of the file. Our own CRUD_BLOCK quotes the guard
+# message (it filters on it), so scanning the whole file would find the string in
+# our own patch and conclude the guard is still there. That happens to be
+# harmless today because detection runs before patching, but it makes the probe
+# silently order-dependent -- so cut our block off explicitly.
+#
+# If the file cannot be read we assume 'no' and keep patching: worst case the
+# patch declines to apply and the option simply stays unavailable.
+try:
+    crud = os.path.join(result['mw_dir'], 'plugins', 'cloud', 'crud.py')
+    with open(crud, encoding='utf-8', errors='replace') as fh:
+        stock_src = fh.read().split('\n# TRUECLOUD_PATCH', 1)[0]
+    # The guard message is SPLIT across adjacent string literals in the source:
+    #
+    #     verrors.add(..., 'This option is only available for datasets that have no further '
+    #                      'nesting')
+    #
+    # Python concatenates those at runtime, so the errmsg is contiguous -- but the
+    # SOURCE never contains the whole phrase. A raw search finds nothing, concludes
+    # iX removed the guard, and silently skips this module FOREVER. (Caught only by
+    # running the probe against real middlewared.)
+    #
+    # Strip whitespace and quote characters, then match the compacted phrase. That
+    # is robust to any wrapping or concatenation style iX may use.
+    _drop = str.maketrans('', '', ' \\t\\n\\r' + chr(34) + chr(39))
+    if 'nofurthernesting' not in stock_src.translate(_drop):
+        result['native_nested'] = 'yes'
+except Exception:
+    pass
+
+print(result['native_b2'])
+print(result['native_nested'])
 print(result['site_pkg'])
 print(result['mw_dir'])
-" 2>/dev/null || printf 'no\n\n\n')
+" 2>/dev/null || printf 'no\nno\n\n\n')
 
-_tc_native=$(printf '%s' "$_tc_info" | sed -n '1p')
-SITE_PKG=$(printf '%s' "$_tc_info"  | sed -n '2p')
-_MW_DIR=$(printf '%s' "$_tc_info"   | sed -n '3p')
+_tc_native_b2=$(printf '%s' "$_tc_info"     | sed -n '1p')
+_tc_native_nested=$(printf '%s' "$_tc_info" | sed -n '2p')
+SITE_PKG=$(printf '%s' "$_tc_info"          | sed -n '3p')
+_MW_DIR=$(printf '%s' "$_tc_info"           | sed -n '4p')
 
-if [ "$_tc_native" = "yes" ]; then
-    echo "NOTICE: TrueNAS now provides native B2 restic support — truecloud-patch is no longer needed."
+# Nested support is opt-in; if it was never enabled, it cannot be the reason to
+# keep the patch alive.
+if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]; then
+    _NESTED_ENABLED=1
+else
+    _NESTED_ENABLED=0
+fi
+
+# ── compatibility preflight ──────────────────────────────────────────────────
+#
+# The native probes above ask "has iX made this module unnecessary?". This asks the
+# other question, the dangerous one: "has iX changed middleware so that this module
+# no longer WORKS?"
+#
+# middlewared is internal API with no stability contract, and TrueNAS 26 rewrites
+# the entire cloud_backup path from async to synchronous. Every block the nested
+# module injects is an `async def` wrapping an `await`ed original; on 26 that hands
+# sync.py a coroutine where it unpacks a tuple. The backup does not fail cleanly --
+# it fails at the point where you needed it.
+#
+# tools/compat.py records what each module assumes and checks it against the
+# middlewared ACTUALLY INSTALLED HERE. A module whose assumptions no longer hold is
+# not applied. Stock TrueNAS without a feature beats TrueNAS with a broken one.
+#
+# Fail direction, deliberately asymmetric:
+#   * a definite "assumption violated"  -> disable that module. Strong evidence.
+#   * the checker cannot run at all     -> change nothing. That is a tooling glitch,
+#     not evidence, and turning it into a disabled module would break working boxes.
+_TC_COMPAT_JSON="$PATCH_DIR/incompatible.json"
+rm -f "$_TC_COMPAT_JSON"
+
+_tc_incompatible=0
+_tc_compat=unknown
+
+# No middlewared directory means the checker has nothing to read -- every module
+# would look "broken" because every file is missing, which is the strongest possible
+# evidence derived from the weakest possible input. Skip the preflight entirely and
+# let the existing "Cannot determine middlewared directory" path handle it.
+if [ -z "$_MW_DIR" ]; then
+    echo "NOTICE: middlewared directory unknown; skipping the compatibility preflight."
+    _tc_compat=$(printf 'unknown\nunknown\n')
+else
+_tc_compat=$("$PYTHON" - "$PATCH_DIR" "$_MW_DIR" "$_TC_COMPAT_JSON" 2>/dev/null <<'PYEOF' || printf 'unknown\nunknown\n'
+import json, os, sys
+
+patch_dir, mw_dir, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
+
+# APPEND, never insert(0) -- see the note by the mw_patch import below. Shadowing
+# the stdlib for this interpreter is a much worse failure than not finding compat.
+sys.path.append(os.path.join(patch_dir, 'tools'))
+try:
+    import compat
+    result = compat.check_tree(mw_dir)
+except Exception:
+    print('unknown')
+    print('unknown')
+    raise SystemExit(0)
+
+def verdict(r):
+    # Deliberately NOT exempting 'native' here, unlike the CI matrix.
+    #
+    # 'native' answers "do we still NEED this module?"; 'ok' answers "is it still
+    # SAFE to inject?". They are different questions, and letting native mask a
+    # broken assumption conflates them: a future TrueNAS that both reworded the
+    # nesting guard (-> native) AND changed the signatures (-> broken) would read
+    # as safe, and we would patch it anyway.
+    #
+    # Refusing to apply is the correct action for BOTH answers -- a native module
+    # is unnecessary and a broken one is dangerous -- so the apply path only has to
+    # ask whether the assumptions hold. Whether the feature went native is decided
+    # separately, by the probes above, and only affects the wording of the notice.
+    return 'ok' if r['ok'] else 'broken'
+
+broken = {m: r for m, r in result.items() if verdict(r) == 'broken'}
+if broken:
+    # The alert source reads this. Written before we print, so a box that is
+    # incompatible always has the evidence on disk even if apply.sh dies later.
+    try:
+        with open(out_path, 'w', encoding='utf-8') as fh:
+            json.dump(broken, fh, indent=2)
+    except OSError:
+        pass
+
+print(verdict(result['providers']))
+print(verdict(result['nested']))
+PYEOF
+)
+fi
+
+_tc_compat_providers=$(printf '%s' "$_tc_compat" | sed -n '1p')
+_tc_compat_nested=$(printf '%s' "$_tc_compat"    | sed -n '2p')
+
+# Is either module still doing something useful -- and can it still be applied?
+_providers_needed=1
+[ "$_tc_native_b2" = "yes" ] && _providers_needed=0
+
+_nested_needed=0
+if [ "$_NESTED_ENABLED" = "1" ] && [ "$_tc_native_nested" != "yes" ]; then
+    _nested_needed=1
+fi
+
+# The native checks above have already zeroed _*_needed for anything TrueNAS now
+# does itself, and printed the (good) news. Only complain about a module that is
+# still NEEDED and no longer fits -- otherwise a version that took a feature native
+# AND reshaped the module would be announced as "NOT COMPATIBLE", which is alarming
+# and false.
+if [ "$_tc_compat_providers" = "broken" ] && [ "$_providers_needed" = "1" ]; then
+    echo "WARNING: truecloud-patch is NOT COMPATIBLE with this TrueNAS version."
+    echo "WARNING:   The B2/S3 providers module will NOT be applied. TrueCloud is"
+    echo "WARNING:   left stock, so B2/S3 tasks will not run until this is fixed."
+    echo "WARNING:   Details: $_TC_COMPAT_JSON"
+    _providers_needed=0
+    _tc_incompatible=1
+fi
+
+if [ "$_tc_compat_nested" = "broken" ] && [ "$_nested_needed" = "1" ]; then
+    echo "WARNING: truecloud-patch's nested-snapshot module is NOT COMPATIBLE with"
+    echo "WARNING:   this TrueNAS version and will NOT be applied. Backups still"
+    echo "WARNING:   run; datasets nested under the target are not included."
+    echo "WARNING:   Details: $_TC_COMPAT_JSON"
+    _nested_needed=0
+    _tc_incompatible=1
+fi
+
+_tc_unmount_overlays() {
+    for _tag in mw ui; do
+        if mount | grep -qF "truecloud-${_tag} on "; then
+            _mnt=$(mount | grep "truecloud-${_tag} on " | awk '{print $3}' | head -1)
+            if umount "$_mnt" 2>/dev/null; then
+                echo "NOTICE: Unmounted overlay on $_mnt"
+            fi
+        fi
+    done
+}
+
+# INCOMPATIBLE is not the same as RETIRED, and must never take the same exit.
+#
+# The kill switch below is permanent -- apply.sh checks for it and returns early on
+# every future boot -- and only install.sh removes it, NOT update.sh. That is right
+# for retirement ("TrueNAS does this natively now; stop forever"), and catastrophic
+# for incompatibility: on TrueNAS 26 the providers module fails its assumptions and
+# nested is opt-out by default, so BOTH would be zero, the kill switch would fire,
+# and the very release that fixes 26 could never re-enable itself. The user would
+# run `bash update.sh` -- exactly what the update alert tells them to do -- and the
+# patch would stay dead, silently, with their B2 backups off.
+#
+# So: incompatible means "apply nothing THIS boot, and try again next boot". The
+# fix ships, update.sh checks it out, the next boot re-runs the preflight, the
+# assumptions hold, and the patch comes back by itself.
+if [ "$_tc_incompatible" = "1" ] && [ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]; then
+    echo "NOTICE: Nothing can be applied on this TrueNAS version — see the WARNINGs above."
+    echo "NOTICE: The kill switch is deliberately NOT set: this is an incompatibility,"
+    echo "NOTICE: not a retirement. Install a release that supports this TrueNAS"
+    echo "NOTICE:   bash $PATCH_DIR/update.sh"
+    echo "NOTICE: and the patch will re-apply itself on the next boot."
+    _tc_unmount_overlays
+    echo "=== done ==="
+    exit 0
+fi
+
+if [ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]; then
+    echo "NOTICE: Nothing left for truecloud-patch to do:"
+    [ "$_tc_native_b2" = "yes" ] && echo "NOTICE:   - TrueNAS now provides native B2 restic support."
+    if [ "$_tc_native_nested" = "yes" ]; then
+        echo "NOTICE:   - TrueNAS now handles snapshots on nested datasets natively."
+    elif [ "$_NESTED_ENABLED" = "0" ]; then
+        echo "NOTICE:   - Nested-dataset snapshots are not enabled (opt-in)."
+    fi
     echo "NOTICE: Setting kill switch; patching will be skipped on all future boots."
     echo "NOTICE: Run the following to fully remove the patch:"
     echo "NOTICE:   bash $PATCH_DIR/uninstall.sh"
     touch "$PATCH_DIR/disabled"
-    for _tag in mw ui; do
-        if mount | grep -qF "truecloud-${_tag} on "; then
-            _mnt=$(mount | grep "truecloud-${_tag} on " | awk '{print $3}' | head -1)
-            umount "$_mnt" 2>/dev/null && echo "NOTICE: Unmounted overlay on $_mnt" || true
-        fi
-    done
+    _tc_unmount_overlays
     echo "=== done ==="
     exit 0
 fi
 
+if [ "$_providers_needed" = "0" ]; then
+    echo "NOTICE: TrueNAS now provides native B2 restic support — the providers module"
+    echo "NOTICE: is superseded and will be skipped. The nested-snapshot module is still"
+    echo "NOTICE: active, so the patch stays installed."
+fi
+if [ "$_NESTED_ENABLED" = "1" ] && [ "$_tc_native_nested" = "yes" ]; then
+    echo "NOTICE: TrueNAS now handles nested-dataset snapshots natively — that module is"
+    echo "NOTICE: superseded and will be skipped. You can drop --enable-nested-snapshots."
+fi
+
 # ── Step 1: backend patch ─────────────────────────────────────────────────────
 
 echo "--- backend patch ---"
 
-_b2_ok=0
-_restic_ok=0
+_backend_ok=0
 
 if [ -z "$SITE_PKG" ]; then
     echo "WARNING: Cannot determine site-packages directory; skipping backend patch."
@@ -175,12 +399,27 @@ elif [ -z "$_MW_DIR" ]; then
 else
     _B2_PY="$_MW_DIR/rclone/remote/b2.py"
     _RESTIC_PY="$_MW_DIR/plugins/cloud_backup/restic.py"
+    _CLOUD_DIR="$_MW_DIR/plugins/cloud"
+    _SYNC_PY="$_MW_DIR/plugins/cloud_backup/sync.py"
+    _NESTED_SRC="$PATCH_DIR/patch/truecloud_nested.py"
 
-        # ── patch b2.py + restic.py + hook_status.json (single subprocess) ──────
-        if "$PYTHON" - "$_B2_PY" "$_RESTIC_PY" "$PATCH_DIR/hook_status.json" << 'PYEOF'
-import json, os, sys, time
+    # Each module is applied only if it is still needed. _providers_needed and
+    # _nested_needed were computed above (native-support detection + the opt-in
+    # marker), so one module going native never disables the other.
+        # ── patch b2.py + restic.py + nested-snapshot + hook_status.json ────────
+        # (single subprocess: PREINIT has a tight timeout budget)
+        if "$PYTHON" - "$_B2_PY" "$_RESTIC_PY" "$PATCH_DIR/hook_status.json" \
+                      "$_CLOUD_DIR" "$_SYNC_PY" "$_NESTED_SRC" \
+                      "$_providers_needed" "$_nested_needed" "$_NESTED_ENABLED" \
+                      "$_tc_native_nested" << 'PYEOF'
+import json, os, shutil, sys, time
 
 b2_path, restic_path, status_path = sys.argv[1], sys.argv[2], sys.argv[3]
+cloud_dir, sync_path, nested_src = sys.argv[4], sys.argv[5], sys.argv[6]
+providers_needed = sys.argv[7] == "1"
+nested_needed = sys.argv[8] == "1"
+nested_enabled = sys.argv[9] == "1"
+nested_native = sys.argv[10] == "yes"
 
 B2_BLOCK = """
 # TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh
@@ -240,45 +479,426 @@ else:
     get_restic_config._truecloud_patched = True
 """
 
-def patch_file(path, block):
-    with open(path, encoding="utf-8") as fh:
-        content = fh.read()
-    marker = "\n# TRUECLOUD_PATCH"
-    idx = content.find(marker)
-    base = content[:idx] if idx != -1 else content
-    with open(path, "w", encoding="utf-8") as fh:
-        fh.write(base.rstrip("\n") + "\n" + block)
+# ── nested-dataset snapshot support ───────────────────────────────────────────
+# Stock middleware refuses `snapshot=true` on a path containing child datasets,
+# because it points the backup tool at the PARENT dataset's .zfs/snapshot/,
+# where child datasets are INVISIBLE -- it would silently back up a near-empty
+# tree. That guard is correct. We implement the missing traversal (a staging
+# tree of per-dataset bind mounts) and only then relax the guard.
+#
+# Fail-safe direction: if any of these three blocks fails to apply, the stock
+# guard remains and the option simply stays unavailable. We never end up with
+# the guard removed but the traversal missing -- that would be a silently empty
+# backup, the worst possible outcome.
+
+# ── nested blocks: one core, two wrappers ─────────────────────────────────────
+#
+# TrueNAS <= 25.10 has an ASYNC cloud_backup path; TrueNAS 26 rewrote it SYNCHRONOUS
+# (`middleware.call_sync` throughout, no awaits). An `async def` wrapper on 26 hands
+# sync.py a coroutine where it unpacks a tuple, and a `def` wrapper on 25.10 blocks
+# the event loop. So each block is assembled from:
+#
+#   * a CORE, written once, synchronous, using middleware.call_sync -- which is safe
+#     from a worker thread and deadlocks on the event loop; and
+#   * a WRAPPER matching the stock function's own flavour, chosen at apply time by
+#     reading whether the installed middlewared declares it `async def`.
+#
+# On <= 25.10 the async wrapper hops to a thread via `await middleware.run_in_thread`
+# -- exactly the thread call_sync needs. On 26 the stock function is already running
+# in middlewared's thread pool (its own code calls call_sync), so the sync wrapper
+# calls the core directly.
+#
+# The logic that matters -- snapshots, bind mounts, failure modes -- exists once.
+# An async twin would mean every future fix had to land twice, and the one that got
+# missed would be the one that eats a backup.
+
+_NESTED_IMPORT = """
+# TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh
+try:
+    from middlewared.plugins.cloud import _truecloud_nested as _tc_nested
+except ImportError:
+    _tc_nested = None
+"""
+
+SNAPSHOT_CORE = _NESTED_IMPORT + """
+if _tc_nested is not None:
+    _tc_orig_create_snapshot = create_snapshot
+
+    def _tc_stage(middleware, path, name, snapshot, snap_path):
+        # Synchronous, and always called from a worker thread (see above).
+        #
+        # ONLY cloud_backup. Bail out before touching anything otherwise.
+        #
+        # create_snapshot is module-global in plugins/cloud/snapshot.py and is
+        # imported by cloud_sync.py as well as cloud_backup/sync.py -- so this
+        # wrapper sits in the path of every rclone/Storj CloudSync task with
+        # snapshot=true, not just ours. Two consequences, and the second is worse:
+        #
+        #   * everything below is a NEW failure mode for tasks that worked before we
+        #     were installed. A `zfs.dataset.query` that errors would break a
+        #     CloudSync job we have no business touching.
+        #   * if a CloudSync task ever were staged, nothing would ever tear it down:
+        #     the teardown is wired into cloud_backup's restic_backup finally, and
+        #     CRUD_BLOCK deliberately leaves CloudSync's nesting guard intact. The
+        #     bind mounts would pin the snapshot forever.
+        #
+        # cloud_backup names its snapshot "cloud_backup-<id>"; cloud_sync names it
+        # "cloud_sync-<id>"; the stock default is "cloud_task-onetime". Anything that
+        # is not ours gets stock behaviour, untouched, with no extra middleware call.
+        if not name.startswith("cloud_backup"):
+            return snapshot, snap_path
+
+        _logger = getattr(middleware, "logger", None)
+        try:
+            # Enumerate datasets AFTER the snapshot, never before. The snapshot is
+            # the point-in-time truth; a list read beforehand could miss a dataset
+            # created in the gap, which the recursive snapshot WOULD capture but
+            # our staging plan would not -- silently omitting it from the backup.
+            # Read afterwards, an unsnapshotted dataset instead trips the isdir()
+            # check in plan_staging and fails the run loudly. Loud beats silent.
+            datasets = middleware.call_sync(
+                "zfs.dataset.query", [["type", "=", "FILESYSTEM"]]
+            )
+            # OUR copy of get_dataset_recursive, not the host module's: TrueNAS 26
+            # deleted that helper (create_snapshot uses filesystem.statfs now), so
+            # calling it out of the module namespace is a NameError there.
+            dataset, nested = _tc_nested.get_dataset_recursive(datasets, path)
+
+            if not nested:
+                # No children: stock behaviour, untouched. Stock's `finally` owns
+                # the snapshot from here (its non-recursive delete is correct,
+                # because a non-nested snapshot has no children).
+                return snapshot, snap_path
+
+            staging_root = _tc_nested.stage_nested(
+                middleware, path, snapshot,
+                dataset["name"], dataset["properties"]["mountpoint"]["value"],
+                name, datasets, logger=_logger,
+            )
+        except Exception:
+            # The snapshot exists, but this exception means sync.py never completes
+            # `snapshot, local_path = create_snapshot(...)`, so its local `snapshot`
+            # stays None and its `finally` deletes NOTHING. Sweep the tree ourselves
+            # or leak the parent plus one snapshot per descendant dataset (160+ here)
+            # on every failed run.
+            _tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger)
+            raise
+
+        return snapshot, staging_root
+"""
+
+SNAPSHOT_ASYNC = SNAPSHOT_CORE + """
+    async def create_snapshot(middleware, path, name="cloud_task-onetime"):
+        snapshot, snap_path = await _tc_orig_create_snapshot(middleware, path, name)
+        return await middleware.run_in_thread(
+            _tc_stage, middleware, path, name, snapshot, snap_path
+        )
+
+    create_snapshot._truecloud_patched = True
+"""
+
+SNAPSHOT_SYNC = SNAPSHOT_CORE + """
+    def create_snapshot(middleware, path, name="cloud_task-onetime"):
+        snapshot, snap_path = _tc_orig_create_snapshot(middleware, path, name)
+        return _tc_stage(middleware, path, name, snapshot, snap_path)
+
+    create_snapshot._truecloud_patched = True
+"""
+
+_CRUD_FILTER = """
+        # Only cloud_backup: staging teardown is wired into cloud_backup.sync's
+        # finally. cloudsync would leak bind mounts, so leave its guard intact.
+        if getattr(getattr(self, "_config", None), "namespace", "") != "cloud_backup":
+            return
+
+        # Drop ONLY the nested-dataset guard. If iX ever rewords the message the
+        # filter stops matching, the guard survives, and the option merely stays
+        # unavailable -- the safe direction to fail.
+        verrors.errors = [
+            e for e in verrors.errors
+            if not (
+                getattr(e, "attribute", "") == f"{name}.snapshot"
+                and "no further nesting" in getattr(e, "errmsg", "")
+            )
+        ]
+
+    CloudTaskServiceMixin._validate = _tc_validate
+    CloudTaskServiceMixin._validate._truecloud_patched = True
+"""
+
+CRUD_ASYNC = _NESTED_IMPORT + """
+if _tc_nested is not None:
+    _tc_orig_validate = CloudTaskServiceMixin._validate
+
+    async def _tc_validate(self, app, verrors, name, data):
+        await _tc_orig_validate(self, app, verrors, name, data)
+""" + _CRUD_FILTER
+
+CRUD_SYNC = _NESTED_IMPORT + """
+if _tc_nested is not None:
+    _tc_orig_validate = CloudTaskServiceMixin._validate
+
+    def _tc_validate(self, app, verrors, name, data):
+        _tc_orig_validate(self, app, verrors, name, data)
+""" + _CRUD_FILTER
+
+# *args/**kwargs, not the stock signature spelled out.
+#
+# 24.10 and 25.04 have `restic_backup(middleware, job, cloud_backup, dry_run)`;
+# 25.10 added `rate_limit`. Naming them and forwarding all five raised
+# `TypeError: takes 4 positional arguments but 5 were given` on every nested backup
+# on the two older releases. Forwarding whatever we were handed makes this wrapper
+# indifferent to iX adding or dropping a trailing parameter -- which they have now
+# done twice.
+#
+# Our bind mounts pin the ZFS snapshot, so stock's `finally` cannot destroy it
+# (EBUSY) and logs one benign warning. We unmount here and then delete it for real.
+SYNC_ASYNC = _NESTED_IMPORT + """
+if _tc_nested is not None:
+    _tc_orig_restic_backup = restic_backup
+
+    async def restic_backup(middleware, job, cloud_backup, *args, **kwargs):
+        try:
+            return await _tc_orig_restic_backup(middleware, job, cloud_backup, *args, **kwargs)
+        finally:
+            try:
+                # logger= must be passed here too, exactly as the sync variant does.
+                # run_in_thread forwards **kwargs (functools.partial), and without it
+                # cleanup_task gets logger=None -- so every teardown warning ("could
+                # not unmount X") is silently swallowed on <= 25.10, which is most
+                # boxes. The two wrappers must differ ONLY in how they reach the core.
+                await middleware.run_in_thread(
+                    _tc_nested.cleanup_task,
+                    middleware,
+                    f"cloud_backup-{cloud_backup.get('id', 'onetime')}",
+                    logger=getattr(middleware, "logger", None),
+                )
+            except Exception as e:
+                middleware.logger.warning("truecloud-patch: staging cleanup failed: %r", e)
+
+    restic_backup._truecloud_patched = True
+"""
+
+SYNC_SYNC = _NESTED_IMPORT + """
+if _tc_nested is not None:
+    _tc_orig_restic_backup = restic_backup
+
+    def restic_backup(middleware, job, cloud_backup, *args, **kwargs):
+        try:
+            return _tc_orig_restic_backup(middleware, job, cloud_backup, *args, **kwargs)
+        finally:
+            try:
+                _tc_nested.cleanup_task(
+                    middleware,
+                    f"cloud_backup-{cloud_backup.get('id', 'onetime')}",
+                    logger=getattr(middleware, "logger", None),
+                )
+            except Exception as e:
+                middleware.logger.warning("truecloud-patch: staging cleanup failed: %r", e)
+
+    restic_backup._truecloud_patched = True
+"""
+
+
+# Single implementation of the block apply/revert logic (patch/mw_patch.py), so
+# uninstall.sh and apply.sh cannot drift apart. Fail-safe: if it cannot be
+# imported, skip the backend patch entirely -- middlewared then starts stock,
+# which is the whole design principle of this script.
+# APPEND, never insert(0): this dir would otherwise take precedence over the
+# stdlib for this interpreter, so a future patch/json.py (say) would shadow the
+# real json module and break the boot. Appending fails safe -- worst case our
+# import misses and the backend patch is skipped.
+sys.path.append(os.path.dirname(nested_src))
+try:
+    from mw_patch import patch_file, revert_nested
+except ImportError as _e:
+    print(f'WARNING: cannot import patch/mw_patch.py ({_e}) — skipping backend patch.')
+    print('WARNING: middlewared will start with stock (unpatched) modules.')
+    sys.exit(1)
+
+# .../middlewared/plugins/cloud -> .../middlewared
+mw_dir = os.path.dirname(os.path.dirname(cloud_dir))
 
 b2_ok = restic_ok = False
+nested_ok = False
 
-if os.path.exists(b2_path):
-    try:
-        patch_file(b2_path, B2_BLOCK)
-        b2_ok = True
-        print(f"OK: Patched b2.py → {b2_path}")
-    except Exception as e:
-        print(f"WARNING: Failed to patch b2.py: {e}")
+# ── module: providers (B2/S3) ─────────────────────────────────────────────────
+# Skipped entirely once TrueNAS ships native B2 restic support. That must not
+# take the nested module down with it, so the two are gated independently.
+providers_detail = ''
+if not providers_needed:
+    providers_detail = 'superseded: TrueNAS provides native B2 restic support'
+    print('INFO: Providers module skipped — TrueNAS now supports B2 natively.')
 else:
-    print(f"WARNING: b2.py not found at {b2_path}")
+    if os.path.exists(b2_path):
+        try:
+            patch_file(b2_path, B2_BLOCK)
+            b2_ok = True
+            print(f"OK: Patched b2.py → {b2_path}")
+        except Exception as e:
+            print(f"WARNING: Failed to patch b2.py: {e}")
+    else:
+        print(f"WARNING: b2.py not found at {b2_path}")
 
-if os.path.exists(restic_path):
-    try:
-        patch_file(restic_path, RESTIC_BLOCK)
-        restic_ok = True
-        print(f"OK: Patched restic.py → {restic_path}")
-    except Exception as e:
-        print(f"WARNING: Failed to patch restic.py: {e}")
+    if os.path.exists(restic_path):
+        try:
+            patch_file(restic_path, RESTIC_BLOCK)
+            restic_ok = True
+            print(f"OK: Patched restic.py → {restic_path}")
+        except Exception as e:
+            print(f"WARNING: Failed to patch restic.py: {e}")
+    else:
+        print(f"WARNING: restic.py not found at {restic_path}")
+    providers_detail = (
+        'patched on disk in overlay at boot' if (b2_ok and restic_ok)
+        else 'b2.py/restic.py not found or write failed'
+    )
+
+# ── module: nested-dataset snapshots ──────────────────────────────────────────
+# Order matters: install the traversal machinery FIRST, relax the validation
+# guard LAST. If anything fails partway, the guard is still in place and the
+# option stays unavailable -- we never expose "guard removed, traversal missing".
+nested_detail = ''
+if not nested_needed:
+    # Not just "skip": actively revert. The overlay lives for the whole boot, so a
+    # previously-applied patch is still sitting there and middlewared would
+    # re-import it on restart. See revert_nested().
+    if not nested_enabled:
+        nested_detail = 'disabled (opt-in; enable with: install.sh --enable-nested-snapshots)'
+        print('INFO: Nested-dataset snapshot support is disabled (opt-in feature).')
+    elif nested_native:
+        nested_detail = 'superseded: TrueNAS handles nested-dataset snapshots natively'
+        print('INFO: Nested module skipped — TrueNAS now handles nesting natively.')
+    else:
+        nested_detail = 'not needed'
+        print('INFO: Nested module skipped.')
+
+    reverted = revert_nested(mw_dir)
+    if reverted:
+        print('OK: Reverted a previously-applied nested patch (' + ', '.join(reverted) + ').')
+        print('    The stock nesting guard is restored once middlewared restarts.')
+        nested_detail += ' — previous patch reverted'
+
+    if not nested_enabled:
+        print('INFO: Enable with: bash install.sh --enable-nested-snapshots')
 else:
-    print(f"WARNING: restic.py not found at {restic_path}")
+    try:
+        snapshot_py = os.path.join(cloud_dir, 'snapshot.py')
+        crud_py = os.path.join(cloud_dir, 'crud.py')
+        nested_dst = os.path.join(cloud_dir, '_truecloud_nested.py')
+
+        missing = [p for p in (snapshot_py, crud_py, sync_path, nested_src) if not os.path.exists(p)]
+        if missing:
+            raise FileNotFoundError('missing: ' + ', '.join(missing))
+
+        # Which flavour of cloud_backup is installed? <= 25.10 is async; TrueNAS 26
+        # rewrote it synchronous. Inject the wrapper that matches: an `async def` on
+        # 26 hands sync.py a coroutine where it unpacks a tuple, and a plain `def` on
+        # 25.10 blocks the event loop.
+        #
+        # None means the three stock functions disagree, or one could not be read.
+        # Refuse rather than guess -- a half-converted middleware is one this patch
+        # has never seen, and guessing wrong there costs a backup, not a feature.
+        # nested_src is <repo>/patch/truecloud_nested.py, so tools/ is its sibling.
+        # APPEND, never insert(0) -- shadowing the stdlib for this interpreter is a
+        # far worse failure than not finding compat.
+        sys.path.append(
+            os.path.join(os.path.dirname(os.path.dirname(nested_src)), 'tools')
+        )
+        import compat
+        _flavour = compat.async_flavour_tree(mw_dir)
+        if _flavour is None:
+            raise RuntimeError(
+                'cannot tell whether this TrueNAS cloud_backup path is async or '
+                'sync (the wrapped functions disagree, or could not be read)'
+            )
+
+        _snapshot_block = SNAPSHOT_ASYNC if _flavour else SNAPSHOT_SYNC
+        _sync_block     = SYNC_ASYNC     if _flavour else SYNC_SYNC
+        _crud_block     = CRUD_ASYNC     if _flavour else CRUD_SYNC
+
+        shutil.copyfile(nested_src, nested_dst)   # 1. traversal implementation
+        patch_file(snapshot_py, _snapshot_block)  # 2. build the staging tree
+        patch_file(sync_path, _sync_block)        # 3. tear it down afterwards
+        patch_file(crud_py, _crud_block)          # 4. ONLY NOW allow nested tasks
+
+        print('OK: cloud_backup is %s; injected the matching wrappers.'
+              % ('async (TrueNAS <= 25.10)' if _flavour else 'synchronous (TrueNAS 26+)'))
+
+        nested_ok = True
+        nested_detail = 'nested-dataset snapshots enabled (staging tree)'
+        print(f'OK: Installed nested-snapshot support → {nested_dst}')
+        print(f'OK: Patched snapshot.py, sync.py, crud.py → {cloud_dir}')
+    except Exception as e:
+        nested_detail = f'not applied: {e}'
+        print(f'WARNING: Failed to apply nested-snapshot patch: {e}')
+        print('WARNING: Stock nesting guard remains; snapshot option stays unavailable')
+        print('WARNING: for nested datasets. Existing backups are unaffected.')
+
+# One entry per MODULE, not per file. `ok` means "nothing is wrong", so a module
+# that is inactive (superseded, or opt-in and off) is ok -- reporting a disabled
+# opt-in feature as FAIL would make `create_task.py verify` fail on a default
+# install. `active` says whether the module is doing anything.
+# ── update-available alert ────────────────────────────────────────────────────
+# Dropped into middlewared/alert/source/, where middlewared discovers and polls it
+# natively — no cron, no timer. It only raises an alert for releases that actually
+# changed something: a docs-only release is ignored (see tools/release_notes.py).
+alert_ok = False
+alert_detail = ''
+_patch_dir = os.path.dirname(os.path.dirname(nested_src))
+alert_src = os.path.join(os.path.dirname(nested_src), 'alert_source.py')
+alert_dst = os.path.join(mw_dir, 'alert', 'source', 'truecloud_patch_update.py')
+
+if os.path.exists(os.path.join(_patch_dir, 'update_alerts_disabled')):
+    alert_detail = 'disabled (update_alerts_disabled)'
+    try:
+        os.unlink(alert_dst)
+        print('OK: Removed update alert (disabled).')
+    except OSError:
+        pass
+elif not os.path.exists(alert_src):
+    alert_detail = 'alert_source.py not found'
+    print(f'WARNING: {alert_src} missing — no update alert.')
+else:
+    try:
+        with open(alert_src, encoding='utf-8') as fh:
+            _body = fh.read()
+
+        # repr() so ANY path becomes a valid Python literal -- a directory
+        # containing a quote or backslash would otherwise produce a syntax error.
+        _body = _body.replace('"@PATCH_DIR@"', repr(_patch_dir))
+
+        # COMPILE BEFORE WRITING. middlewared's alert.load() imports every file in
+        # alert/source/ with NO try/except, and it runs at startup -- a module that
+        # raises on import takes middlewared's setup down with it. An uninstalled
+        # alert is a missing convenience; a broken one is a broken box.
+        compile(_body, alert_dst, 'exec')
+
+        with open(alert_dst, 'w', encoding='utf-8') as fh:
+            fh.write(_body)
+        alert_ok = True
+        alert_detail = 'update alert installed'
+        print(f'OK: Installed update alert → {alert_dst}')
+    except Exception as e:
+        alert_detail = f'not applied: {e}'
+        print(f'WARNING: could not install update alert: {e}')
+        print('WARNING: no update alert; everything else is unaffected.')
 
 patches = {
-    'middlewared.rclone.remote.b2': {
-        'ok': b2_ok,
-        'detail': 'patched on disk in overlay at boot' if b2_ok else 'b2.py not found or write failed',
+    'providers': {
+        'ok': (not providers_needed) or bool(b2_ok and restic_ok),
+        'active': providers_needed,
+        'detail': providers_detail,
     },
-    'middlewared.plugins.cloud_backup.restic': {
-        'ok': restic_ok,
-        'detail': 'patched on disk in overlay at boot' if restic_ok else 'restic.py not found or write failed',
+    'nested_snapshots': {
+        'ok': (not nested_needed) or nested_ok,
+        'active': nested_needed,
+        'detail': nested_detail,
+    },
+    'update_alert': {
+        'ok': True,          # never a failure: it is a convenience, not a patch
+        'active': alert_ok,
+        'detail': alert_detail,
     },
 }
 payload = {'patched_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'patches': patches}
@@ -291,35 +911,61 @@ try:
 except OSError as e:
     print(f'WARNING: Could not write hook_status.json: {e}')
 
-sys.exit(0 if (b2_ok and restic_ok) else 1)
+# Exit code tells the caller whether a middlewared restart is still worth doing:
+#
+#   0  every module that was needed applied cleanly
+#   2  PARTIAL -- one module failed but another landed, so there IS something new
+#      on disk waiting to be loaded
+#   1  nothing landed; a restart would accomplish nothing
+#
+# Collapsing 2 into 1 would mean a failing providers patch suppresses the restart
+# that a freshly-applied nested patch needs, leaving it on disk and never loaded.
+_providers_done = (not providers_needed) or bool(b2_ok and restic_ok)
+_nested_done = (not nested_needed) or nested_ok
+_landed = (providers_needed and b2_ok and restic_ok) or (nested_needed and nested_ok)
+
+if _providers_done and _nested_done:
+    sys.exit(0)
+sys.exit(2 if _landed else 1)
 PYEOF
         then
-            _b2_ok=1
-            _restic_ok=1
+            _backend_ok=1
         else
-            # Individual results already printed above; exit code 1 means at least one failed.
-            true
+            _rc=$?
+            if [ "$_rc" = "2" ]; then
+                # One module failed, but another was applied and still needs loading.
+                _backend_ok=1
+                echo "WARNING: a module failed to apply; the other landed and will be loaded."
+            else
+                _backend_ok=0
+            fi
         fi
 fi
 
 # ── Step 2: Angular bundle ────────────────────────────────────────────────────
+# Belongs to the providers module (it widens the credential dropdown), so it is
+# skipped along with it once TrueNAS supports B2 natively.
 
 echo "--- UI patch ---"
 
-# Ensure the webui directory is writable before patch_ui.py tries to create a
-# backup and write the patched bundle.  On immutable OS we mount an overlay.
-_webui_dir=""
-for _d in /usr/share/truenas/webui /usr/share/truenas-ui /var/www/truenas; do
-    if [ -d "$_d" ]; then
-        _webui_dir="$_d"
-        break
+if [ "$_providers_needed" = "0" ]; then
+    echo "Skipped — providers module superseded by native B2 support."
+else
+    # Ensure the webui directory is writable before patch_ui.py tries to create a
+    # backup and write the patched bundle.  On immutable OS we mount an overlay.
+    _webui_dir=""
+    for _d in /usr/share/truenas/webui /usr/share/truenas-ui /var/www/truenas; do
+        if [ -d "$_d" ]; then
+            _webui_dir="$_d"
+            break
+        fi
+    done
+    if [ -n "$_webui_dir" ]; then
+        _ensure_writable "$_webui_dir" "ui" || true   # non-fatal; patch_ui.py reports the error
     fi
-done
-if [ -n "$_webui_dir" ]; then
-    _ensure_writable "$_webui_dir" "ui" || true   # non-fatal; patch_ui.py reports the error
-fi
 
-"$PYTHON" "$PATCH_DIR/patch/patch_ui.py" || echo "WARNING: patch_ui.py exited non-zero; UI dropdown may still show Storj only."
+    "$PYTHON" "$PATCH_DIR/patch/patch_ui.py" || echo "WARNING: patch_ui.py exited non-zero; UI dropdown may still show Storj only."
+fi
 
 # ── Step 3: deferred middlewared restart (boot runs only) ─────────────────────
 # At boot this script is spawned by middlewared, which already imported the
@@ -329,23 +975,36 @@ fi
 # ix-* boot units still need midclt to answer.
 # Boot context is detected by the parent process being middlewared; manual
 # runs (install.sh, recovery) never trigger a restart.
+#
+# The unit runs wait_restart.sh, which blocks until boot has actually
+# settled (systemd job queue drained, docker/apps state terminal) before
+# restarting. systemd ordering alone (After=multi-user.target, ≤ v0.0.4)
+# fired while ix-reporting and the docker/apps startup were still in flight
+# and killed both — apps and dashboard stats stayed down until the next
+# boot. No Type=oneshot: a oneshot's start job would hold the boot queue
+# open against the `is-system-running --wait` inside the script.
 
 echo "--- deferred restart ---"
 
+# Restart when ANY still-needed backend module landed (_backend_ok, incl. the
+# partial case). Keying this off the providers module alone would skip the restart
+# on a box where B2 has gone native but the nested module was freshly patched —
+# leaving it on disk and never loaded.
+#
+# "No module active at all" cannot reach here: that is the kill-switch branch
+# above, which exits.
 if ! grep -aq middlewared "/proc/$PPID/cmdline" 2>/dev/null; then
     echo "Manual run (parent is not middlewared) — no restart scheduled."
-elif [ "$_b2_ok" != "1" ] || [ "$_restic_ok" != "1" ]; then
-    echo "Backend patch incomplete — no restart scheduled (nothing new to load)."
+elif [ "$_backend_ok" != "1" ]; then
+    echo "Nothing landed on disk — no restart scheduled (nothing new to load)."
 else
     # A failed unit from an earlier attempt this boot would block systemd-run.
     systemctl reset-failed truecloud-mw-restart.service 2>/dev/null
     if systemd-run --no-block --collect --unit=truecloud-mw-restart \
-           --property=Type=oneshot \
-           --property=After=multi-user.target \
-           --property=After=ix-postinit.service \
-           systemctl try-restart middlewared; then
+           /bin/bash "$PATCH_DIR/patch/wait_restart.sh"; then
         echo "OK: Scheduled deferred middlewared restart (unit: truecloud-mw-restart)."
-        echo "    Backend patch becomes active once boot completes."
+        echo "    It waits for boot to fully settle (apps started, reporting up),"
+        echo "    then restarts middlewared so the backend patch actually loads."
     else
         echo "WARNING: Could not schedule deferred restart — backend patch is on disk but NOT loaded."
         echo "  Activate manually: systemctl restart middlewared"

patch/create_task.py

@@ -3,88 +3,91 @@
 create_task.py — create TrueNAS TrueCloud Backup tasks with S3 or B2 credentials.
 
 The TrueNAS UI normally restricts the credential dropdown to Storj only.
-This script bypasses that restriction by calling the REST API directly.
+This script bypasses that restriction by talking to the TrueNAS middleware
+directly via `midclt` (the /api/v2.0 REST API is removed in TrueNAS 26.04).
 
 Compatible providers (after the truecloud-patch backend patch is applied):
   S3     — any S3-compatible endpoint (AWS, Wasabi, Cloudflare R2, MinIO, …)
   B2     — Backblaze B2 native API
   STORJ_IX — Storj (unchanged, always worked)
 
-Requires a TrueNAS API key: UI → System → API Keys → Add.
+Run this ON the TrueNAS host — it uses the local middleware socket via `midclt`,
+so no host address or API key is needed.
 
 Examples
 --------
 List available cloud credentials:
-    python3 create_task.py --host 192.168.1.1 --api-key <key> list-credentials
+    python3 create_task.py list-credentials
 
 Create a task backed by a B2 credential (id=3):
-    python3 create_task.py --host 192.168.1.1 --api-key <key> create \\
+    python3 create_task.py create \\
         --name "tank-to-b2" \\
         --path /mnt/tank/data \\
         --credential 3 \\
         --bucket my-bucket \\
         --folder backups/tank \\
-        --password "restic-repo-password" \\
+        --password-stdin \\
         --keep-last 14
+    (pipe the password in:  echo -n "s3cret" | python3 create_task.py create ... )
 
 Create a task using an S3-compatible credential (Wasabi, R2, etc.):
-    python3 create_task.py --host 192.168.1.1 --api-key <key> create \\
+    python3 create_task.py create \\
         --name "tank-to-wasabi" \\
         --path /mnt/tank/data \\
         --credential 5 \\
         --bucket my-bucket \\
         --folder backups \\
-        --password "restic-repo-password"
+        --password-stdin
 
 List existing TrueCloud Backup tasks:
-    python3 create_task.py --host 192.168.1.1 --api-key <key> list-tasks
+    python3 create_task.py list-tasks
 """
 
 import argparse
 import calendar
+import getpass
 import json
 import os
-import ssl
 import subprocess
 import sys
 import time
-import urllib.error
-import urllib.request
 
-__version__ = "0.0.4"
+__version__ = "0.6.0"
 
 _PATCH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 _STATUS_FILE = os.path.join(_PATCH_DIR, "hook_status.json")
 
 
-def make_client(host, api_key, insecure=False):
-    """Return a callable that makes authenticated REST API calls."""
-    base = f"https://{host}/api/v2.0"
-    headers = {
-        "Authorization": f"Bearer {api_key}",
-        "Content-Type": "application/json",
-    }
-    ctx = ssl.create_default_context()
-    if insecure:
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
+def midclt_call(method, *args):
+    """Call a middleware method on the local host.
 
-    def call(method, path, body=None):
-        url = base + path
-        data = json.dumps(body).encode() if body is not None else None
-        req = urllib.request.Request(url, data=data, headers=headers, method=method)
-        try:
-            with urllib.request.urlopen(req, context=ctx) as resp:
-                return json.loads(resp.read())
-        except urllib.error.HTTPError as exc:
-            detail = exc.read().decode(errors="replace")
-            print(f"HTTP {exc.code} {exc.reason}: {detail}", file=sys.stderr)
-            sys.exit(1)
-        except urllib.error.URLError as exc:
-            print(f"Connection error: {exc.reason}", file=sys.stderr)
-            sys.exit(1)
+    Uses `truenas_api_client` -- the library that backs `midclt` itself -- rather
+    than shelling out to `midclt`.
 
-    return call
+    This is a SECURITY requirement, not a style choice. `midclt call <method>
+    <json>` puts its arguments in the process's **argv**, and `cloud_backup.create`
+    carries the restic repository password. argv is world-readable via `ps`, so
+    shelling out would expose the key to the entire backup repo to every local
+    user for the duration of the call. Going through the client library keeps it
+    in this process's memory.
+    """
+    try:
+        from truenas_api_client import Client
+    except ImportError:
+        print(
+            "ERROR: `truenas_api_client` not importable — run this script ON the\n"
+            "       TrueNAS host. (It ships with midclt.)",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    try:
+        with Client() as client:
+            return client.call(method, *args)
+    except Exception as exc:  # noqa: BLE001 - surface any middleware error verbatim
+        # Never echo `args` here: for cloud_backup.create it contains the password.
+        print(f"ERROR: {method}: {exc}", file=sys.stderr)
+        sys.exit(1)
 
 
 # ── Sub-commands ──────────────────────────────────────────────────────────────
@@ -92,8 +95,11 @@ def make_client(host, api_key, insecure=False):
 def _middlewared_start_epoch():
     """Epoch timestamp of the running middlewared main process, or None."""
     try:
+        # Partial path (S607) is fine here: this runs as root on TrueNAS, so an
+        # attacker who can poison PATH already has root. Hard-coding a path would
+        # be less portable (/bin vs /usr/bin) for no security gain.
         pid = int(subprocess.run(
-            ["systemctl", "show", "--property=MainPID", "--value", "middlewared"],
+            ["systemctl", "show", "--property=MainPID", "--value", "middlewared"],  # noqa: S607
             capture_output=True, text=True, timeout=10, check=True,
         ).stdout.strip())
         if pid <= 0:
@@ -135,13 +141,21 @@ def cmd_verify():
     print(f"Hook status (recorded at {status.get('patched_at', 'unknown')})")
     print()
     all_ok = True
+    any_active = False
     for module, info in status.get("patches", {}).items():
         ok = info.get("ok", False)
+        # A module can be inactive because TrueNAS now does it natively, or
+        # because it is opt-in and switched off. Neither is a failure.
+        active = info.get("active", True)
         label = "OK  " if ok else "FAIL"
+        if ok and not active:
+            label = "SKIP"
         detail = f"  — {info['detail']}" if info.get("detail") else ""
         print(f"  [{label}] {module}{detail}")
         if not ok:
             all_ok = False
+        if active:
+            any_active = True
 
     # The disk status alone can false-positive: at boot the files are patched
     # while middlewared is already running with the stock modules imported.
@@ -154,7 +168,11 @@ def cmd_verify():
     mw_start = _middlewared_start_epoch()
 
     proc_stale = False
-    if patched_epoch is None or mw_start is None:
+    if not any_active:
+        # Nothing is patched into middlewared, so whether it restarted since is
+        # irrelevant -- there is nothing for it to have loaded.
+        print("  [--  ] running middlewared process — no active module; nothing to load")
+    elif patched_epoch is None or mw_start is None:
         print("  [??  ] running middlewared process — could not compare start time;")
         print("         the results above reflect the on-disk state only")
     elif mw_start + 2 < patched_epoch:
@@ -185,8 +203,8 @@ def _provider_type(cred):
     return p or "?"
 
 
-def cmd_list_credentials(client, _args):
-    creds = client("GET", "/cloudsync/credentials")
+def cmd_list_credentials(_args):
+    creds = midclt_call("cloudsync.credentials.query")
     if not creds:
         print("No cloud credentials configured.")
         return
@@ -196,8 +214,8 @@ def cmd_list_credentials(client, _args):
         print(f"{c['id']:>4}  {_provider_type(c):<14}  {c['name']}")
 
 
-def cmd_list_tasks(client, _args):
-    tasks = client("GET", "/cloud_backup")
+def cmd_list_tasks(_args):
+    tasks = midclt_call("cloud_backup.query")
     if not tasks:
         print("No TrueCloud Backup tasks configured.")
         return
@@ -209,7 +227,38 @@ def cmd_list_tasks(client, _args):
         print(f"{t['id']:>4}  {enabled:<8}  {ptype:<14}  {t.get('description', '')}")
 
 
-def cmd_create(client, args):
+def _resolve_password(args):
+    """Get the restic repo password without writing it to the user's shell history.
+
+    That password is the key to the whole backup repository. `--password <secret>`
+    persists it in ~/.bash_history and exposes it in `ps` for the lifetime of the
+    shell command, so it is accepted but warned about; stdin and an interactive
+    prompt are the safe paths.
+    """
+    if args.password_stdin:
+        if args.password:
+            print("ERROR: use either --password or --password-stdin, not both.",
+                  file=sys.stderr)
+            sys.exit(1)
+        password = sys.stdin.readline().rstrip("\n")
+    elif args.password:
+        print(
+            "WARNING: --password puts the restic repository password in your shell\n"
+            "         history. Prefer:  echo -n 'pw' | ... --password-stdin",
+            file=sys.stderr,
+        )
+        password = args.password
+    else:
+        password = getpass.getpass("Restic repository password: ")
+
+    if not password:
+        print("ERROR: the restic repository password must not be empty.",
+              file=sys.stderr)
+        sys.exit(1)
+    return password
+
+
+def cmd_create(args):
     parts = args.schedule.split()
     if len(parts) != 5:
         print(
@@ -219,6 +268,8 @@ def cmd_create(client, args):
         sys.exit(1)
     minute, hour, dom, month, dow = parts
 
+    password = _resolve_password(args)
+
     body = {
         "description": args.name,
         "path": args.path,
@@ -227,7 +278,7 @@ def cmd_create(client, args):
             "bucket": args.bucket,
             "folder": args.folder,
         },
-        "password": args.password,
+        "password": password,
         "keep_last": args.keep_last,
         "transfer_setting": args.transfer_setting,
         "schedule": {
@@ -242,7 +293,18 @@ def cmd_create(client, args):
         "enabled": not args.disabled,
     }
 
-    result = client("POST", "/cloud_backup", body)
+    if args.cache_path:
+        body["cache_path"] = args.cache_path
+    else:
+        print(
+            "WARNING: no --cache-path given. TrueNAS will run restic with --no-cache, "
+            "which is very slow for large repositories (it re-reads all repo metadata "
+            "from the provider every run). Set --cache-path to a writable dir on a pool "
+            "with free space.",
+            file=sys.stderr,
+        )
+
+    result = midclt_call("cloud_backup.create", body)
     try:
         print(f"Created task id={result['id']}  name={result['description']!r}")
     except (KeyError, TypeError):
@@ -258,14 +320,12 @@ def main():
         epilog=__doc__.split("Examples")[1] if __doc__ and "Examples" in __doc__ else "",
     )
     p.add_argument("--version", "-V", action="version", version=f"truecloud-patch {__version__}")
-    p.add_argument("--host", default=None, metavar="HOST",
-                   help="TrueNAS hostname or IP address (required except for verify)")
-    p.add_argument("--api-key", default=None, metavar="KEY",
-                   help="TrueNAS API key — System → API Keys (required except for verify)")
-    p.add_argument("--insecure", action="store_true",
-                   help="Skip TLS certificate verification (self-signed certs). "
-                        "WARNING: exposes your API key to network interception. "
-                        "Prefer adding your cert to the trust store instead.")
+    # Deprecated & ignored: the tool now uses the local middleware via `midclt` (the
+    # /api/v2.0 REST API is removed in TrueNAS 26.04), so it must run ON the TrueNAS
+    # host and needs no host/API key. Kept accepted-but-ignored for compatibility.
+    p.add_argument("--host", default=None, help=argparse.SUPPRESS)
+    p.add_argument("--api-key", default=None, help=argparse.SUPPRESS)
+    p.add_argument("--insecure", action="store_true", help=argparse.SUPPRESS)
 
     sub = p.add_subparsers(dest="cmd", required=True)
 
@@ -284,12 +344,20 @@ def main():
                    help="Bucket (S3) or container (B2) name")
     c.add_argument("--folder",      default="",
                    help="Path within the bucket (default: root)")
-    c.add_argument("--password",    required=True,
-                   help="Restic repository encryption password (choose a strong one)")
+    c.add_argument("--password",    default=None,
+                   help="Restic repository password. UNSAFE: it lands in your shell "
+                        "history. Prefer --password-stdin, or omit both and be prompted.")
+    c.add_argument("--password-stdin", action="store_true",
+                   help="Read the restic repository password from stdin (recommended)")
     c.add_argument("--keep-last",   type=int, default=14, metavar="N",
                    help="Snapshots to retain after each run (default: 14)")
     c.add_argument("--schedule",    default="0 2 * * *",
                    help="Cron schedule (default: '0 2 * * *' — daily at 02:00)")
+    c.add_argument("--cache-path",  default="", metavar="PATH",
+                   help="restic cache directory (e.g. /mnt/pool/.restic-cache). "
+                        "STRONGLY recommended: without it TrueNAS runs restic with "
+                        "--no-cache, which re-fetches all repo metadata from the "
+                        "provider every run and is extremely slow on large repos.")
     c.add_argument("--transfer-setting",
                    choices=["DEFAULT", "PERFORMANCE", "FAST_STORAGE"],
                    default="DEFAULT",
@@ -307,16 +375,17 @@ def main():
         cmd_verify()
         return
 
-    if not args.host or not args.api_key:
-        p.error("--host and --api-key are required for this command")
+    if args.host or args.api_key or args.insecure:
+        print("NOTE: --host/--api-key/--insecure are deprecated and ignored; this tool "
+              "now uses the local middleware (midclt) and must run on the TrueNAS host.",
+              file=sys.stderr)
 
-    client = make_client(args.host, args.api_key, args.insecure)
     if args.cmd == "list-credentials":
-        cmd_list_credentials(client, args)
+        cmd_list_credentials(args)
     elif args.cmd == "list-tasks":
-        cmd_list_tasks(client, args)
+        cmd_list_tasks(args)
     elif args.cmd == "create":
-        cmd_create(client, args)
+        cmd_create(args)
 
 
 if __name__ == "__main__":

patch/mw_patch.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Apply and revert truecloud-patch's blocks in middlewared's modules.
+
+Every patch this project makes to a middlewared module is an appended block that
+begins with the MARKER line. That makes patching idempotent (truncate at the
+marker, re-append) and reverting exact (truncate at the marker, stop).
+
+This is the single implementation of that. It used to live in two places --
+apply.sh's heredoc and an inline heredoc in uninstall.sh -- and the uninstall copy
+was the untested one.
+
+    python3 mw_patch.py revert-all       # remove every block + the nested module
+    python3 mw_patch.py revert-nested    # remove only the nested module's blocks
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+MARKER = "\n# TRUECLOUD_PATCH"
+
+#: Modules the providers module (B2/S3) patches.
+PROVIDER_RELPATHS = [
+    ("rclone", "remote", "b2.py"),
+    ("plugins", "cloud_backup", "restic.py"),
+]
+
+#: Modules the nested-snapshot module patches. Order matters on revert -- see
+#: revert(): the loadable module goes first.
+NESTED_RELPATHS = [
+    ("plugins", "cloud", "crud.py"),
+    ("plugins", "cloud_backup", "sync.py"),
+    ("plugins", "cloud", "snapshot.py"),
+]
+
+#: The importable module the nested blocks depend on.
+NESTED_MODULE = ("plugins", "cloud", "_truecloud_nested.py")
+
+#: The update-available alert source. Not a "patch" (it appends nothing to a stock
+#: file), but it is a file we install into middlewared and must therefore remove.
+ALERT_MODULE = ("alert", "source", "truecloud_patch_update.py")
+
+
+def patch_file(path, block):
+    """Append `block`, replacing any block we appended before. Idempotent."""
+    with open(path, encoding="utf-8") as fh:
+        content = fh.read()
+    idx = content.find(MARKER)
+    base = content[:idx] if idx != -1 else content
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write(base.rstrip("\n") + "\n" + block)
+
+
+def unpatch_file(path):
+    """Strip our appended block, restoring the stock file. True if it was patched."""
+    try:
+        with open(path, encoding="utf-8") as fh:
+            content = fh.read()
+    except OSError:
+        return False
+    idx = content.find(MARKER)
+    if idx == -1:
+        return False
+    try:
+        with open(path, "w", encoding="utf-8") as fh:
+            fh.write(content[:idx].rstrip("\n") + "\n")
+    except OSError:
+        return False
+    return True
+
+
+def revert(mw_dir, relpaths, module_relpath=None):
+    """Remove our blocks from `relpaths`, and the module at `module_relpath`.
+
+    The module is deleted FIRST. Every injected block is guarded by
+    `if _tc_nested is not None`, so once the module is gone the blocks all no-op
+    even if a later unpatch fails -- the stock guard comes back regardless.
+
+    Returns the names of what was actually reverted.
+    """
+    reverted = []
+
+    if module_relpath:
+        try:
+            os.unlink(os.path.join(mw_dir, *module_relpath))
+            reverted.append(module_relpath[-1])
+        except OSError:
+            pass
+
+    for rel in relpaths:
+        if unpatch_file(os.path.join(mw_dir, *rel)):
+            reverted.append(rel[-1])
+
+    return reverted
+
+
+def revert_nested(mw_dir):
+    """Undo the nested-snapshot patch only. Leaves the providers patch alone.
+
+    restic.py also carries a block, but it belongs to the providers module --
+    reverting it would silently break B2 backups.
+    """
+    return revert(mw_dir, NESTED_RELPATHS, NESTED_MODULE)
+
+
+def revert_all(mw_dir):
+    """Undo every patch this project applies, and remove every file it installs."""
+    reverted = revert(mw_dir, NESTED_RELPATHS + PROVIDER_RELPATHS, NESTED_MODULE)
+    try:
+        os.unlink(os.path.join(mw_dir, *ALERT_MODULE))
+        reverted.append(ALERT_MODULE[-1])
+    except OSError:
+        pass
+    return reverted
+
+
+def find_middlewared_dir():
+    """Directory of the installed `middlewared` package, or None."""
+    try:
+        import middlewared
+    except ImportError:
+        return None
+    return os.path.dirname(os.path.abspath(middlewared.__file__))
+
+
+def main(argv):
+    if len(argv) < 2 or argv[1] not in ("revert-all", "revert-nested"):
+        print(__doc__, file=sys.stderr)
+        return 2
+
+    mw_dir = find_middlewared_dir()
+    if mw_dir is None:
+        print("  middlewared not importable — nothing to revert.")
+        return 0
+
+    fn = revert_all if argv[1] == "revert-all" else revert_nested
+    reverted = fn(mw_dir)
+    if reverted:
+        print("  Reverted: " + ", ".join(reverted))
+    else:
+        print("  Nothing to revert (overlay already removed, or never patched).")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

patch/patch_ui.py

@@ -9,8 +9,8 @@ in the minified JS in one of two forms depending on TrueNAS / Angular version:
   TrueNAS 24.x (static inline array):
       "filterByProviders",["STORJ_IX"]
 
-  TrueNAS 25.x+ (Angular pureFunction binding):
-      "filterByProviders",pe(115,Rn,i.CloudSyncProviderName.Storj)
+  TrueNAS 25.x+ (Angular pureFunction binding, inside a chained property call):
+      c(2,"filterByProviders",pe(115,Rn,i.CloudSyncProviderName.Storj))("required",!0)
 
 Both are replaced so the dropdown includes S3 and B2. The file is backed up
 before modification so uninstall.sh can restore it.
@@ -19,6 +19,7 @@ Safe to run multiple times — a marker string detects an already-patched file.
 Exits 0 in all cases (warnings are printed to stdout and logged by apply.sh).
 """
 
+import contextlib
 import os
 import re
 import shutil
@@ -32,11 +33,21 @@ WEBUI_CANDIDATES = [
 # Patterns tried in order; the first match wins.
 # Each entry is (compiled_regex, replacement_string).
 _PATTERNS = [
-    # TrueNAS 25.x+: Angular emits a pureFunction call instead of a literal array.
-    # pe / slot-index / factory-var / component-var are all minified and change
-    # across builds; CloudSyncProviderName.Storj is stable (TypeScript enum name).
-    (re.compile(r'("filterByProviders",)\w+\(\d+,\w+,\w+\.CloudSyncProviderName\.Storj\)'),
-     r'\1["STORJ_IX","S3","B2"]'),
+    # TrueNAS 25.x+: Angular emits a pureFunction call instead of a literal array,
+    # inside a CHAINED property binding — so the call is followed by two closing
+    # parens, one for pe(...) and one for the property(...) it sits in:
+    #
+    #     c(2,"filterByProviders",pe(115,Rn,i.CloudSyncProviderName.Storj))("required",!0)
+    #                                                                    ^^
+    # The pattern consumes both and re-emits one, leaving the paren balance
+    # unchanged. Getting that wrong is a syntax error in the bundle and the whole
+    # web UI goes blank — see tests/test_patch_ui.py.
+    #
+    # The minified names (pe / slot index / Rn / i) change across builds;
+    # CloudSyncProviderName.Storj is stable because it is a TypeScript enum name.
+    (re.compile(r'("filterByProviders",)\w+\(\d+,\w+,\w+\.CloudSyncProviderName\.Storj\)\)'),
+     r'\1["STORJ_IX","S3","B2"])'),
+
     # TrueNAS 24.x and earlier: static inline array.
     (re.compile(r'("filterByProviders",)\["STORJ_IX"\]'),
      r'\1["STORJ_IX","S3","B2"]'),
@@ -54,6 +65,11 @@ def _match_pattern(content):
     return None, None
 
 
+def _paren_delta(s):
+    """Net parenthesis balance. Patching must not change it — see main()."""
+    return s.count("(") - s.count(")")
+
+
 def find_bundle():
     """
     Search WEBUI_CANDIDATES for the JS chunk containing the filterByProviders
@@ -124,6 +140,24 @@ def main():
         )
         return
 
+    # Never write JS whose parentheses we have unbalanced. A pattern that eats one
+    # paren too many is a syntax error in the bundle and the entire TrueNAS web UI
+    # goes blank -- and because MARKER is then present, every later run reports
+    # "already patched" and skips, so the patch cannot heal itself. Recovery means
+    # hand-restoring the .pre-truecloud-patch backup.
+    #
+    # This is not hypothetical: it shipped once. Refuse instead.
+    if _paren_delta(patched) != _paren_delta(content):
+        print(
+            "[truecloud-patch] ERROR: the replacement would unbalance the bundle's "
+            "parentheses — refusing to write.\n"
+            "[truecloud-patch] The UI is UNCHANGED and still works. This means the "
+            "pattern no longer fits this TrueNAS build.\n"
+            "[truecloud-patch] File an issue at "
+            "https://github.com/sudolulo/truenas-truecloud-patch"
+        )
+        return
+
     tmp = path + ".tmp"
     try:
         with open(tmp, "w", encoding="utf-8") as fh:
@@ -131,10 +165,8 @@ def main():
         os.replace(tmp, path)
     except OSError as exc:
         print(f"[truecloud-patch] ERROR: Could not write {path}: {exc}")
-        try:
+        with contextlib.suppress(OSError):
             os.unlink(tmp)
-        except OSError:
-            pass
         return
 
     print(f"[truecloud-patch] UI bundle patched ({count} replacement(s)): {path}")

patch/truecloud_nested.py

@@ -0,0 +1,636 @@
+"""Nested-dataset snapshot support for TrueCloud Backup.
+
+Why this exists
+---------------
+Stock TrueNAS refuses ``snapshot = true`` when the backup path contains child
+datasets::
+
+    This option is only available for datasets that have no further nesting
+
+That guard is *correct* and it is not laziness. ``plugins/cloud/snapshot.py``
+already takes a **recursive** ZFS snapshot, but it then points the backup tool
+at the *parent* dataset's ``.zfs/snapshot/<snap>/`` directory -- and ZFS does
+not expose child datasets through a parent's snapshot directory::
+
+    /mnt/Tap/.zfs/snapshot/<snap>/apps/                 ->  0 entries
+    /mnt/Tap/apps/lidarr/config/.zfs/snapshot/<snap>/   ->  the real data
+
+So without the guard, the backup tool would walk a near-empty tree, report
+SUCCESS, and upload almost nothing. A backup that lies about succeeding is the
+worst failure a backup system can have, so middleware refuses the config
+instead.
+
+This module implements the missing half: after the (already recursive) snapshot
+is taken, every descendant dataset's *own* ``.zfs/snapshot/<snap>`` directory is
+bind-mounted into a staging tree that mirrors the original layout. The backup
+tool is then pointed at the staging root, which is a complete, consistent,
+point-in-time view of the whole subtree.
+
+Cardinal safety rule
+--------------------
+**If the tree cannot be staged completely, fail loudly.** Never return a partial
+tree. Silently backing up an incomplete tree is precisely the failure this
+feature exists to prevent, and it would be worse than not having the feature.
+
+Snapshot lifecycle -- read this before changing anything
+--------------------------------------------------------
+``zfs.snapshot.delete`` defaults to ``recursive=False``, and stock
+``restic_backup()`` calls it with no options. Stock gets away with that because
+its validation means ``recursive`` is never actually True in the field. Enabling
+nested datasets makes recursive snapshots real, so the parent
+(``Tap@snap``) has one child snapshot per descendant dataset (160+ here).
+Deleting only the parent would orphan every child on **every successful run**.
+
+Therefore this module owns the whole lifecycle:
+
+* :func:`delete_snapshot_tree` sweeps the parent *and* every child snapshot, and
+  is idempotent -- it copes with stock's ``finally`` having already removed the
+  parent.
+* The snapshot name is recorded in a sidecar file next to the staging root, not
+  only in memory, so a middlewared restart mid-backup cannot orphan it.
+* Bind-mounting ``.zfs/snapshot/<snap>`` pins the snapshot, so stock's delete
+  fails with EBUSY and logs one benign warning; we unmount and then sweep.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import stat
+import subprocess
+
+__all__ = [
+    "STAGING_BASE",
+    "StagingError",
+    "apply_plan",
+    "cleanup_all",
+    "cleanup_task",
+    "current_mounts_under",
+    "delete_snapshot_tree",
+    "plan_staging",
+    "sidecar_for",
+    "snapshot_tree_names",
+    "stage_nested",
+    "staging_root_for",
+    "teardown",
+    "verify_staged",
+]
+
+#: Where staging trees are assembled. tmpfs; bind mounts consume no space.
+STAGING_BASE = "/run/truecloud-nested"
+
+# Which snapshot a staging tree pins is recorded ONLY in the sidecar file, never
+# also in memory. An in-process dict would be a second source of truth that a
+# middlewared restart silently empties -- and it is exactly the restart case that
+# must not orphan a 250-snapshot tree. One record, on disk, or none.
+
+
+class StagingError(Exception):
+    """Staging could not produce a complete tree. The backup must not proceed."""
+
+
+# ── pure helpers ──────────────────────────────────────────────────────────────
+
+
+def staging_root_for(name: str, base: str | None = None) -> str:
+    """Stable staging root for a task name (e.g. ``cloud_backup-5``).
+
+    ``base`` defaults to :data:`STAGING_BASE` at CALL time, not at import time --
+    a ``base=STAGING_BASE`` default would freeze the value into the function
+    object and silently ignore any later override.
+    """
+    if base is None:
+        base = STAGING_BASE
+    safe = "".join(c if (c.isalnum() or c in "-_.") else "_" for c in name)
+    # A component of "." or ".." would escape STAGING_BASE once joined.
+    if not safe or safe.strip(".") == "":
+        safe = "task"
+    return os.path.join(base, safe)
+
+
+def sidecar_for(staging_root: str) -> str:
+    """Path of the file recording which ZFS snapshot a staging tree pins."""
+    return staging_root + ".snapshot"
+
+
+def _write_sidecar(staging_root: str, snapshot: str) -> None:
+    """Record the pinned snapshot on disk. Blocking; call via run_in_thread."""
+    with contextlib.suppress(OSError):
+        os.makedirs(os.path.dirname(staging_root), exist_ok=True)
+        with open(sidecar_for(staging_root), "w", encoding="utf-8") as fh:
+            fh.write(snapshot)
+
+
+def _read_sidecar(staging_root: str) -> str | None:
+    """The snapshot a previous run recorded here, if any."""
+    try:
+        with open(sidecar_for(staging_root), encoding="utf-8") as fh:
+            return fh.read().strip() or None
+    except OSError:
+        return None
+
+
+def _remove_sidecar(staging_root: str) -> None:
+    with contextlib.suppress(OSError):
+        os.unlink(sidecar_for(staging_root))
+
+
+def _depth(path: str) -> int:
+    return len([p for p in path.split("/") if p])
+
+
+def snapshot_tree_names(snapshot: str, all_names) -> list[str]:
+    """Every snapshot produced by ``zfs snapshot -r <dataset>@<snap>``.
+
+    That is the parent plus one per descendant dataset, all sharing the same
+    name after the ``@``. Pure, so the sweep logic is testable without ZFS.
+    """
+    dataset, _, snapname = snapshot.partition("@")
+    if not snapname:
+        return []
+    parent = f"{dataset}@{snapname}"
+    prefix = dataset + "/"
+    suffix = "@" + snapname
+    return [
+        n for n in all_names
+        if n == parent or (n.startswith(prefix) and n.endswith(suffix))
+    ]
+
+
+def _probe_snapdir(path):
+    """Classify a snapshot directory: ``ok``, ``missing``, or why it is unusable.
+
+    ``os.path.isdir()`` collapses "does not exist" and "cannot stat" into the
+    same ``False``, so an EACCES would report itself as "has no snapshot" and
+    send someone hunting for a snapshot that is sitting right there. Both cases
+    still abort the backup -- but it has to say which one.
+    """
+    try:
+        st = os.stat(path)
+    except FileNotFoundError:
+        return "missing"
+    except OSError as e:
+        return f"cannot be read ({e.strerror})"
+    return "ok" if stat.S_ISDIR(st.st_mode) else "is not a directory"
+
+
+def plan_staging(base_dataset, base_mountpoint, path, snapshot_name, datasets,
+                 staging_root, probe=_probe_snapdir):
+    """Compute the bind-mount plan for staging a nested tree. Pure function.
+
+    ``datasets`` is a list of dicts shaped like ``zfs.dataset.query`` results:
+    ``{"name": str, "properties": {"mountpoint": {"value": str},
+    "mounted": {"value": "yes"|"no"}}}``.
+
+    Returns ``(mounts, skipped)`` where ``mounts`` is an ordered list of
+    ``(source, target)`` pairs (parents before children) and ``skipped`` is a
+    list of ``(dataset_name, reason)`` covering only datasets that are *in
+    scope* -- i.e. descendants of ``base_dataset``. Datasets elsewhere on the
+    system are ignored silently; reporting them would bury the ones that matter.
+
+    Raises StagingError if an in-scope descendant holds data we would otherwise
+    silently omit.
+    """
+    def snapdir(mountpoint):
+        return os.path.join(mountpoint, ".zfs", "snapshot", snapshot_name)
+
+    # Root of the staging tree: the backup path as seen inside the base
+    # dataset's own snapshot.
+    rel = os.path.relpath(path, base_mountpoint)
+    root_src = snapdir(base_mountpoint)
+    if rel != ".":
+        root_src = os.path.join(root_src, rel)
+
+    mounts = [(root_src, staging_root)]
+    skipped = []
+
+    ds_prefix = base_dataset.rstrip("/") + "/"
+    path_prefix = path.rstrip("/") + "/"
+
+    for ds in datasets:
+        name = ds.get("name", "")
+        # Scope by DATASET NAME, not mountpoint: a dataset with no mountpoint
+        # cannot be scoped by path, and scoping by path first would drag in
+        # every mountpoint-less dataset on the box (all of Tank/.system/*, ...).
+        if not name.startswith(ds_prefix):
+            continue
+
+        props = ds.get("properties", {})
+        mp = props.get("mountpoint", {}).get("value", "")
+
+        if not mp or mp in ("none", "legacy", "-"):
+            skipped.append((name, f"mountpoint is {mp or 'unset'}"))
+            continue
+
+        if not mp.startswith(path_prefix):
+            # A descendant dataset mounted outside the backed-up path is
+            # genuinely not part of this tree. Not an omission.
+            continue
+
+        if props.get("mounted", {}).get("value", "yes") == "no":
+            # An unmounted (e.g. locked/encrypted) dataset contributes nothing to
+            # the live tree either, so skipping matches stock semantics -- but it
+            # is a real gap and must be visible, never silent.
+            skipped.append((name, "dataset is not mounted (locked/encrypted?)"))
+            continue
+
+        src = snapdir(mp)
+        status = probe(src)
+        if status != "ok":
+            # Either the recursive snapshot missed this dataset, or we cannot read
+            # it. Either way its data would be silently omitted. Refuse -- but say
+            # WHICH, because "no snapshot" and "permission denied" send you to
+            # completely different places.
+            detail = (
+                f"has no snapshot {snapshot_name!r}" if status == "missing"
+                else f"snapshot {snapshot_name!r} {status}"
+            )
+            raise StagingError(
+                f"dataset {name!r} {detail} at {src!r}; "
+                f"refusing to back up an incomplete tree"
+            )
+
+        mounts.append((src, os.path.join(staging_root, os.path.relpath(mp, path))))
+
+    # Parents before children, so each mountpoint exists before we mount onto it.
+    mounts.sort(key=lambda m: _depth(m[1]))
+    return mounts, skipped
+
+
+def current_mounts_under(root, mounts_file="/proc/self/mounts"):
+    """Mountpoints at or under ``root``, deepest first. Used for teardown."""
+    found = []
+    try:
+        with open(mounts_file, encoding="utf-8") as fh:
+            for line in fh:
+                parts = line.split()
+                if len(parts) < 2:
+                    continue
+                mp = parts[1].replace("\\040", " ").replace("\\011", "\t")
+                if mp == root or mp.startswith(root.rstrip("/") + "/"):
+                    found.append(mp)
+    except OSError:
+        return []
+    found.sort(key=_depth, reverse=True)
+    return found
+
+
+# ── mount / unmount ───────────────────────────────────────────────────────────
+
+
+def _run(cmd):
+    # List form, never shell=True: `cmd` is built from our own mount plan, so ZFS
+    # dataset names cannot inject. Runs as root by definition (it mounts).
+    return subprocess.run(  # noqa: S603
+        cmd, capture_output=True, text=True, check=False
+    )
+
+
+def apply_plan(mounts, runner=_run, isdir=os.path.isdir):
+    """Execute the bind-mount plan. Blocking; call via ``run_in_thread``.
+
+    Raises StagingError on the first failure, after rolling back what was
+    mounted -- a half-built tree must never be handed to the backup tool.
+    """
+    if not mounts:
+        raise StagingError("empty staging plan")
+
+    staging_root = mounts[0][1]
+    done = []
+    try:
+        os.makedirs(staging_root, exist_ok=True)
+        for src, target in mounts:
+            if not isdir(target):
+                # Child mountpoint dirs come from the parent snapshot, which is
+                # read-only -- we cannot mkdir them. Only the root is ours.
+                raise StagingError(f"staging target {target!r} does not exist")
+            res = runner(["mount", "--bind", src, target])
+            if res.returncode != 0:
+                raise StagingError(
+                    f"bind-mount {src!r} -> {target!r} failed: "
+                    f"{(res.stderr or '').strip() or res.returncode}"
+                )
+            done.append(target)
+    except Exception:
+        for target in reversed(done):
+            runner(["umount", "-l", target])
+        with contextlib.suppress(OSError):
+            os.rmdir(staging_root)
+        raise
+    return staging_root
+
+
+def verify_staged(mounts, ismount=os.path.ismount, listdir=os.listdir):
+    """Assert the staged tree is real and complete. Raises StagingError if not.
+
+    This is the anti-regression guard: it is what stops this feature from ever
+    degrading back into the silently-empty backup that the stock validation
+    refuses to allow.
+    """
+    if not mounts:
+        raise StagingError("nothing was staged")
+
+    staging_root = mounts[0][1]
+    for _src, target in mounts:
+        if not ismount(target):
+            raise StagingError(f"staging target {target!r} is not a mountpoint")
+
+    try:
+        if not listdir(staging_root):
+            raise StagingError(f"staging root {staging_root!r} is empty")
+    except OSError as e:
+        raise StagingError(f"staging root {staging_root!r} unreadable: {e}") from e
+
+    return True
+
+
+def teardown(staging_root, runner=_run, mounts_file="/proc/self/mounts"):
+    """Unmount the staging tree (deepest first) and remove the root.
+
+    Idempotent, and does not depend on an in-memory plan -- so it also cleans up
+    leftovers from a crashed run.
+    """
+    errors = []
+    for mp in current_mounts_under(staging_root, mounts_file=mounts_file):
+        res = runner(["umount", mp])
+        if res.returncode != 0:
+            res = runner(["umount", "-l", mp])  # lazy: better than leaking
+            if res.returncode != 0:
+                errors.append(f"{mp}: {(res.stderr or '').strip()}")
+    with contextlib.suppress(OSError):
+        os.rmdir(staging_root)
+    return errors
+
+
+# ── orchestration (middleware is duck-typed; no middlewared import) ───────────
+#
+# These are SYNCHRONOUS and talk to middlewared via `middleware.call_sync`, which
+# is safe from a worker thread and deadlocks on the event loop. That is the whole
+# reason this file has one implementation instead of two:
+#
+#   TrueNAS <= 25.10  cloud_backup is async. The injected wrapper is `async def` and
+#                     hands these to `await middleware.run_in_thread(...)`, which is
+#                     exactly the thread `call_sync` needs.
+#   TrueNAS >= 26     cloud_backup is synchronous and already runs in middlewared's
+#                     thread pool (its own code calls `call_sync`). The injected
+#                     wrapper calls these directly.
+#
+# So the async/sync difference lives entirely in the three injected blocks, and the
+# logic below -- the part with the snapshots, the bind mounts and the failure modes
+# -- is written once. Duplicating it as an async twin would mean every future fix
+# had to be made twice, and the one that got missed would be the one that eats a
+# backup.
+
+
+def get_dataset_recursive(datasets, directory):
+    """The dataset containing `directory`, and whether anything is nested under it.
+
+    Vendored from middlewared's own plugins/cloud/snapshot.py (TrueNAS <= 25.10),
+    because TrueNAS 26 DELETED it -- create_snapshot there uses filesystem.statfs
+    instead. The injected block used to call it out of the host module's namespace,
+    which on 26 is a straight NameError.
+
+    Carrying our own copy removes the dependency on both versions rather than adding
+    an assumption about it. It is ~10 lines of pure list arithmetic over data we
+    already have in hand, and it has no reason to change.
+
+    Returns (dataset, has_children):
+      dataset      -- the DEEPEST dataset whose mountpoint is a prefix of `directory`
+      has_children -- whether any OTHER dataset is mounted beneath `directory`
+    """
+    datasets = [
+        dict(dataset, prefixlen=len(
+            os.path.dirname(os.path.commonprefix(
+                [dataset["properties"]["mountpoint"]["value"] + "/", directory + "/"]))
+        ))
+        for dataset in datasets
+        if dataset["properties"]["mountpoint"]["value"] != "none"
+    ]
+
+    dataset = sorted(
+        [
+            dataset
+            for dataset in datasets
+            if (directory + "/").startswith(dataset["properties"]["mountpoint"]["value"] + "/")
+        ],
+        key=lambda dataset: dataset["prefixlen"],
+        reverse=True,
+    )[0]
+
+    return dataset, any(
+        (ds["properties"]["mountpoint"]["value"] + "/").startswith(directory + "/")
+        for ds in datasets
+        if ds != dataset
+    )
+
+
+def delete_snapshot_tree(middleware, snapshot, logger=None):
+    """Delete the parent snapshot AND every child created by ``zfs snapshot -r``.
+
+    ``zfs.snapshot.delete`` is non-recursive by default and stock calls it with
+    no options, so relying on stock would orphan one snapshot per descendant
+    dataset on every run. Idempotent: tolerates the parent already being gone
+    (stock's ``finally`` may have won the race once our mounts were released).
+    """
+    dataset = snapshot.partition("@")[0]
+
+    # Fast path: ONE recursive delete removes the parent and every child that
+    # `zfs snapshot -r` created (252 on a real pool). Deleting them individually
+    # also works, but it is neither cheap nor atomic -- a run killed part-way
+    # through 252 sequential deletes leaves exactly the orphans this function
+    # exists to prevent.
+    try:
+        middleware.call_sync("zfs.snapshot.delete", snapshot, {"recursive": True})
+        return
+    except Exception as e:  # noqa: BLE001 - fall through to the explicit sweep
+        # Usually just "parent already gone" (stock's finally won the race once our
+        # mounts were released), which the sweep below handles. Log it rather than
+        # swallow it: if the real cause is something else, this is the only place
+        # it is visible -- the sweep would report a different, downstream failure.
+        if logger:
+            logger.debug(
+                "truecloud-patch: recursive delete of %s failed (%r); sweeping "
+                "the tree by name instead", snapshot, e,
+            )
+
+    # The parent may already be gone -- stock's `finally` can win the race once
+    # our mounts are released -- which fails the recursive delete while the
+    # children survive. Sweep them by name.
+    try:
+        snaps = middleware.call_sync(
+            "zfs.snapshot.query", [["name", "^", dataset]], {"select": ["name"]}
+        )
+        # An empty result means the tree is already gone -- delete nothing, and
+        # do not fall back to the parent, which would only log a spurious
+        # "does not exist" warning on every clean run.
+        names = snapshot_tree_names(snapshot, [s["name"] for s in snaps])
+    except Exception as e:  # noqa: BLE001 - fall back to at least the parent
+        if logger:
+            logger.warning(
+                "truecloud-patch: could not enumerate snapshot tree for %s: %r",
+                snapshot, e,
+            )
+        names = [snapshot]
+
+    for name in names:
+        try:
+            middleware.call_sync("zfs.snapshot.delete", name)
+        except Exception as e:  # noqa: BLE001 - already gone is fine
+            if logger:
+                logger.warning(
+                    "truecloud-patch: could not delete snapshot %s: %r", name, e
+                )
+
+
+def stage_nested(middleware, path, snapshot, base_dataset, base_mountpoint,
+                 task_name, datasets, logger=None):
+    """Build a complete staging tree for `path` from the already-taken `snapshot`.
+
+    `snapshot` is a full ZFS snapshot name ("Tap@cloud_backup-5-2026...").
+
+    `datasets` is the FILESYSTEM dataset list. **It MUST have been enumerated
+    AFTER `snapshot` was taken.** A list read beforehand can miss a dataset
+    created in the gap: the recursive snapshot would capture it, but the staging
+    plan would not, and its data would be silently omitted from the backup.
+    Enumerated afterwards, an unsnapshotted dataset instead trips the isdir()
+    check in plan_staging and fails the run loudly.
+
+    Returns the staging root to hand to the backup tool.
+
+    Raises StagingError if the tree cannot be staged completely -- the caller
+    must let that propagate so the backup fails instead of silently uploading a
+    partial tree. The caller is responsible for deleting `snapshot` in that case
+    (see SNAPSHOT_BLOCK in apply.sh).
+    """
+    snapshot_name = snapshot.split("@", 1)[1]
+    staging_root = staging_root_for(task_name)
+
+    # A previous run may have crashed mid-flight; never build on top of that.
+    teardown(staging_root)
+
+    # ...and if it left a sidecar behind, that snapshot tree is still on disk and
+    # nothing else will ever reclaim it. Sweep it before we overwrite the record,
+    # or a single crashed run orphans 160+ snapshots permanently.
+    stale = _read_sidecar(staging_root)
+    if stale and stale != snapshot:
+        if logger:
+            logger.warning(
+                "truecloud-patch: reclaiming snapshot tree from an earlier "
+                "interrupted run: %s", stale,
+            )
+        delete_snapshot_tree(middleware, stale, logger=logger)
+
+    # Record the snapshot BEFORE mounting anything, not after. middlewared can
+    # die at any point (this patch even schedules a restart at boot), and the
+    # sidecar is the only thing that survives it -- an in-process dict would take
+    # the sole record of a 160-snapshot tree with it. Writing it after apply_plan
+    # would leave exactly the crash window the sidecar exists to close.
+    _write_sidecar(staging_root, snapshot)
+
+    try:
+        mounts, skipped = plan_staging(
+            base_dataset, base_mountpoint, path, snapshot_name,
+            datasets, staging_root,
+        )
+        if logger:
+            for name, reason in skipped:
+                logger.warning(
+                    "truecloud-patch: not staging dataset %r: %s", name, reason
+                )
+
+        apply_plan(mounts)
+        verify_staged(mounts)
+    except Exception:
+        teardown(staging_root)
+        _remove_sidecar(staging_root)
+        raise
+
+    if logger:
+        logger.info(
+            "truecloud-patch: staged %d dataset(s) from %s at %s",
+            len(mounts), snapshot, staging_root,
+        )
+    return staging_root
+
+
+def cleanup_task(middleware, task_name, logger=None):
+    """Tear down a task's staging tree and delete the snapshot it pinned.
+
+    Safe to call unconditionally: a no-op when the task was never staged.
+    """
+    staging_root = staging_root_for(task_name)
+    snapshot = _read_sidecar(staging_root)
+
+    if snapshot is None and not os.path.isdir(staging_root):
+        return  # never staged; nothing to do
+
+    errors = teardown(staging_root)
+    if errors and logger:
+        for err in errors:
+            logger.warning("truecloud-patch: staging teardown: %s", err)
+
+    if snapshot is not None:
+        delete_snapshot_tree(middleware, snapshot, logger=logger)
+
+    _remove_sidecar(staging_root)
+
+
+# ── offline cleanup (uninstall.sh / recover.sh) ───────────────────────────────
+
+
+def cleanup_all(base=None, runner=_run, mounts_file="/proc/self/mounts",
+                glob_fn=None, read_sidecar=_read_sidecar):
+    """Tear down every staging tree. Used by uninstall.sh and recover.sh.
+
+    Those scripts must work when middlewared is dead, so they cannot go through
+    the async path -- but they must not reimplement the teardown either: the
+    depth-ordering and lazy-umount fallback are fiddly, and a second copy in
+    shell would be the untested one. This is the same tested code.
+
+    Returns ``(lines, errors)``: report lines to print, and unmount errors.
+    """
+    import glob as _glob
+
+    base = base or STAGING_BASE
+    glob_fn = glob_fn or _glob.glob
+    lines = []
+
+    # Report orphaned snapshots BEFORE removing the sidecars that name them --
+    # a sidecar is the only record that an interrupted run's snapshot tree (one
+    # snapshot per descendant dataset) is still on disk.
+    for sc in sorted(glob_fn(os.path.join(base, "*.snapshot"))):
+        snap = read_sidecar(sc[: -len(".snapshot")])
+        if snap:
+            lines.append(f"  NOTE: an interrupted backup left snapshot '{snap}' behind.")
+            lines.append(f"        Remove it and its children:  zfs destroy -r '{snap}'")
+
+    mounts = current_mounts_under(base, mounts_file=mounts_file)
+    if not mounts:
+        lines.append("  None active.")
+    for mp in mounts:
+        lines.append(f"  Unmounting: {mp}")
+
+    errors = teardown(base, runner=runner, mounts_file=mounts_file)
+    for err in errors:
+        lines.append(f"  WARNING: could not unmount {err}")
+
+    if not errors:
+        for sc in glob_fn(os.path.join(base, "*.snapshot")):
+            with contextlib.suppress(OSError):
+                os.unlink(sc)
+        with contextlib.suppress(OSError):
+            os.rmdir(base)
+
+    return lines, errors
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) > 1 and sys.argv[1] == "cleanup":
+        _lines, _errors = cleanup_all()
+        for _line in _lines:
+            print(_line)
+        sys.exit(1 if _errors else 0)
+    print("usage: truecloud_nested.py cleanup", file=sys.stderr)
+    sys.exit(2)

patch/wait_restart.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# patch/wait_restart.sh — payload of the transient `truecloud-mw-restart`
+# unit that apply.sh schedules in boot context (Step 3).
+#
+# Why not restart middlewared directly from the unit: systemd ordering
+# (`After=multi-user.target`, used up to v0.0.4) cannot see middlewared's
+# *internal* boot work. When the boot targets are reached, two things are
+# typically still in flight inside middlewared:
+#
+#   - ix-reporting.service's `midclt call reporting.start_service` (netdata,
+#     which feeds the dashboard hardware stats), and
+#   - the docker/apps startup task middlewared creates on its own
+#     system-ready event (`docker.state.start_service`).
+#
+# Restarting middlewared while those run kills them, and nothing retries
+# them until the next boot: every app stays down (`docker.status` FAILED),
+# the dashboard shows no stats, and middleware-internal service state (e.g.
+# the SMB backend) is left uninitialized. Observed on 25.10.4 with v0.0.4.
+#
+# So this script waits for both layers to settle before restarting. Every
+# wait is bounded and fails open: worst case the restart still happens, just
+# later — a restart on a settled system is harmless (docker, apps and
+# netdata are independent processes; only the middleware API blips).
+#
+# NOTE: the unit must NOT be Type=oneshot. A oneshot's start job stays in
+# the systemd job queue until the process exits, and `is-system-running
+# --wait` below waits for that same queue to drain — the unit would deadlock
+# on itself until the timeout. apply.sh schedules this with the default
+# service type, whose start job completes at fork.
+
+# 1. systemd layer: wait for the boot job queue to drain. This covers every
+#    ix-* oneshot still activating, including ix-reporting's in-flight midclt
+#    call. The exit code is irrelevant — a "degraded" boot (any unrelated
+#    failed unit) is still a finished boot. The timeout only guards against
+#    a boot that never settles (e.g. a unit stuck on a network wait).
+timeout 900 systemctl is-system-running --wait > /dev/null 2>&1
+
+# 2. middlewared layer: poll the docker state machine until it leaves the
+#    transitional states (PENDING/INITIALIZING/STOPPING/MIGRATING — see
+#    middlewared/plugins/docker/state_utils.py). An empty answer means
+#    midclt could not respond at all; keep waiting. Cap at 10 minutes.
+for _ in $(seq 1 120); do
+    _status=$(midclt call docker.status 2>/dev/null \
+                  | grep -oE '"status": "[A-Z_]+"' | cut -d'"' -f4)
+    case "$_status" in
+        RUNNING|STOPPED|UNCONFIGURED|FAILED|MIGRATION_FAILED) break ;;
+    esac
+    sleep 5
+done
+
+# 3. Grace period for middleware-internal ready-event tasks that expose no
+#    queryable state (smb.configure and friends). Bounded insurance.
+sleep 30
+
+exec systemctl try-restart middlewared

pyproject.toml

@@ -0,0 +1,15 @@
+# Tooling config only — this project is not a Python package. The patch modules
+# are copied into middlewared's site-packages by patch/apply.sh at boot.
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "B", "UP", "SIM"]
+ignore = [
+  "E501",  # long lines in explanatory comments are fine
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

recover.sh

@@ -17,10 +17,11 @@
 #   bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh
 #   systemctl restart middlewared
 
-VERSION="0.0.4"
+VERSION="0.6.0"
 
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 
+
 echo "=== TrueNAS TrueCloud Provider Patch v${VERSION} — Recover ==="
 echo ""
 
@@ -52,6 +53,14 @@ for _tag in mw ui; do
 done
 [ "$_any" -eq 0 ] && echo "  No overlays active."
 
+# Nested-snapshot staging trees are bind mounts that PIN their ZFS snapshots, so
+# leaving them mounted blocks those snapshots from ever being destroyed. The
+# overlays above are volatile, but these are not self-healing without a reboot,
+# and recover.sh is expected to work without one.
+echo "Unmounting nested-snapshot staging trees ..."
+# Best-effort: never block recovery. Same tested implementation as uninstall.sh.
+python3 "$PATCH_DIR/patch/truecloud_nested.py" cleanup || true
+
 # Cancel a deferred boot restart if one is still queued — we restart ourselves.
 systemctl stop truecloud-mw-restart.service 2>/dev/null
 systemctl reset-failed truecloud-mw-restart.service 2>/dev/null

release.sh

@@ -0,0 +1,332 @@
+#!/usr/bin/env bash
+# Cut a release. Two stages, and you cannot skip the first one.
+#
+#   bash release.sh 0.6.0 --rc        stage 1: candidate. Invisible to users.
+#   bash release.sh 0.6.0 --promote   stage 2: stable. Only if an rc passed HERE.
+#
+# WHY IT WORKS THIS WAY
+#
+# This repo once cut twelve releases in a day, several of them fixing the release
+# before. Every one of those raises an update alert on every user's box. An alert
+# people learn to ignore is worse than no alert, because one day it will be
+# carrying a security fix.
+#
+# So: debugging happens across rc1, rc2, rc3 -- which update.sh and the alert
+# source both filter out, so no user ever sees them -- and a stable tag is only
+# reachable from a candidate that already went green on the identical commit.
+# tools/release_gate.py enforces that here, and .github/workflows/release.yml
+# enforces it again where it cannot be bypassed.
+#
+# Day to day you do not touch this script. You write your changes under
+# `## Unreleased` in CHANGELOG.md and push to main. Releasing is a separate,
+# deliberate act.
+
+set -euo pipefail
+
+# This file IS on every user's box -- update.sh clones the whole repo -- so it
+# carries no VERSION= not because it is "not shipped", but because nothing reads
+# it. VERSION= exists so the running system can say which patch it is; this script
+# never runs on a running system. (Anything that DOES carry a VERSION= must be in
+# release_notes.VERSIONED_FILES or it silently rots: create_task.py sat three
+# releases behind for exactly that reason.)
+#
+# Running it on a user's box is a no-op by construction, and that is checked below
+# rather than left to luck: update.sh pins the checkout to a tag in detached HEAD,
+# and this refuses to run anywhere but an up-to-date `main` with push access.
+
+cd "$(dirname "$(readlink -f "$0")")"
+
+die() { printf '\033[31merror:\033[0m %s\n' "$*" >&2; exit 1; }
+note() { printf '\033[36m==>\033[0m %s\n' "$*"; }
+ok() { printf '\033[32m ok\033[0m %s\n' "$*"; }
+
+usage() {
+  sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'
+  exit "${1:-0}"
+}
+
+# ── args ─────────────────────────────────────────────────────────────────────
+
+target=""
+mode=""
+assume_yes=0
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --rc)       mode="rc" ;;
+    --promote)  mode="promote" ;;
+    --check)    mode="check" ;;
+    -y|--yes)   assume_yes=1 ;;
+    -h|--help)  usage 0 ;;
+    -*)         die "unknown option: $1" ;;
+    *)
+      [ -n "$target" ] && die "give exactly one version"
+      target="${1#v}"
+      ;;
+  esac
+  shift
+done
+
+[ -n "$target" ] || usage 2
+[ -n "$mode" ] || die "pick a stage: --rc (candidate) or --promote (stable)"
+
+printf '%s' "$target" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$' \
+  || die "version must be plain X.Y.Z (the -rcN suffix is added for you)"
+
+# ── preflight ────────────────────────────────────────────────────────────────
+
+[ -d .git ] || die "not a git checkout"
+
+# UNTRACKED files count as dirty, because --rc runs `git add -A`: an untracked file
+# lying around would be swept into the release commit and pushed. This checkout is
+# routinely shared between sessions, so stray files are the normal state here, not
+# an exotic one. (Anything genuinely ignorable belongs in .gitignore.)
+if [ -n "$(git status --porcelain)" ]; then
+  echo "  Working tree is not clean:" >&2
+  git status --short >&2
+  die "commit, stash, or ignore the above first -- a release must be reproducible
+       from a commit, not from whatever happened to be on disk. --rc runs
+       'git add -A', so an untracked file here ships inside the release."
+fi
+
+branch="$(git rev-parse --abbrev-ref HEAD)"
+if [ "$branch" = "HEAD" ]; then
+  # This is what an INSTALLED patch looks like: update.sh pins the checkout to a
+  # release tag in detached HEAD. Someone has found this script in their clone and
+  # run it. Say so plainly rather than emitting a confusing branch error.
+  die "this is an installed checkout (detached at $(git describe --tags --always)),
+       not a development one. release.sh is the maintainer tool that publishes new
+       versions of the patch; it is not how you install or update one.
+
+       To update this box:   bash update.sh"
+fi
+[ "$branch" = "main" ] || die "releases are cut from main, not '$branch'"
+
+note "fetching tags"
+git fetch --tags --quiet origin
+
+if [ -n "$(git log --oneline "origin/$branch..$branch" 2>/dev/null)" ]; then
+  die "local main has commits that are not pushed. Push first: the tag must point
+       at a commit the world can actually fetch."
+fi
+
+# BEHIND is just as bad as ahead, and less obvious. --rc commits the version stamp,
+# creates the tag, and only THEN pushes -- so on a stale main the push is rejected
+# (non-fast-forward) *after* the tag exists and `## Unreleased` has already been
+# consumed. Re-running then sees rc1, cuts rc2, and silently skips the stamping
+# step; the rc1 tag dangles locally forever.
+if [ -n "$(git log --oneline "$branch..origin/$branch" 2>/dev/null)" ]; then
+  die "local main is BEHIND origin. Pull first:  git pull --ff-only
+       Releasing from a stale main half-completes: the tag is cut locally, the push
+       is rejected, and '## Unreleased' has already been consumed."
+fi
+
+# ── the gates: identical to the ones CI will run ─────────────────────────────
+
+run_gates() {
+  local tag="$1"
+  note "gate: versions agree, CHANGELOG is complete"
+  python3 tools/release_notes.py check "$tag" \
+    || die "content gate failed (see above)"
+  ok "content"
+
+  note "gate: provenance"
+  python3 tools/release_gate.py "$tag" -C . \
+    || die "provenance gate failed (see above)"
+  ok "provenance"
+}
+
+# ── tests, because a tag that fails its own tests is not a release ───────────
+
+# The interpreter that actually has the dev deps. A bare `python3` is usually the
+# system one with no pytest -- and "No module named pytest" would read as "tests
+# fail", i.e. the gate blocking a release for a reason that is not true.
+PY=python3
+[ -x .venv/bin/python ] && PY=.venv/bin/python
+
+RUFF=""
+if [ -x .venv/bin/ruff ]; then RUFF=.venv/bin/ruff
+elif command -v ruff >/dev/null 2>&1; then RUFF=ruff
+fi
+
+run_tests() {
+  note "running the suite"
+  "$PY" -c 'import pytest' 2>/dev/null || die "no pytest in $PY. Install the dev deps:
+       python3 -m venv .venv && .venv/bin/pip install pytest ruff"
+  "$PY" -m pytest tests -q || die "tests fail. Fix them; do not release around them."
+  [ -n "$RUFF" ] && { "$RUFF" check patch tests tools || die "lint fails"; }
+  local f
+  while IFS= read -r f; do
+    bash -n "$f" || die "bash syntax error in $f"
+  done < <(find . -name '*.sh' -not -path './.git/*')
+  ok "suite"
+}
+
+confirm() {
+  [ "$assume_yes" -eq 1 ] && return 0
+  printf '\n%s [y/N] ' "$1"
+  read -r reply </dev/tty
+  case "$reply" in [yY]*) return 0 ;; *) die "aborted" ;; esac
+}
+
+# ── check ────────────────────────────────────────────────────────────────────
+
+if [ "$mode" = "check" ]; then
+  echo
+  python3 - "$target" <<'PY'
+import sys, os
+sys.path.insert(0, os.path.join(os.getcwd(), "tools"))
+from release_notes import unreleased_body
+with open("CHANGELOG.md", encoding="utf-8") as fh:
+    body = unreleased_body(fh.read())
+if body:
+    print("Unreleased, and would ship as v%s:\n" % sys.argv[1])
+    print("\n".join("    " + line for line in body.splitlines()))
+else:
+    print("Nothing under `## Unreleased`. There is nothing to release.")
+PY
+  echo
+  next_rc="$(python3 tools/release_gate.py "$target" --next-rc -C .)"
+  echo "Next candidate would be: $next_rc"
+  exit 0
+fi
+
+# ── stage 1: release candidate ───────────────────────────────────────────────
+
+if [ "$mode" = "rc" ]; then
+  tag="$(python3 tools/release_gate.py "$target" --next-rc -C .)"
+
+  # Tests BEFORE the stamping commit, deliberately.
+  #
+  # Stamping consumes `## Unreleased` and makes a "release vX.Y.Z" commit. If the
+  # suite then failed, that commit was already on main and a re-run died inside
+  # promote() with "no `## Unreleased` content" -- the release was wedged, and the
+  # only way out was to hand-unpick a commit. Failing first leaves the tree
+  # untouched.
+  run_tests
+
+  # The first candidate promotes `## Unreleased` and stamps the version into every
+  # script. Later candidates (rc2+) are re-cuts of an already-stamped version, so
+  # they only tag -- the CHANGELOG section for this version already exists, and
+  # fixes found during rc go into it.
+  #
+  # "Already stamped" is decided by the TREE, not by the rc1 tag: if a previous run
+  # stamped and committed but died before tagging (or before pushing), the tag is
+  # absent while the stamp is present, and re-stamping would try to promote an
+  # `## Unreleased` section that is no longer there.
+  if python3 tools/release_notes.py check "v$target-rc0" >/dev/null 2>&1; then
+    note "v$target is already stamped; cutting a follow-up candidate"
+  else
+    note "promoting '## Unreleased' -> v$target and stamping the scripts"
+    python3 - "$target" <<'PY'
+import datetime, os, re, sys
+sys.path.insert(0, os.path.join(os.getcwd(), "tools"))
+from release_notes import VERSIONED_FILES, promote
+
+version = sys.argv[1]
+today = datetime.date.today().isoformat()
+
+with open("CHANGELOG.md", encoding="utf-8") as fh:
+    text = fh.read()
+try:
+    out = promote(text, version, today)
+except ValueError as e:
+    sys.exit(f"error: {e}")
+with open("CHANGELOG.md", "w", encoding="utf-8") as fh:
+    fh.write(out)
+print(f"    CHANGELOG.md  Unreleased -> v{version} - {today}")
+
+for rel in VERSIONED_FILES:
+    with open(rel, encoding="utf-8") as fh:
+        src = fh.read()
+    new, n = re.subn(
+        r'^(VERSION=|__version__\s*=\s*)"[^"]+"',
+        lambda m: f'{m.group(1)}"{version}"',
+        src, count=1, flags=re.M,
+    )
+    if not n:
+        sys.exit(f"error: {rel} has no VERSION= line to stamp")
+    if new != src:
+        with open(rel, "w", encoding="utf-8") as fh:
+            fh.write(new)
+        print(f"    {rel}  -> {version}")
+PY
+    git add -A
+    git commit -q -m "release v$target"
+    ok "stamped"
+  fi
+
+  # Gated as the rc tag it is: content is checked against the base version, and the
+  # provenance gate is a no-op for candidates -- being one is the whole point.
+  # (run_tests already ran, above, before anything was committed.)
+  run_gates "$tag"
+
+  echo
+  note "about to cut $tag"
+  echo "    commit:  $(git rev-parse --short HEAD)  $(git log -1 --format=%s)"
+  echo
+  echo "    A candidate is invisible to users: update.sh and the update alert both"
+  echo "    ignore -rc tags. Install it on a real box, exercise it, and only then"
+  echo "    run:  bash release.sh $target --promote"
+  confirm "cut $tag?"
+
+  # Push the branch FIRST. If it is rejected, no tag has been created yet -- a tag
+  # pointing at a commit nobody else has is worse than no tag, because the next run
+  # sees it, counts it as a candidate, and cuts rc2 against a commit that was never
+  # published.
+  git push --quiet origin main
+  git tag -a "$tag" -m "$tag"
+  git push --quiet origin "$tag"
+  ok "pushed $tag"
+  echo
+  echo "CI is now testing $tag and publishing it as a PRE-RELEASE."
+  echo "When you are satisfied:  bash release.sh $target --promote"
+  exit 0
+fi
+
+# ── stage 2: promote to stable ───────────────────────────────────────────────
+
+if [ "$mode" = "promote" ]; then
+  tag="v$target"
+
+  if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then
+    next="$(echo "$target" | awk -F. '{printf "%d.%d.%d", $1, $2, $3+1}')"
+    die "$tag already exists. A published version is immutable -- if it is broken,
+       the fix ships as v$next, and it goes through a candidate like everything else."
+  fi
+
+  # The barrier. Fails unless an rc points at THIS commit.
+  note "gate: was this exact commit a release candidate?"
+  python3 tools/release_gate.py "$tag" -C . || {
+    echo
+    die "not promotable (see above)"
+  }
+  ok "provenance"
+
+  run_gates "$tag"
+  run_tests
+
+  rcs="$(python3 - "$target" <<'PY'
+import os, sys
+sys.path.insert(0, os.path.join(os.getcwd(), "tools"))
+from release_gate import rc_tags
+print(", ".join(rc_tags(sys.argv[1])) or "none")
+PY
+)"
+
+  echo
+  note "about to publish $tag to every user"
+  echo "    commit:      $(git rev-parse --short HEAD)"
+  echo "    candidates:  $rcs"
+  echo
+  echo "    This raises an update alert on every installed box (unless the only"
+  echo "    CHANGELOG section is Docs). Make sure it is worth interrupting people."
+  confirm "publish $tag?"
+
+  git tag -a "$tag" -m "$tag"
+  git push --quiet origin "$tag"
+  ok "pushed $tag"
+  echo
+  echo "CI is publishing the release. Users will be alerted within 24h."
+  exit 0
+fi

tests/test_alert_source.py

@@ -0,0 +1,132 @@
+"""Tests for the update-available alert source.
+
+This file is imported by middlewared's `alert.load()`, which runs at STARTUP and
+has **no try/except**:
+
+    def load(self):
+        for module in load_modules(.../alert/source):
+            for cls in load_classes(module, AlertSource, (ThreadedAlertSource,)):
+                source = cls(self.middleware)
+                if source.name in ALERT_SOURCES:
+                    raise RuntimeError(...)
+
+So a module that raises on import takes middlewared's setup down with it. These
+tests guard the realistic ways that could happen.
+"""
+
+import ast
+import os
+import re
+
+import pytest
+
+ALERT_SRC = os.path.join(os.path.dirname(__file__), "..", "patch", "alert_source.py")
+APPLY_SH = os.path.join(os.path.dirname(__file__), "..", "patch", "apply.sh")
+
+
+def source():
+    with open(ALERT_SRC, encoding="utf-8") as fh:
+        return fh.read()
+
+
+def tree():
+    return ast.parse(source())
+
+
+class TestCannotBreakMiddlewaredAtImport:
+    def test_it_compiles(self):
+        compile(source(), "alert_source.py", "exec")
+
+    def test_apply_sh_compiles_it_before_writing_it(self):
+        # The substituted file is what middlewared imports. If it does not compile,
+        # installing it would break startup — so apply.sh must refuse to write it.
+        with open(APPLY_SH, encoding="utf-8") as fh:
+            sh = fh.read()
+        i = sh.index("alert_dst = os.path.join(mw_dir, 'alert', 'source'")
+        block = sh[i:i + 1600]
+        assert "compile(_body, alert_dst, 'exec')" in block
+        assert block.index("compile(_body") < block.index("open(alert_dst, 'w'")
+
+    def test_patch_dir_is_substituted_with_repr(self):
+        # A directory containing a quote or backslash would otherwise produce a
+        # syntax error in the installed module.
+        with open(APPLY_SH, encoding="utf-8") as fh:
+            sh = fh.read()
+        assert "_body.replace('\"@PATCH_DIR@\"', repr(_patch_dir))" in sh
+
+    @pytest.mark.parametrize("path", [
+        "/mnt/tank/patch",
+        '/mnt/we"ird/patch',      # a quote in the path
+        "/mnt/back\\slash/patch",  # a backslash
+    ])
+    def test_substituted_module_compiles_for_awkward_paths(self, path):
+        body = source().replace('"@PATCH_DIR@"', repr(path))
+        compile(body, "alert_source.py", "exec")  # must not raise
+
+    def test_no_io_at_module_import_time(self):
+        # Anything at module scope runs during alert.load(). Only imports,
+        # constants and class definitions are allowed.
+        allowed = (ast.Import, ast.ImportFrom, ast.Assign, ast.AnnAssign,
+                   ast.ClassDef, ast.FunctionDef, ast.Expr)
+        for node in tree().body:
+            assert isinstance(node, allowed), f"module-level {type(node).__name__}"
+            if isinstance(node, ast.Expr):
+                assert isinstance(node.value, ast.Constant), "only the docstring"
+
+
+class TestAlertClassNaming:
+    """middlewared's AlertClassMeta raises NameError unless the name ends in
+    'AlertClass' — at import, inside alert.load(), which has no try/except."""
+
+    def alert_classes(self):
+        return [n for n in tree().body
+                if isinstance(n, ast.ClassDef)
+                and any(getattr(b, "id", "") == "AlertClass" for b in n.bases)]
+
+    def test_there_are_alert_classes(self):
+        assert self.alert_classes()
+
+    def test_every_alert_class_name_ends_in_AlertClass(self):
+        for cls in self.alert_classes():
+            assert cls.name.endswith("AlertClass"), (
+                f"{cls.name}: AlertClassMeta raises NameError on this"
+            )
+
+    def test_every_alert_class_defines_the_required_attrs(self):
+        # category/level/title are NotImplemented on the base; a missing one shows
+        # up as a broken alert rather than an error.
+        for cls in self.alert_classes():
+            names = {t.id for n in cls.body if isinstance(n, ast.Assign)
+                     for t in n.targets if isinstance(t, ast.Name)}
+            assert {"category", "level", "title", "text"} <= names, cls.name
+
+    def test_alert_text_placeholders_match_the_args_we_pass(self):
+        src = source()
+        placeholders = set(re.findall(r"%\((\w+)\)s", src))
+        # These are the keys built in _check().
+        assert placeholders <= {"current", "latest", "summary", "dir"}
+
+
+class TestNoSysPathMutation:
+    def test_release_notes_is_loaded_by_path_not_sys_path(self):
+        # sys.path.insert(0, ...) would shadow the stdlib for this interpreter, and
+        # ThreadedAlertSource runs in a thread pool — mutating sys.path is a race.
+        #
+        # Check for actual MUTATION, not the string: the docstring legitimately
+        # mentions sys.path to explain why it is avoided.
+        src = source()
+        assert "sys.path.insert" not in src
+        assert "sys.path.append" not in src
+        assert not re.search(r"^import sys$", src, re.M), "sys is not needed"
+        assert "spec_from_file_location" in src
+
+
+class TestNeverWritesToGit:
+    def test_only_read_only_git_commands(self):
+        # A `git fetch` from middlewared (running as root) would leave root-owned
+        # objects in .git and break every later non-root git command — which is
+        # exactly the breakage this project already hit once.
+        src = source()
+        for forbidden in ("fetch", "pull", "checkout", "clone", "reset"):
+            assert f'"{forbidden}"' not in src, f"git {forbidden} writes to .git"
+        assert '"ls-remote"' in src

tests/test_apply_blocks.py

@@ -0,0 +1,445 @@
+"""The *_BLOCK strings in apply.sh are Python source injected into middleware.
+
+A syntax error in one of them would be appended to a live middlewared module and
+break the box at boot. They are string literals, so nothing type-checks them --
+these tests do.
+"""
+
+import ast
+import os
+import re
+import textwrap
+
+import pytest
+
+APPLY_SH = os.path.join(os.path.dirname(__file__), "..", "patch", "apply.sh")
+
+#: Every block that is actually injected into a middlewared module.
+#:
+#: The three nested blocks come in two flavours. TrueNAS <= 25.10 has an ASYNC
+#: cloud_backup path; TrueNAS 26 rewrote it synchronous. apply.sh reads which one is
+#: installed and injects the matching wrapper -- an `async def` on 26 would hand
+#: sync.py a coroutine where it unpacks a tuple, and a plain `def` on 25.10 would
+#: block the event loop. Both flavours must therefore be valid Python, always.
+EXPECTED_BLOCKS = {
+    "B2_BLOCK",
+    "RESTIC_BLOCK",
+    "SNAPSHOT_ASYNC",
+    "SNAPSHOT_SYNC",
+    "CRUD_ASYNC",
+    "CRUD_SYNC",
+    "SYNC_ASYNC",
+    "SYNC_SYNC",
+}
+
+NESTED_BLOCKS = ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC", "CRUD_ASYNC", "CRUD_SYNC",
+                 "SYNC_ASYNC", "SYNC_SYNC"]
+
+
+def heredoc_source():
+    with open(APPLY_SH, encoding="utf-8") as fh:
+        src = fh.read()
+    m = re.search(r"<< 'PYEOF'\n(.*?)\nPYEOF", src, re.S)
+    assert m, "could not find the PYEOF heredoc in apply.sh"
+    return m.group(1)
+
+
+def extract_blocks():
+    """The blocks as apply.sh actually builds them.
+
+    EVALUATED, not read off as string literals: each nested block is a CORE
+    concatenated with a flavour-specific wrapper, so reading only `ast.Constant`
+    would silently return nothing for them -- a green suite over blocks nobody
+    checked. Assignments that need the runtime (argv, imports) simply fail to
+    evaluate and are skipped.
+    """
+    tree = ast.parse(heredoc_source())
+    ns, blocks = {}, {}
+    for node in tree.body:
+        if not isinstance(node, ast.Assign):
+            continue
+        try:
+            value = eval(  # noqa: S307 - our own shipped source, on purpose
+                compile(ast.Expression(node.value), "<blocks>", "eval"), {}, ns
+            )
+        except Exception:
+            continue
+        for tgt in node.targets:
+            if isinstance(tgt, ast.Name) and isinstance(value, str):
+                ns[tgt.id] = value
+                if tgt.id in EXPECTED_BLOCKS:
+                    blocks[tgt.id] = value
+    return blocks
+
+
+def _nested_native_detector():
+    """The REAL native-nested probe, lifted out of apply.sh.
+
+    Extracted rather than reimplemented: a reimplementation would happily pass
+    while the shipped probe stayed broken, which is precisely the bug this guards.
+    """
+    with open(APPLY_SH, encoding="utf-8") as fh:
+        sh = fh.read()
+
+    m = re.search(
+        r"^(\s*)_drop = str\.maketrans\(.*?\n\s*if 'nofurthernesting' not in "
+        r"stock_src\.translate\(_drop\):\n\s*result\['native_nested'\] = 'yes'",
+        sh, re.S | re.M,
+    )
+    assert m, "could not find the native-nested probe in apply.sh"
+
+    # The block lives inside a double-quoted shell string; undo bash's escaping.
+    body = m.group(0)
+    body = body.replace("\\\\", "\x00").replace('\\"', '"').replace("\x00", "\\")
+    body = textwrap.dedent(body)
+
+    def detect(stock_src):
+        ns = {"stock_src": stock_src, "result": {"native_nested": "no"}, "chr": chr}
+        exec(body, ns)  # noqa: S102 - executing our own shipped code, on purpose
+        return ns["result"]["native_nested"]
+
+    return detect
+
+
+def test_heredoc_itself_compiles():
+    compile(heredoc_source(), "apply.sh:PYEOF", "exec")
+
+
+def test_all_expected_blocks_present():
+    assert set(extract_blocks()) == EXPECTED_BLOCKS
+
+
+@pytest.mark.parametrize("name", sorted(EXPECTED_BLOCKS))
+def test_injected_block_is_valid_python(name):
+    block = extract_blocks()[name]
+    compile(block, f"apply.sh:{name}", "exec")
+
+
+@pytest.mark.parametrize("name", sorted(EXPECTED_BLOCKS))
+def test_injected_block_carries_the_idempotency_marker(name):
+    # patch_file() truncates each target file at "\n# TRUECLOUD_PATCH" before
+    # re-appending, so every block must start with that marker or repeated runs
+    # would stack duplicate copies into the middleware module.
+    assert extract_blocks()[name].lstrip("\n").startswith("# TRUECLOUD_PATCH")
+
+
+@pytest.mark.parametrize("name", NESTED_BLOCKS)
+def test_nested_blocks_degrade_safely_without_the_module(name):
+    # If _truecloud_nested failed to install, every nested block must no-op.
+    # Critically this includes CRUD_BLOCK: relaxing the guard without the
+    # traversal in place would mean silently-empty backups.
+    block = extract_blocks()[name]
+    assert "_tc_nested = None" in block
+    assert "if _tc_nested is not None:" in block
+
+
+class TestSnapshotLeak:
+    """zfs.snapshot.delete is non-recursive and stock calls it with no options.
+
+    A recursive snapshot has one child per descendant dataset (160+ here), so
+    every path that creates one must also sweep the whole tree.
+    """
+
+    def test_staging_failure_deletes_the_snapshot_tree(self):
+        # On a staging failure, sync.py's `snapshot, local_path = await
+        # create_snapshot(...)` never completes, so its local `snapshot` stays
+        # None and its finally deletes nothing. We must sweep it ourselves.
+        block = extract_blocks()["SNAPSHOT_ASYNC"]
+        assert "except Exception:" in block
+        assert "delete_snapshot_tree" in block
+        assert "raise" in block
+
+    def test_sync_block_cleans_up_on_every_path(self):
+        block = extract_blocks()["SYNC_ASYNC"]
+        assert "finally:" in block
+        assert "cleanup_task" in block
+
+
+def test_crud_block_is_scoped_to_cloud_backup():
+    # cloudsync has no staging teardown wired in, so its guard must stay.
+    for name in ("CRUD_ASYNC", "CRUD_SYNC"):
+        assert '!= "cloud_backup"' in extract_blocks()[name]
+
+
+class TestIndependentModules:
+    """The two modules must retire independently.
+
+    TrueNAS may ship native B2 support long before (or after) it handles nested
+    datasets. A single all-or-nothing kill switch would silently take a
+    still-needed module down with the superseded one.
+    """
+
+    def _sh(self):
+        with open(APPLY_SH, encoding="utf-8") as fh:
+            return fh.read()
+
+    def test_native_support_is_detected_per_module(self):
+        sh = self._sh()
+        assert "native_b2" in sh
+        assert "native_nested" in sh
+        assert "no further nesting" in sh, "nested native-support probe"
+
+    def test_kill_switch_only_when_both_modules_are_done(self):
+        sh = self._sh()
+        assert '[ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]' in sh
+        # ...and that is the only place the kill switch is actually set. (Ignore
+        # comment lines, which mention the same path.)
+        code = [ln for ln in sh.splitlines() if not ln.lstrip().startswith("#")]
+        sets = [ln for ln in code if 'touch "$PATCH_DIR/disabled"' in ln]
+        assert len(sets) == 1, f"kill switch set in {len(sets)} places"
+
+    def test_each_module_is_gated_separately(self):
+        src = heredoc_source()
+        assert "if not providers_needed:" in src
+        assert "elif nested_native:" in src
+
+    def test_ui_patch_is_tied_to_the_providers_module(self):
+        # The UI change widens the credential dropdown; it is meaningless once B2
+        # is native, but must NOT be skipped merely because nested is off.
+        sh = self._sh()
+        i = sh.index("--- UI patch ---")
+        assert '[ "$_providers_needed" = "0" ]' in sh[i:i + 400]
+
+    def test_status_reports_an_inactive_module_as_ok(self):
+        # `create_task.py verify` fails if any patches[*].ok is false. An opt-in
+        # module that is switched off (the DEFAULT) must not report FAIL, or a
+        # stock install fails verification out of the box.
+        src = heredoc_source()
+        assert "'ok': (not nested_needed) or nested_ok" in src
+        assert "'ok': (not providers_needed) or bool(b2_ok and restic_ok)" in src
+        assert "'active': nested_needed" in src
+
+    def test_nested_native_probe_matches_the_real_wrapped_source(self):
+        """Stock splits the guard message across adjacent string literals.
+
+        Python concatenates them at runtime, so the errmsg is contiguous -- but the
+        SOURCE never contains the whole phrase. A raw substring search finds
+        nothing, concludes iX removed the guard, and silently skips this module
+        forever. This is exactly what happened, and only a run against real
+        middlewared caught it.
+        """
+        detect = _nested_native_detector()
+
+        # Verbatim shape from TrueNAS plugins/cloud/crud.py.
+        stock_wrapped = (
+            '            verrors.add(f"{name}.snapshot", '
+            '"This option is only available for datasets that have no further "\n'
+            '                                            "nesting")\n'
+        )
+        assert detect(stock_wrapped) == "no", "guard is present; must NOT report native"
+
+        # Same message on a single line — must also be detected.
+        assert detect('verrors.add(x, "... have no further nesting")\n') == "no"
+
+        # Single-quoted, three-way split — still the guard.
+        assert detect(
+            "verrors.add(x, 'This option is only available for '\n"
+            "               'datasets that have no further '\n"
+            "               'nesting')\n"
+        ) == "no"
+
+        # Guard genuinely gone -> native support.
+        assert detect("def _validate(self):\n    pass\n") == "yes"
+
+    def test_nested_native_probe_ignores_our_own_block(self):
+        # CRUD_BLOCK quotes the guard message, so scanning the whole file would
+        # find the string in our own patch and never detect native support.
+        sh = self._sh()
+        assert "split('\\n# TRUECLOUD_PATCH', 1)[0]" in sh
+        assert "no further nesting" in extract_blocks()["CRUD_ASYNC"], (
+            "if this ever stops being true, the probe comment is stale"
+        )
+
+    def test_restart_fires_when_any_needed_module_landed(self):
+        # Keying the restart off providers alone would leave a freshly-patched
+        # nested module on disk and never loaded on a native-B2 box.
+        sh = self._sh()
+        i = sh.index("--- deferred restart ---")
+        tail = sh[i:]
+        assert '_backend_ok' in tail
+        assert '"$_b2_ok"' not in tail
+
+    def test_partial_failure_still_schedules_the_restart(self):
+        # If providers fails but nested landed (or vice versa), something new IS
+        # on disk. Collapsing that into "nothing to do" would leave the module
+        # that succeeded permanently unloaded.
+        src = heredoc_source()
+        assert "sys.exit(2 if _landed else 1)" in src
+        assert "_landed = (providers_needed and b2_ok and restic_ok) or (nested_needed and nested_ok)" in src
+
+        sh = self._sh()
+        assert '_rc=$?' in sh
+        assert '[ "$_rc" = "2" ]' in sh
+
+
+class TestOptIn:
+    """Nested-snapshot support must be opt-in and must never self-enable."""
+
+    def test_heredoc_gates_on_the_opt_in_flag(self):
+        src = heredoc_source()
+        assert re.search(r"nested_enabled = sys\.argv\[\d+\] == \"1\"", src)
+        assert "if not nested_enabled:" in src
+
+    def test_apply_sh_reads_the_marker_file(self):
+        with open(APPLY_SH, encoding="utf-8") as fh:
+            sh = fh.read()
+        assert 'if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]' in sh
+        assert '"$_NESTED_ENABLED"' in sh
+
+    def test_patching_is_skipped_entirely_when_disabled(self):
+        # The guard-relaxing crud.py patch must be inside the enabled branch.
+        src = heredoc_source()
+        gate = src.index("if not nested_needed:")
+        crud = src.index("patch_file(crud_py, _crud_block)")
+        assert gate < crud, "crud.py patch must sit inside the opt-in branch"
+
+    def test_disabling_REVERTS_the_patch_rather_than_merely_skipping_it(self):
+        """Skipping is not disabling.
+
+        The overlay persists for the whole boot, so a patch applied by an earlier
+        run this boot is still on disk — and middlewared re-imports it on the
+        restart install.sh performs. Without an active revert,
+        `--disable-nested-snapshots` reports "disabled" while the feature keeps
+        running until the next reboot.
+        """
+        src = heredoc_source()
+        # The implementation lives in patch/mw_patch.py (see test_mw_patch.py);
+        # apply.sh must import and actually call it.
+        assert "from mw_patch import patch_file, revert_nested" in src
+        gate = src.index("if not nested_needed:")
+        revert = src.index("reverted = revert_nested(")
+        patch = src.index("patch_file(crud_py, _crud_block)")
+        assert gate < revert < patch, "revert belongs in the not-needed branch"
+
+    def test_import_failure_skips_the_patch_rather_than_crashing(self):
+        # apply.sh runs at PREINIT. If mw_patch.py cannot be imported it must
+        # degrade to "middlewared starts stock", never take the boot down.
+        src = heredoc_source()
+        i = src.index("from mw_patch import")
+        tail = src[i:i + 400]
+        assert "except ImportError" in tail
+        assert "skipping backend patch" in tail
+
+
+def test_guard_is_relaxed_only_after_traversal_is_installed():
+    # Ordering in apply.sh is a safety property: copy module -> patch snapshot.py
+    # -> patch sync.py -> patch crud.py. crud.py (which unlocks the feature) must
+    # come last, so a partial failure never leaves "guard removed, traversal gone".
+    src = heredoc_source()
+    order = [
+        src.index("shutil.copyfile(nested_src, nested_dst)"),
+        src.index("patch_file(snapshot_py, _snapshot_block)"),
+        src.index("patch_file(sync_path, _sync_block)"),
+        src.index("patch_file(crud_py, _crud_block)"),
+    ]
+    assert order == sorted(order), "crud.py must be patched last"
+
+
+class TestWrappersDoNotHardcodeStockArity:
+    """iX changes the tail of these signatures between releases.
+
+    SYNC_BLOCK used to spell out `(middleware, job, cloud_backup, dry_run, rate_limit)`
+    and forward all five. But 24.10 and 25.04 declare only four -- `rate_limit` arrived
+    in 25.10 -- so every nested backup on those two releases raised
+    `TypeError: restic_backup() takes 4 positional arguments but 5 were given`.
+    It shipped broken and nothing noticed, because the compat check at the time only
+    asked whether the parameter NAMES still appeared somewhere in the signature.
+
+    Forwarding *args/**kwargs makes the wrapper indifferent to a trailing parameter
+    being added or dropped, which is the only part iX actually churns.
+    """
+
+    def test_restic_backup_forwards_rather_than_naming_stock_params(self):
+        block = extract_blocks()["SYNC_ASYNC"]
+        assert "async def restic_backup(middleware, job, cloud_backup, *args, **kwargs)" in block
+        assert "_tc_orig_restic_backup(middleware, job, cloud_backup, *args, **kwargs)" in block
+
+        # Comments stripped: the block's own commentary explains the rate_limit
+        # history, and that must not be mistaken for the code re-declaring it.
+        code = "\n".join(
+            line for line in block.splitlines()
+            if not line.lstrip().startswith("#")
+        )
+        assert "rate_limit" not in code, (
+            "naming a trailing stock parameter re-introduces the arity bug"
+        )
+
+
+class TestTheTwoNativeProbesCannotDrift:
+    """The split-literal trick is implemented TWICE: inline in apply.sh's probe, and
+    as compat._squash. It has already caused one silent bug.
+
+    Stock middleware writes the guard as an implicitly-concatenated literal, so the
+    contiguous phrase never appears in the source. A naive search finds nothing,
+    concludes iX removed the guard, and reports "native" -- which means "retire the
+    module". That would disable nested snapshots on every box that depends on them.
+
+    apply.sh (runtime, on the box) and compat.py (static, in CI) must therefore agree
+    on every input, or one of them is wrong about whether to retire a module.
+    """
+
+    CASES = [
+        # (crud.py source, expected native?)
+        ("verrors.add('x', 'datasets that have no further nesting')", False),
+        # THE case: split across adjacent literals, as stock actually writes it.
+        ("verrors.add('x', 'datasets that have no further '\n"
+         "                 'nesting')", False),
+        ('verrors.add("x", "no further "\n              "nesting")', False),
+        # Guard genuinely gone -> iX implemented it -> native.
+        ("verrors.add('x', 'some other validation entirely')", True),
+        ("", True),
+    ]
+
+    @pytest.mark.parametrize("src,expect_native", CASES)
+    def test_both_probes_agree(self, src, expect_native):
+        import sys as _sys
+        _sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
+        import compat
+
+        shipped = _nested_native_detector()(src)
+        assert (shipped == "yes") == expect_native, (
+            f"apply.sh's probe says native={shipped!r} for {src!r}"
+        )
+
+        path, phrase, native_when_present = compat.NATIVE_PROBES[compat.NESTED]
+        present = compat._squash(phrase) in compat._squash(src)
+        static_native = (present == native_when_present)
+        assert static_native == expect_native, (
+            f"compat.py says native={static_native} for {src!r}"
+        )
+
+
+class TestOnlyOurOwnTasksAreTouched:
+    """create_snapshot is module-global, and cloud_sync.py imports it too.
+
+    plugins/cloud/snapshot.py::create_snapshot is imported by BOTH
+    cloud_backup/sync.py and cloud_sync.py, so our wrapper sits in the path of every
+    rclone/Storj CloudSync task with snapshot=true -- tasks this patch has no business
+    touching. Two consequences, the second much worse than the first:
+
+      * every middleware call we add is a NEW failure mode for a job that worked
+        before we were installed;
+      * a staged CloudSync task would NEVER be torn down. The teardown is wired into
+        cloud_backup's restic_backup finally, and CRUD_BLOCK deliberately leaves
+        CloudSync's nesting guard intact -- so the bind mounts would pin the ZFS
+        snapshot forever.
+
+    cloud_backup names its snapshot "cloud_backup-<id>", cloud_sync "cloud_sync-<id>",
+    and stock's default is "cloud_task-onetime".
+    """
+
+    @pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
+    def test_the_staging_path_is_gated_on_cloud_backup(self, name):
+        block = extract_blocks()[name]
+        assert 'if not name.startswith("cloud_backup"):' in block
+
+    @pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
+    def test_the_bail_out_precedes_every_middleware_call(self, name):
+        # The point is to add NO new failure mode to a CloudSync task. If any
+        # middleware call happened before the bail-out, we would already have broken
+        # the thing we are trying not to touch.
+        block = extract_blocks()[name]
+        gate = block.index('if not name.startswith("cloud_backup"):')
+        for call in ("middleware.call_sync(", "_tc_nested.stage_nested(",
+                     "_tc_nested.delete_snapshot_tree("):
+            assert gate < block.index(call), f"{call} runs before the cloud_backup gate"

tests/test_compat.py

@@ -0,0 +1,361 @@
+"""Tests for the middlewared compatibility manifest.
+
+Two failure directions, and they are NOT symmetric:
+
+  * a false **BROKEN** makes a module decline to apply on a box where it works.
+    Worse, if both modules go quiet, apply.sh used to set a PERMANENT kill switch
+    that only install.sh clears -- so a network blip or an innocent refactor could
+    take a working box's B2 backups down until someone noticed by hand.
+
+  * a false **OK** lets the patch inject into middleware it does not fit, which is a
+    broken backup discovered at restore time.
+
+Both are tested. The `native` verdict gets its own scrutiny because it is the most
+dangerous thing this file can say -- it means "TrueNAS does this now, retire the
+module" -- and it rests on nothing more than a substring match.
+"""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
+
+import compat  # noqa: E402
+from compat import (  # noqa: E402
+    NESTED,
+    PROVIDERS,
+    Unreadable,
+    check,
+    is_broken,
+)
+
+# A middlewared that the patch fits: TrueNAS 25.10 in miniature.
+GOOD = {
+    "rclone/remote/b2.py": "class B2RcloneRemote(BaseRcloneRemote):\n    pass\n",
+    "plugins/cloud_backup/restic.py": (
+        "class ResticConfig:\n    cmd: list\n\n"
+        "def get_restic_config(cloud_backup):\n    return ResticConfig([], {})\n"
+    ),
+    "plugins/cloud/snapshot.py": (
+        'async def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+    ),
+    "plugins/cloud/crud.py": (
+        "class CloudTaskServiceMixin:\n"
+        "    async def _validate(self, app, verrors, name, data):\n"
+        "        verrors.add('x', 'datasets that have no further '\n"
+        "                         'nesting')\n"
+    ),
+    "plugins/cloud_backup/sync.py": (
+        "async def restic_backup(middleware, job, cloud_backup, dry_run=False, "
+        "rate_limit=None):\n    pass\n"
+    ),
+    # The middlewared METHODS the injected code calls. TrueNAS 26 deleted both of
+    # these files, taking zfs.dataset.query / zfs.snapshot.query / zfs.snapshot.delete
+    # with them -- see TestMiddlewareMethodsWeCall.
+    "plugins/zfs_/dataset.py": (
+        "class ZFSDataset(CRUDService):\n"
+        "    class Config:\n"
+        "        namespace = 'zfs.dataset'\n"
+        "    def query(self, filters, options):\n        pass\n"
+    ),
+    "plugins/zfs_/snapshot.py": (
+        "class ZFSSnapshot(CRUDService):\n"
+        "    class Config:\n"
+        "        namespace = 'zfs.snapshot'\n"
+        "    def query(self, filters, options):\n        pass\n"
+        "    def delete(self, id_, options={}):\n        pass\n"
+    ),
+}
+
+
+def loader(files):
+    def load(path):
+        if path not in files:
+            return None
+        v = files[path]
+        if isinstance(v, Exception):
+            raise v
+        return v
+    return load
+
+
+def check_files(files, modules=None):
+    return check(loader(files), modules)
+
+
+def with_(**overrides):
+    files = dict(GOOD)
+    files.update(overrides)
+    return files
+
+
+class TestTheBaseline:
+    def test_a_good_tree_is_ok_and_not_native(self):
+        r = check_files(GOOD)
+        for mod in (PROVIDERS, NESTED):
+            assert r[mod]["ok"], r[mod]["problems"]
+            assert not r[mod]["native"]
+            assert not r[mod]["unknown"]
+
+
+class TestFalseOkWouldBreakBackups:
+    """The patch calls the originals POSITIONALLY. A name-subset check passed all of
+    these, and each is a TypeError or -- worse -- silently swapped arguments."""
+
+    def test_reordered_parameters_are_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                'async def create_snapshot(name, path, middleware):\n    return 1, 2\n',
+        }))
+        assert is_broken(r[NESTED])
+
+    def test_a_keyword_only_conversion_is_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                'async def create_snapshot(middleware, *, path, name="x"):\n    return 1, 2\n',
+        }))
+        assert is_broken(r[NESTED])
+
+    def test_a_new_required_parameter_is_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                'async def create_snapshot(middleware, path, name, dataset):\n    return 1, 2\n',
+        }))
+        assert is_broken(r[NESTED])
+
+    def test_a_new_optional_parameter_is_fine(self):
+        # The patch simply will not pass it. Refusing here would be false BROKEN.
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                'async def create_snapshot(middleware, path, name="x", quiet=False):\n'
+                "    return 1, 2\n",
+        }))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+    def test_the_master_signature_change_is_caught(self):
+        # iX really did rename this on master: get_restic_config(entry, credentials).
+        # RESTIC_BLOCK rebinds the module-level name to a 1-arg wrapper, so getting
+        # this wrong kills EVERY TrueCloud task -- Storj included.
+        r = check_files(with_(**{
+            "plugins/cloud_backup/restic.py":
+                "class ResticConfig:\n    cmd: list\n\n"
+                "def get_restic_config(entry, credentials):\n    pass\n",
+        }))
+        assert is_broken(r[PROVIDERS])
+
+    def test_a_vanished_symbol_is_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py": "def something_else():\n    pass\n",
+        }))
+        assert is_broken(r[NESTED])
+
+
+class TestFalseBrokenWouldDisableWorkingBoxes:
+    def test_a_conditionally_defined_symbol_is_not_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py":
+                "try:\n"
+                "    from .fast import create_snapshot\n"
+                "except ImportError:\n"
+                '    async def create_snapshot(middleware, path, name="x"):\n'
+                "        return 1, 2\n",
+        }))
+        assert not is_broken(r[NESTED]), r[NESTED]["problems"]
+
+    def test_a_re_exported_symbol_is_unknown_not_broken(self):
+        r = check_files(with_(**{
+            "plugins/cloud_backup/restic.py":
+                "from ._impl import ResticConfig, get_restic_config\n",
+        }))
+        assert not is_broken(r[PROVIDERS])
+        assert r[PROVIDERS]["unknown"]
+
+    def test_an_unreadable_source_is_unknown_not_broken(self):
+        # A rate limit (the matrix makes ~30 unauthenticated requests) must not be
+        # able to say "iX deleted six files, both modules are broken".
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py": Unreadable("HTTP 429"),
+        }))
+        assert not is_broken(r[NESTED])
+        assert r[NESTED]["unknown"]
+
+    def test_a_definite_break_still_wins_over_an_unknown(self):
+        r = check_files(with_(**{
+            "plugins/cloud/snapshot.py": Unreadable("HTTP 429"),
+            "plugins/cloud_backup/sync.py":
+                "async def restic_backup(job, middleware, cloud_backup):\n    pass\n",
+        }))
+        assert is_broken(r[NESTED]), "unknown must not launder away a proven break"
+
+
+class TestTheNativeVerdict:
+    """"native" means "retire the module". It is the most destructive thing this file
+    can say, and it is only a substring match — so it must never outrank BROKEN."""
+
+    def test_broken_outranks_native(self):
+        # Guard reworded (reads as native) AND the signatures changed (really broken).
+        # This used to render as good news: green CI, no bug report, and a README row
+        # telling users the feature went native while it was in fact broken.
+        r = check_files(with_(**{
+            "plugins/cloud/crud.py":
+                "class CloudTaskServiceMixin:\n"
+                "    async def _validate(self, verrors, name):\n"
+                "        verrors.add('x', 'no children allowed')\n",
+        }))
+        assert r[NESTED]["native"]
+        assert is_broken(r[NESTED])
+        assert compat._verdict(r[NESTED]) == "BROKEN"
+
+    def test_an_already_patched_tree_does_not_read_as_native(self):
+        # B2_BLOCK writes `B2RcloneRemote.restic = True` into b2.py. Scanning the whole
+        # file finds OUR OWN line and concludes TrueNAS went native — so the command
+        # compat.py's docstring recommends for a live box (`--tree /usr/lib/...`)
+        # reported providers as native on every patched machine.
+        r = check_files(with_(**{
+            "rclone/remote/b2.py":
+                "class B2RcloneRemote(BaseRcloneRemote):\n    pass\n"
+                "\n# TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh\n"
+                "B2RcloneRemote.restic = True\n",
+        }))
+        assert not r[PROVIDERS]["native"], "read its own patch as native support"
+        assert r[PROVIDERS]["ok"]
+
+    def test_a_genuinely_native_b2_is_native(self):
+        r = check_files(with_(**{
+            "rclone/remote/b2.py":
+                "class B2RcloneRemote(BaseRcloneRemote):\n    restic = True\n",
+        }))
+        assert r[PROVIDERS]["native"]
+
+
+class TestUpdateReadmeCannotPublishAGuess:
+    def test_it_refuses_when_anything_is_unknown(self, tmp_path):
+        readme = tmp_path / "README.md"
+        readme.write_text(f"x\n{compat.BEGIN}\nold\n{compat.END}\ny\n")
+        rows = [{
+            "ref": "TS-25.10.4", "unreleased": False,
+            "modules": check_files(with_(**{
+                "plugins/cloud/snapshot.py": Unreadable("HTTP 429"),
+            })),
+        }]
+        with pytest.raises(Unreadable):
+            compat.update_readme(rows, path=str(readme))
+        assert "old" in readme.read_text(), "a blip must not repaint the matrix"
+
+
+class TestAsyncFlavour:
+    """TrueNAS <= 25.10 is async; 26 is synchronous. Both are supported -- apply.sh
+    injects the wrapper that matches. So asyncness is DETECTED, never assumed."""
+
+    def test_async_middleware_is_detected(self):
+        assert compat.async_flavour(loader(GOOD)) is True
+
+    def test_sync_middleware_is_detected(self):
+        sync = dict(GOOD)
+        sync["plugins/cloud/snapshot.py"] = (
+            'def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+        )
+        sync["plugins/cloud/crud.py"] = (
+            "class CloudTaskServiceMixin:\n"
+            "    def _validate(self, app, verrors, name, data):\n"
+            "        verrors.add('x', 'no further nesting')\n"
+        )
+        sync["plugins/cloud_backup/sync.py"] = (
+            "def restic_backup(middleware, job, cloud_backup, dry_run=False, "
+            "rate_limit=None):\n    pass\n"
+        )
+        assert compat.async_flavour(loader(sync)) is False
+
+    def test_a_HALF_converted_middleware_is_refused(self):
+        # The dangerous middle. If iX converts create_snapshot but not restic_backup,
+        # there is no single wrapper flavour that works -- and guessing means either
+        # a coroutine unpacked as a tuple, or the event loop blocked. None means
+        # "do not patch"; apply.sh turns that into a skip, not a guess.
+        half = dict(GOOD)
+        half["plugins/cloud/snapshot.py"] = (
+            'def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+        )
+        assert compat.async_flavour(loader(half)) is None
+
+    def test_an_unreadable_source_refuses_rather_than_guesses(self):
+        broken = dict(GOOD)
+        broken["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429")
+        assert compat.async_flavour(loader(broken)) is None
+
+    def test_the_real_truenas_versions(self):
+        # Pinning the actual fact this whole port exists for.
+        assert compat.async_flavour(loader(GOOD)) is True
+
+
+class TestMiddlewareMethodsWeCall:
+    """The assumption class that was MISSING, and that hid a catastrophic break.
+
+    The manifest recorded the symbols the patch WRAPS. It said nothing about the
+    middlewared methods the patch CALLS -- and TrueNAS 26 deleted
+    plugins/zfs_/dataset.py and plugins/zfs_/snapshot.py outright, taking
+    `zfs.dataset.query`, `zfs.snapshot.query` and `zfs.snapshot.delete` with them.
+
+    Nothing about the five cloud_backup files reveals that. The patch applied
+    perfectly and every other check went green. The first backup would have failed --
+    or, far worse, snapshotted fine and then failed to DELETE, orphaning one snapshot
+    per descendant dataset (250 on a real pool) on every run, forever.
+    """
+
+    ZFS_SNAPSHOT = (
+        "class ZFSSnapshot(CRUDService):\n"
+        "    class Config:\n"
+        "        namespace = 'zfs.snapshot'\n"
+        "    def query(self, filters, options):\n        pass\n"
+        "    def delete(self, id_, options={}):\n        pass\n"
+    )
+    ZFS_DATASET = (
+        "class ZFSDataset(CRUDService):\n"
+        "    class Config:\n"
+        "        namespace = 'zfs.dataset'\n"
+        "    def query(self, filters, options):\n        pass\n"
+    )
+
+    def _tree(self, **over):
+        files = dict(GOOD)
+        files["plugins/zfs_/snapshot.py"] = self.ZFS_SNAPSHOT
+        files["plugins/zfs_/dataset.py"] = self.ZFS_DATASET
+        files.update(over)
+        return files
+
+    def test_present_methods_are_ok(self):
+        r = check_files(self._tree())
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+    def test_a_deleted_plugin_file_is_broken(self):
+        # Literally TrueNAS 26: plugins/zfs_/snapshot.py does not exist.
+        r = check_files(self._tree(**{"plugins/zfs_/snapshot.py": None}))
+        assert is_broken(r[NESTED])
+        details = " ".join(p["detail"] for p in r[NESTED]["problems"])
+        assert "zfs.snapshot.delete" in details
+
+    def test_a_renamed_namespace_is_broken(self):
+        r = check_files(self._tree(**{
+            "plugins/zfs_/snapshot.py": self.ZFS_SNAPSHOT.replace(
+                "'zfs.snapshot'", "'zfs.resource.snapshot'"),
+        }))
+        assert is_broken(r[NESTED])
+
+    def test_the_CRUDService_do_prefix_is_accepted(self):
+        # 24.10 and 25.04 declare `do_delete`; 25.10 renamed it to `delete`. BOTH
+        # answer to zfs.snapshot.delete. Accepting only the literal name reported the
+        # two older releases as broken -- a false BROKEN that would have switched off
+        # nested snapshots on boxes where they work perfectly.
+        r = check_files(self._tree(**{
+            "plugins/zfs_/snapshot.py": self.ZFS_SNAPSHOT.replace(
+                "def delete(", "def do_delete("),
+        }))
+        assert r[NESTED]["ok"], r[NESTED]["problems"]
+
+    def test_the_snapshot_delete_reason_names_the_orphan_risk(self):
+        # If this ever regresses, whoever reads the bug report must understand that
+        # it is not a cosmetic failure.
+        r = check_files(self._tree(**{"plugins/zfs_/snapshot.py": None}))
+        whys = " ".join(p["why"] for p in r[NESTED]["problems"])
+        assert "orphan" in whys

tests/test_create_task.py

@@ -0,0 +1,107 @@
+"""Tests for create_task.py, focused on the restic repository password.
+
+That password is the encryption key for the entire cloud backup repository. It
+used to travel through `midclt call cloud_backup.create '<json>'` -- i.e. through
+the subprocess's **argv**, which is world-readable via `ps` -- and `--password`
+wrote it into the user's shell history forever.
+"""
+
+import importlib.util
+import io
+import os
+import sys
+import types
+
+import pytest
+
+SPEC = importlib.util.spec_from_file_location(
+    "create_task",
+    os.path.join(os.path.dirname(__file__), "..", "patch", "create_task.py"),
+)
+
+
+def load():
+    mod = importlib.util.module_from_spec(SPEC)
+    SPEC.loader.exec_module(mod)
+    return mod
+
+
+class Args:
+    def __init__(self, password=None, password_stdin=False):
+        self.password = password
+        self.password_stdin = password_stdin
+
+
+class TestPasswordNeverReachesArgv:
+    """The whole reason this module talks to the client library."""
+
+    def test_midclt_call_spawns_no_subprocess(self):
+        import inspect
+
+        src = inspect.getsource(load().midclt_call)
+        assert "subprocess" not in src, (
+            "shelling out to `midclt` puts cloud_backup.create's JSON -- including "
+            "the restic repo password -- into argv, which any local user can read "
+            "with ps"
+        )
+        assert "truenas_api_client" in src
+
+    def test_errors_never_echo_the_call_arguments(self):
+        # A failed cloud_backup.create must not print the body back at the user;
+        # it contains the password.
+        import inspect
+
+        src = inspect.getsource(load().midclt_call)
+        assert "{args}" not in src
+        assert "args!r" not in src
+
+
+class TestResolvePassword:
+    def test_reads_from_stdin(self, monkeypatch):
+        mod = load()
+        monkeypatch.setattr(sys, "stdin", io.StringIO("s3cret\n"))
+        assert mod._resolve_password(Args(password_stdin=True)) == "s3cret"
+
+    def test_strips_only_the_trailing_newline(self, monkeypatch):
+        # A password may legitimately contain spaces; only the line ending goes.
+        mod = load()
+        monkeypatch.setattr(sys, "stdin", io.StringIO("  pass word  \n"))
+        assert mod._resolve_password(Args(password_stdin=True)) == "  pass word  "
+
+    def test_cli_password_still_works_but_warns(self, monkeypatch, capsys):
+        mod = load()
+        pw = mod._resolve_password(Args(password="cli-secret"))
+        assert pw == "cli-secret"
+        assert "shell" in capsys.readouterr().err.lower(), "must warn about history"
+
+    def test_prompts_when_neither_flag_given(self, monkeypatch):
+        mod = load()
+        monkeypatch.setattr(
+            mod, "getpass", types.SimpleNamespace(getpass=lambda _p: "prompted")
+        )
+        assert mod._resolve_password(Args()) == "prompted"
+
+    def test_rejects_both_flags(self, monkeypatch):
+        mod = load()
+        monkeypatch.setattr(sys, "stdin", io.StringIO("x\n"))
+        with pytest.raises(SystemExit):
+            mod._resolve_password(Args(password="a", password_stdin=True))
+
+    def test_rejects_an_empty_password(self, monkeypatch):
+        # An empty restic password would silently create an unencrypted-ish repo.
+        mod = load()
+        monkeypatch.setattr(sys, "stdin", io.StringIO("\n"))
+        with pytest.raises(SystemExit):
+            mod._resolve_password(Args(password_stdin=True))
+
+
+class TestVersion:
+    def test_version_is_not_stale(self):
+        # __version__ sat at 0.2.0 through three releases because the drift check
+        # only looked at VERSION= in shell scripts. It covers this file now.
+        sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
+        from release_notes import normalise, script_versions
+
+        repo = os.path.join(os.path.dirname(__file__), "..")
+        versions = {normalise(v) for v in script_versions(repo).values()}
+        assert len(versions) == 1, f"version drift: {sorted(versions)}"

tests/test_docs.py

@@ -0,0 +1,96 @@
+"""The docs must not lie about themselves.
+
+The README was 969 lines with the install instructions at line 517. Splitting it into
+docs/ fixed that and broke every cross-reference in the process -- which is the normal
+outcome of moving Markdown around, and exactly why this is a test rather than a
+careful afternoon.
+
+A dead link in a recovery doc is worse than a dead link anywhere else: the person
+following it is, by definition, already having a bad day.
+"""
+
+import os
+import re
+
+import pytest
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DOCS = os.path.join(ROOT, "docs")
+
+LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
+HEADING_RE = re.compile(r"^#{1,6}\s+(.*)$", re.M)
+
+
+def markdown_files():
+    files = [os.path.join(ROOT, "README.md"), os.path.join(ROOT, "CHANGELOG.md")]
+    if os.path.isdir(DOCS):
+        files += [os.path.join(DOCS, f) for f in sorted(os.listdir(DOCS))
+                  if f.endswith(".md")]
+    return files
+
+
+def anchors(text):
+    """GitHub/Gitea slugs for every heading in `text`."""
+    out = set()
+    for h in HEADING_RE.findall(text):
+        slug = re.sub(r"[^a-z0-9 -]", "", h.lower()).replace(" ", "-")
+        out.add(slug)
+    return out
+
+
+@pytest.mark.parametrize("path", markdown_files(), ids=os.path.basename)
+def test_every_internal_link_resolves(path):
+    with open(path, encoding="utf-8") as fh:
+        text = fh.read()
+    here = anchors(text)
+    base = os.path.dirname(path)
+
+    broken = []
+    for label, target in LINK_RE.findall(text):
+        if target.startswith(("http://", "https://", "mailto:")):
+            continue
+        rel, _, anchor = target.partition("#")
+
+        if not rel:                                  # same-file anchor
+            if anchor and anchor not in here:
+                broken.append(f"[{label}](#{anchor}) — no such heading here")
+            continue
+
+        dest = os.path.normpath(os.path.join(base, rel))
+        if not os.path.exists(dest):
+            broken.append(f"[{label}]({target}) — file does not exist")
+            continue
+
+        if anchor and dest.endswith(".md"):
+            with open(dest, encoding="utf-8") as fh:
+                if anchor not in anchors(fh.read()):
+                    broken.append(f"[{label}]({target}) — no such heading there")
+
+    assert not broken, "broken links in {}:\n  {}".format(
+        os.path.basename(path), "\n  ".join(broken)
+    )
+
+
+class TestTheReadmeStaysAReadme:
+    def test_install_is_near_the_top(self):
+        # It was at line 517 of 969, under a wall of internals. Somebody deciding
+        # whether to use this should not have to scroll past the boot sequence.
+        with open(os.path.join(ROOT, "README.md"), encoding="utf-8") as fh:
+            lines = fh.read().splitlines()
+        install = next(i for i, ln in enumerate(lines, 1) if ln.startswith("## Install"))
+        assert install < 40, f"## Install is at line {install}"
+
+    def test_the_readme_does_not_grow_back(self):
+        with open(os.path.join(ROOT, "README.md"), encoding="utf-8") as fh:
+            n = len(fh.read().splitlines())
+        assert n < 300, (
+            f"README is {n} lines. Detail belongs in docs/ — the README is what "
+            f"someone reads before they trust this with their backups."
+        )
+
+    def test_the_minimum_version_is_stated_before_the_install_command(self):
+        with open(os.path.join(ROOT, "README.md"), encoding="utf-8") as fh:
+            text = fh.read()
+        assert "24.10" in text[:text.index("## Install")], (
+            "the minimum TrueNAS version must be visible above the install steps"
+        )

tests/test_mw_patch.py

@@ -0,0 +1,160 @@
+"""Tests for mw_patch — the single implementation of apply/revert.
+
+apply.sh and uninstall.sh both go through this. It used to be duplicated in an
+untested shell heredoc, which is exactly how the two could have drifted apart:
+apply.sh reverting one set of files and uninstall.sh another.
+"""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "patch"))
+
+from mw_patch import (  # noqa: E402
+    MARKER,
+    NESTED_MODULE,
+    NESTED_RELPATHS,
+    PROVIDER_RELPATHS,
+    patch_file,
+    revert_all,
+    revert_nested,
+    unpatch_file,
+)
+
+STOCK = "import os\n\n\ndef stock():\n    return 1\n"
+BLOCK = "\n# TRUECLOUD_PATCH\ninjected = 1\n"
+
+
+def build_mw(root):
+    """A fake middlewared tree with every file this project touches."""
+    mw = os.path.join(root, "middlewared")
+    for rel in NESTED_RELPATHS + PROVIDER_RELPATHS:
+        path = os.path.join(mw, *rel)
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "w", encoding="utf-8") as fh:
+            fh.write(STOCK)
+    with open(os.path.join(mw, *NESTED_MODULE), "w", encoding="utf-8") as fh:
+        fh.write("# module\n")
+    return mw
+
+
+def read(mw, rel):
+    with open(os.path.join(mw, *rel), encoding="utf-8") as fh:
+        return fh.read()
+
+
+class TestPatchFile:
+    def test_appends_the_block(self, tmp_path):
+        p = tmp_path / "m.py"
+        p.write_text(STOCK)
+        patch_file(str(p), BLOCK)
+        assert MARKER in p.read_text()
+        assert p.read_text().startswith("import os")
+
+    def test_is_idempotent(self, tmp_path):
+        # apply.sh runs on EVERY boot. Without truncate-then-append, repeated runs
+        # would stack duplicate copies of the block into a middlewared module.
+        p = tmp_path / "m.py"
+        p.write_text(STOCK)
+        for _ in range(5):
+            patch_file(str(p), BLOCK)
+        assert p.read_text().count("# TRUECLOUD_PATCH") == 1
+        assert p.read_text().count("injected = 1") == 1
+
+    def test_round_trips_back_to_stock(self, tmp_path):
+        p = tmp_path / "m.py"
+        p.write_text(STOCK)
+        patch_file(str(p), BLOCK)
+        assert unpatch_file(str(p)) is True
+        assert p.read_text() == STOCK
+
+
+class TestUnpatchFile:
+    def test_returns_false_on_an_unpatched_file(self, tmp_path):
+        p = tmp_path / "m.py"
+        p.write_text(STOCK)
+        assert unpatch_file(str(p)) is False
+        assert p.read_text() == STOCK
+
+    def test_returns_false_on_a_missing_file(self, tmp_path):
+        assert unpatch_file(str(tmp_path / "nope.py")) is False
+
+
+class TestRevertNested:
+    def test_reverts_only_the_nested_files(self, tmp_path):
+        mw = build_mw(str(tmp_path))
+        for rel in NESTED_RELPATHS + PROVIDER_RELPATHS:
+            patch_file(os.path.join(mw, *rel), BLOCK)
+
+        reverted = revert_nested(mw)
+
+        for rel in NESTED_RELPATHS:
+            assert read(mw, rel) == STOCK, f"{rel[-1]} should be stock"
+            assert rel[-1] in reverted
+
+    def test_never_touches_the_providers_patch(self, tmp_path):
+        # restic.py carries a TRUECLOUD_PATCH block too, but it belongs to the
+        # providers module. Reverting it would silently break B2 backups.
+        mw = build_mw(str(tmp_path))
+        for rel in NESTED_RELPATHS + PROVIDER_RELPATHS:
+            patch_file(os.path.join(mw, *rel), BLOCK)
+
+        revert_nested(mw)
+
+        for rel in PROVIDER_RELPATHS:
+            assert MARKER in read(mw, rel), f"{rel[-1]} must keep its providers block"
+
+    def test_removes_the_module(self, tmp_path):
+        mw = build_mw(str(tmp_path))
+        assert os.path.exists(os.path.join(mw, *NESTED_MODULE))
+        reverted = revert_nested(mw)
+        assert not os.path.exists(os.path.join(mw, *NESTED_MODULE))
+        assert "_truecloud_nested.py" in reverted
+
+    def test_module_is_removed_before_the_files_are_unpatched(self, tmp_path):
+        # Every injected block is guarded by `if _tc_nested is not None`, so once
+        # the module is gone they all no-op — the stock guard is restored even if
+        # a later unpatch fails.
+        mw = build_mw(str(tmp_path))
+        for rel in NESTED_RELPATHS:
+            patch_file(os.path.join(mw, *rel), BLOCK)
+        reverted = revert_nested(mw)
+        assert reverted[0] == "_truecloud_nested.py"
+
+    def test_is_idempotent(self, tmp_path):
+        mw = build_mw(str(tmp_path))
+        for rel in NESTED_RELPATHS:
+            patch_file(os.path.join(mw, *rel), BLOCK)
+        revert_nested(mw)
+        assert revert_nested(mw) == []
+
+
+class TestRevertAll:
+    def test_reverts_providers_and_nested(self, tmp_path):
+        mw = build_mw(str(tmp_path))
+        for rel in NESTED_RELPATHS + PROVIDER_RELPATHS:
+            patch_file(os.path.join(mw, *rel), BLOCK)
+
+        revert_all(mw)
+
+        for rel in NESTED_RELPATHS + PROVIDER_RELPATHS:
+            assert read(mw, rel) == STOCK, f"{rel[-1]} should be stock"
+        assert not os.path.exists(os.path.join(mw, *NESTED_MODULE))
+
+    def test_is_a_noop_on_a_stock_tree(self, tmp_path):
+        mw = build_mw(str(tmp_path))
+        os.unlink(os.path.join(mw, *NESTED_MODULE))
+        assert revert_all(mw) == []
+
+
+class TestTargetsAreDisjoint:
+    def test_no_file_is_in_both_module_lists(self):
+        # If restic.py ever appeared in NESTED_RELPATHS, revert_nested would break
+        # B2 backups.
+        assert not set(NESTED_RELPATHS) & set(PROVIDER_RELPATHS)
+
+    @pytest.mark.parametrize("rel", PROVIDER_RELPATHS)
+    def test_provider_targets_are_not_nested_targets(self, rel):
+        assert rel not in NESTED_RELPATHS

tests/test_patch_ui.py

@@ -0,0 +1,171 @@
+"""Tests for the Angular bundle patch.
+
+This is the one part of the patch that edits *minified third-party JavaScript* by
+regex, so it is the easiest place to silently produce a broken bundle: a pattern
+that matches nothing leaves the dropdown Storj-only, and a pattern that matches
+sloppily can unbalance the parentheses and take the whole web UI down.
+
+Nothing checked it until now. The snippets below are verbatim from a real
+TrueNAS 25.x bundle (chunk-*.js, pre-patch).
+"""
+
+import os
+import re
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "patch"))
+
+from patch_ui import MARKER, _PATTERNS, _match_pattern  # noqa: E402
+
+# Verbatim from /usr/share/truenas/webui/chunk-FX2QXNQU.js on TrueNAS 25.10.
+# Angular emits the binding as a chained ɵɵproperty(...)(...) call, so the
+# pureFunction call is followed by TWO closing parens: one for pe(...), one for
+# property(...).
+REAL_25X = (
+    'c(2,"filterByProviders",pe(115,Rn,i.CloudSyncProviderName.Storj))'
+    '("required",!0),r(3'
+)
+
+# TrueNAS 24.x and earlier emitted a literal array.
+REAL_24X = 'c(2,"filterByProviders",["STORJ_IX"])("required",!0),r(3'
+
+
+def apply_patch(content):
+    """Run the same match-and-substitute main() does."""
+    find, replace = _match_pattern(content)
+    assert find is not None, "no pattern matched"
+    patched, count = find.subn(replace, content)
+    return patched, count
+
+
+def paren_delta(s):
+    """Net paren balance. The snippets are fragments of a minified file, so they
+    are not balanced on their own -- what must hold is that patching does not
+    CHANGE the balance. Consuming one paren too many is a syntax error in the
+    bundle, and the whole TrueNAS web UI goes blank."""
+    return s.count("(") - s.count(")")
+
+
+@pytest.mark.parametrize("source", [REAL_25X, REAL_24X], ids=["25.x", "24.x"])
+class TestAgainstRealBundles:
+    def test_matches_exactly_once(self, source):
+        # main() refuses to write unless count == 1 — more than one match would
+        # mean the pattern is too loose to trust against a minified bundle.
+        _patched, count = apply_patch(source)
+        assert count == 1
+
+    def test_result_contains_all_three_providers(self, source):
+        patched, _ = apply_patch(source)
+        assert MARKER in patched
+        assert '"filterByProviders",["STORJ_IX","S3","B2"]' in patched
+
+    def test_patch_does_not_change_paren_balance(self, source):
+        # Consuming one paren too many (or too few) is a syntax error in the
+        # bundle and the entire TrueNAS web UI goes blank. This is the invariant
+        # the 25.x pattern has to get right: it eats `pe(...)` which sits inside
+        # a chained property(...)(...) call.
+        patched, _ = apply_patch(source)
+        assert paren_delta(patched) == paren_delta(source)
+
+    def test_surrounding_code_is_untouched(self, source):
+        patched, _ = apply_patch(source)
+        assert patched.startswith("c(2,")
+        assert patched.endswith('("required",!0),r(3')
+
+    def test_patch_is_idempotent(self, source):
+        # apply.sh re-runs every boot; MARKER short-circuits an already-patched
+        # file, but the pattern must also not match its own output.
+        patched, _ = apply_patch(source)
+        find, _replace = _match_pattern(patched)
+        if find is not None:
+            # Only the 24.x literal-array pattern may still "match" — and only if
+            # it would produce the same text. Anything else means double-patching.
+            again, _ = apply_patch(patched)
+            assert again == patched, "re-patching must be a no-op"
+
+
+def test_storj_only_bundle_is_recognised():
+    assert _match_pattern(REAL_25X)[0] is not None
+
+
+def test_unrelated_javascript_is_never_touched():
+    # A pattern loose enough to hit unrelated code would corrupt the bundle.
+    for noise in (
+        'c(2,"filterByProviders",pe(115,Rn,i.SomethingElse.Storj))',
+        'c(2,"otherBinding",pe(115,Rn,i.CloudSyncProviderName.Storj))',
+        '"filterByProviders"',
+    ):
+        find, _ = _match_pattern(noise)
+        assert find is None, f"pattern must not match: {noise}"
+
+
+def test_every_pattern_is_anchored_to_filterbyproviders():
+    # Guards against a future pattern broad enough to rewrite arbitrary JS.
+    for find, _replace in _PATTERNS:
+        assert "filterByProviders" in find.pattern
+
+
+def test_patterns_compile_and_replacements_reference_group_one():
+    for find, replace in _PATTERNS:
+        assert isinstance(find, re.Pattern)
+        assert r"\1" in replace, "replacement must preserve the binding name"
+
+
+class TestCorruptionGuard:
+    """A bad pattern must never reach the bundle.
+
+    This is not hypothetical. Commit 47cdf72 shipped a pattern that consumed one
+    closing paren and emitted one, netting an extra `)`:
+
+        c(2,"filterByProviders",["STORJ_IX","S3","B2"]))("required",!0)
+                                                      ^^ syntax error
+
+    The web UI went blank. And because MARKER was then present in the file, every
+    subsequent run reported "already patched" and skipped — so the patch could not
+    heal itself, and the bundle had to be hand-restored from the backup.
+    """
+
+    # Verbatim from 47cdf72.
+    BROKEN = (
+        re.compile(r'("filterByProviders",)\w+\(\d+,\w+,\w+\.CloudSyncProviderName\.Storj\)'),
+        r'\1["STORJ_IX","S3","B2"])',
+    )
+
+    def test_the_regression_that_blanked_the_ui_is_detectable(self):
+        find, replace = self.BROKEN
+        patched, count = find.subn(replace, REAL_25X)
+        assert count == 1, "it did match — that is why it got written"
+        assert paren_delta(patched) != paren_delta(REAL_25X), (
+            "the paren balance changes; this is the signal main() now refuses on"
+        )
+
+    def test_main_refuses_to_write_an_unbalanced_bundle(self, monkeypatch, tmp_path, capsys):
+        import patch_ui
+
+        bundle = tmp_path / "chunk-TEST.js"
+        bundle.write_text(REAL_25X, encoding="utf-8")
+
+        monkeypatch.setattr(patch_ui, "WEBUI_CANDIDATES", [str(tmp_path)])
+        monkeypatch.setattr(patch_ui, "_PATTERNS", [self.BROKEN])
+
+        patch_ui.main()
+
+        out = capsys.readouterr().out
+        assert "refusing to write" in out
+        # The bundle must be byte-for-byte untouched — a broken UI is far worse
+        # than an unpatched one.
+        assert bundle.read_text(encoding="utf-8") == REAL_25X
+
+    def test_a_good_pattern_still_writes(self, monkeypatch, tmp_path):
+        import patch_ui
+
+        bundle = tmp_path / "chunk-TEST.js"
+        bundle.write_text(REAL_25X, encoding="utf-8")
+        monkeypatch.setattr(patch_ui, "WEBUI_CANDIDATES", [str(tmp_path)])
+
+        patch_ui.main()
+
+        assert MARKER in bundle.read_text(encoding="utf-8")
+        assert (tmp_path / "chunk-TEST.js.pre-truecloud-patch").exists()

tests/test_release_gate.py

@@ -0,0 +1,264 @@
+"""Tests for the barrier: a stable release must have been a release candidate.
+
+This exists because the repo cut twelve releases in one day, several of them
+fixing the release before -- and with the update alert live, every one of those
+interrupts every user. The gate makes that path impossible rather than impolite.
+
+The provenance gate is the one thing here that can wrongly PASS in a way nobody
+notices (a wrongly-failing gate is loud; a wrongly-passing gate silently restores
+the old behaviour), so it gets tested against real git repositories, not mocks.
+"""
+
+import os
+import subprocess
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
+
+from release_gate import (  # noqa: E402
+    check_promotable,
+    next_rc,
+    rc_tags,
+)
+from release_notes import (  # noqa: E402
+    base_version,
+    is_prerelease,
+    promote,
+    unreleased_body,
+)
+
+
+# ── a real git repo, because the gate reads real tags ────────────────────────
+
+class Repo:
+    """A throwaway git repo. The gate reads real tags, so the tests build real ones."""
+
+    def __init__(self, path):
+        self.path = path
+
+    def __str__(self):
+        return str(self.path)
+
+    def git(self, *args):
+        return subprocess.run(
+            ["git", *args], cwd=self.path, capture_output=True, text=True, check=True,
+        ).stdout.strip()
+
+    def commit(self, msg):
+        (self.path / "f").write_text(msg)
+        self.git("add", "-A")
+        self.git("commit", "-q", "-m", msg)
+        return self.git("rev-parse", "HEAD")
+
+
+@pytest.fixture
+def repo(tmp_path):
+    d = tmp_path / "repo"
+    d.mkdir()
+    r = Repo(d)
+    r.git("init", "-q", "-b", "main")
+    r.git("config", "user.email", "t@example.com")
+    r.git("config", "user.name", "t")
+    r.commit("initial")
+    return r
+
+
+class TestTheBarrierBeforeTheTagExists:
+    """release.sh calls the gate BEFORE creating the stable tag.
+
+    Every other test here tags first, and that is what let a fatal bug ship green:
+    `check_promotable` began with `commit_for(vX.Y.Z)` and returned "does not exist",
+    while release.sh's own guard refuses to run at all IF the tag exists. The two
+    conditions were mutually exclusive, so `--promote` could never succeed -- the
+    only way to cut a stable release was to hand-tag, bypassing every gate.
+
+    A gate that can only be satisfied after the thing it gates is not a gate.
+    """
+
+    def test_promote_is_allowed_when_head_was_a_candidate_and_the_tag_is_absent(self, repo):
+        repo.git("tag", "v1.0.0-rc1")          # rc on HEAD, no stable tag yet
+        assert check_promotable("v1.0.0", cwd=str(repo)) == []
+
+    def test_promote_is_refused_when_head_was_never_a_candidate(self, repo):
+        assert check_promotable("v1.0.0", cwd=str(repo))
+
+    def test_promote_is_refused_when_head_moved_past_the_candidate(self, repo):
+        repo.git("tag", "v1.0.0-rc1")
+        repo.commit("one more little fix")     # HEAD is no longer the candidate
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "no release candidate does" in problems[0]
+
+
+class TestTheBarrier:
+    def test_a_tag_with_no_candidate_is_refused(self, repo):
+        repo.git("tag", "v1.0.0")
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "never a release candidate" in problems[0]
+
+    def test_a_tag_whose_candidate_is_on_the_same_commit_is_allowed(self, repo):
+        repo.git("tag", "v1.0.0-rc1")
+        repo.git("tag", "v1.0.0")
+        assert check_promotable("v1.0.0", cwd=str(repo)) == []
+
+    def test_one_more_little_fix_after_the_rc_is_refused(self, repo):
+        # THE case this whole mechanism exists for. The candidate passed, then a
+        # "trivial" commit landed, and the stable tag ships code no candidate ever
+        # tested. That is how v0.5.1 happened.
+        repo.git("tag", "v1.0.0-rc1")
+        repo.commit("just a tiny fix, surely fine")
+        repo.git("tag", "v1.0.0")
+
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "no release candidate does" in problems[0]
+        assert "v1.0.0-rc2" in problems[0], "must say how to fix it"
+
+    def test_an_rc_for_a_different_version_does_not_count(self, repo):
+        repo.git("tag", "v0.9.0-rc1")
+        repo.git("tag", "v1.0.0")
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "never a release candidate" in problems[0]
+
+    def test_candidates_themselves_are_never_gated(self, repo):
+        # Requiring an rc to have an rc would be a deadlock.
+        repo.git("tag", "v1.0.0-rc1")
+        assert check_promotable("v1.0.0-rc1", cwd=str(repo)) == []
+
+    def test_a_later_candidate_on_the_right_commit_rescues_it(self, repo):
+        repo.git("tag", "v1.0.0-rc1")
+        repo.commit("fix found during rc1")
+        repo.git("tag", "v1.0.0-rc2")   # re-cut on the fixed commit
+        repo.git("tag", "v1.0.0")
+        assert check_promotable("v1.0.0", cwd=str(repo)) == []
+
+    def test_a_tag_the_numbering_does_not_understand_is_not_a_candidate(self, repo):
+        # `v1.0.0-rc*` also globs `v1.0.0-rc1-hotfix`, which _rc_number reads as 0.
+        # The barrier must be satisfied only by something that really was a candidate.
+        repo.git("tag", "v1.0.0-rc1-hotfix")
+        repo.git("tag", "v1.0.0")
+        problems = check_promotable("v1.0.0", cwd=str(repo))
+        assert problems
+        assert "never a release candidate" in problems[0]
+        assert rc_tags("1.0.0", cwd=str(repo)) == []
+
+
+class TestRcNumbering:
+    def test_first_candidate_is_rc1(self, repo):
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc1"
+
+    def test_it_counts_up(self, repo):
+        repo.git("tag", "v1.0.0-rc1")
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc2"
+        repo.git("tag", "v1.0.0-rc2")
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc3"
+
+    def test_rc10_sorts_after_rc9_not_before(self, repo):
+        # Lexicographic sorting would rank rc10 before rc9 and hand out a duplicate.
+        for n in range(1, 11):
+            repo.git("tag", f"v1.0.0-rc{n}")
+        assert rc_tags("1.0.0", cwd=str(repo))[-1] == "v1.0.0-rc10"
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc11"
+
+    def test_other_versions_do_not_leak_in(self, repo):
+        repo.git("tag", "v0.9.0-rc7")
+        assert next_rc("1.0.0", cwd=str(repo)) == "v1.0.0-rc1"
+
+
+# ── the content half of the gate ─────────────────────────────────────────────
+
+class TestPrereleaseDetection:
+    @pytest.mark.parametrize("tag", ["v1.2.3-rc1", "v1.2.3-rc10", "1.2.3-beta",
+                                     "v1.2.3-alpha2", "V1.2.3-RC1"])
+    def test_prereleases(self, tag):
+        assert is_prerelease(tag)
+
+    @pytest.mark.parametrize("tag", ["v1.2.3", "1.2.3", "v0.0.1"])
+    def test_stable(self, tag):
+        assert not is_prerelease(tag)
+
+    def test_base_version_strips_the_suffix(self):
+        assert base_version("v1.2.3-rc4") == "1.2.3"
+        assert base_version("v1.2.3") == "1.2.3"
+
+
+class TestUnreleasedSection:
+    def test_body_is_extracted(self):
+        text = "# C\n\n## Unreleased\n\n### Fixed\n- a thing\n\n## v1.0.0 — 2026-01-01\n\n- old\n"
+        assert "- a thing" in unreleased_body(text)
+        assert "old" not in unreleased_body(text)
+
+    def test_empty_section_reads_as_empty(self):
+        text = "# C\n\n## Unreleased\n\n## v1.0.0 — 2026-01-01\n\n- old\n"
+        assert unreleased_body(text) == ""
+
+    def test_absent_section_reads_as_empty(self):
+        assert unreleased_body("# C\n\n## v1.0.0 — 2026-01-01\n\n- old\n") == ""
+
+    def test_promote_renames_the_heading_and_keeps_the_body(self):
+        text = "# C\n\n## Unreleased\n\n### Fixed\n- a thing\n\n## v1.0.0 — 2026-01-01\n"
+        out = promote(text, "1.1.0", "2026-07-13")
+        assert "## v1.1.0 — 2026-07-13" in out
+        assert "## Unreleased" not in out
+        assert "- a thing" in out
+        assert "## v1.0.0 — 2026-01-01" in out, "older sections survive"
+
+    def test_promoting_nothing_is_refused(self):
+        # A release with no content is a release nobody needed -- and it still
+        # alerts every box.
+        with pytest.raises(ValueError, match="nothing to release"):
+            promote("# C\n\n## v1.0.0 — 2026-01-01\n", "1.1.0", "2026-07-13")
+
+
+class TestStrandedWorkBlocksAStableRelease:
+    """`check()` refuses a stable tag that leaves work under `## Unreleased`.
+
+    Either it is finished and belongs in the release, or the release is premature.
+    """
+
+    def _tree(self, tmp_path, changelog):
+        from release_notes import VERSIONED_FILES
+        for rel in VERSIONED_FILES:
+            p = tmp_path / rel
+            p.parent.mkdir(parents=True, exist_ok=True)
+            marker = "__version__ = " if rel.endswith(".py") else "VERSION="
+            p.write_text(f'{marker}"1.0.0"\n')
+        (tmp_path / "CHANGELOG.md").write_text(changelog)
+        return str(tmp_path)
+
+    def test_stranded_work_is_refused_for_a_stable_tag(self, tmp_path):
+        from release_notes import check
+        root = self._tree(tmp_path, (
+            "# C\n\n## Unreleased\n\n### Fixed\n- not done yet\n\n"
+            "## v1.0.0 — 2026-07-13\n\n### Added\n- the thing\n"
+        ))
+        problems = check("v1.0.0", root=root)
+        assert any("Unreleased" in p for p in problems)
+
+    def test_stranded_work_is_fine_for_a_candidate(self, tmp_path):
+        # An rc may legitimately have more work queued behind it.
+        from release_notes import check
+        root = self._tree(tmp_path, (
+            "# C\n\n## Unreleased\n\n### Fixed\n- later\n\n"
+            "## v1.0.0 — 2026-07-13\n\n### Added\n- the thing\n"
+        ))
+        assert check("v1.0.0-rc1", root=root) == []
+
+    def test_a_clean_stable_release_passes(self, tmp_path):
+        from release_notes import check
+        root = self._tree(tmp_path, (
+            "# C\n\n## v1.0.0 — 2026-07-13\n\n### Added\n- the thing\n"
+        ))
+        assert check("v1.0.0", root=root) == []
+
+    def test_a_candidate_checks_against_its_base_version(self, tmp_path):
+        # The scripts say 1.0.0; the tag says v1.0.0-rc3. That must agree, not clash.
+        from release_notes import check
+        root = self._tree(tmp_path, (
+            "# C\n\n## v1.0.0 — 2026-07-13\n\n### Added\n- the thing\n"
+        ))
+        assert check("v1.0.0-rc3", root=root) == []

tests/test_release_notes.py

@@ -0,0 +1,235 @@
+"""Tests for the release automation.
+
+The release workflow refuses to publish unless these hold, so a bad tag fails
+loudly in CI instead of shipping a release whose notes are empty, wrong, or whose
+scripts announce a different version than the tag.
+
+That last one is not hypothetical: VERSION= drifted to three different values
+across install.sh / uninstall.sh / recover.sh / apply.sh and nothing noticed.
+"""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
+
+from release_notes import (  # noqa: E402
+    changelog_versions,
+    check,
+    extract_notes,
+    normalise,
+    script_versions,
+    significance,
+    version_tuple,
+)
+
+REPO = os.path.join(os.path.dirname(__file__), "..")
+
+SAMPLE = """\
+# Changelog
+
+## v0.3.0 — 2026-07-13
+
+### Added
+
+- the new thing
+
+## v0.2.1 — 2026-07-09
+
+### Fixed
+
+- the old thing
+
+## v0.2.0 — 2026-07-08
+
+- first
+"""
+
+
+class TestExtractNotes:
+    def test_returns_only_that_versions_body(self):
+        body = extract_notes(SAMPLE, "v0.3.0")
+        assert "the new thing" in body
+        assert "the old thing" not in body
+        # The version heading itself is dropped (GitHub renders its own title),
+        # but sub-headings like "### Added" must survive.
+        assert not body.startswith("## v")
+        assert body.startswith("### Added")
+
+    def test_stops_at_the_next_version_heading(self):
+        body = extract_notes(SAMPLE, "v0.2.1")
+        assert "the old thing" in body
+        assert "first" not in body
+
+    def test_last_section_runs_to_end_of_file(self):
+        assert "first" in extract_notes(SAMPLE, "v0.2.0")
+
+    def test_accepts_the_tag_with_or_without_the_v(self):
+        assert extract_notes(SAMPLE, "0.3.0") == extract_notes(SAMPLE, "v0.3.0")
+
+    def test_unknown_version_raises_rather_than_returning_empty(self):
+        # An empty release body is worse than a failed release.
+        with pytest.raises(KeyError, match="no section"):
+            extract_notes(SAMPLE, "v9.9.9")
+
+
+class TestChangelogVersions:
+    def test_lists_versions_newest_first(self):
+        assert changelog_versions(SAMPLE) == ["0.3.0", "0.2.1", "0.2.0"]
+
+
+class TestAgainstTheRealRepo:
+    """These run against the actual files, so drift breaks the build."""
+
+    def test_every_script_declares_a_version(self):
+        from release_notes import VERSIONED_FILES
+
+        found = script_versions(REPO)
+        missing = [f for f in VERSIONED_FILES if f not in found]
+        assert not missing, f"no VERSION= in: {missing}"
+
+    def test_all_scripts_agree_on_the_version(self):
+        versions = {normalise(v) for v in script_versions(REPO).values()}
+        assert len(versions) == 1, f"scripts disagree on version: {sorted(versions)}"
+
+    def test_the_current_version_has_a_changelog_section(self):
+        version = next(iter({normalise(v) for v in script_versions(REPO).values()}))
+        with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh:
+            body = extract_notes(fh.read(), version)
+        assert body, f"CHANGELOG.md has no content for v{version}"
+
+    def test_the_current_version_is_the_newest_changelog_entry(self):
+        version = next(iter({normalise(v) for v in script_versions(REPO).values()}))
+        with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh:
+            newest = changelog_versions(fh.read())[0]
+        assert newest == version, (
+            f"scripts say v{version} but the newest CHANGELOG entry is v{newest}"
+        )
+
+    def test_the_repo_is_always_releasable_as_a_candidate(self):
+        # Deliberately checked as an rc, not as a stable release. `main` carries
+        # work under `## Unreleased` most of the time, and the stable gate refuses
+        # that on purpose -- shipping with work stranded mid-section is how you get
+        # a release that needs another release. So the invariant main must uphold is
+        # the candidate one: versions agree, and the CHANGELOG section exists.
+        version = next(iter({normalise(v) for v in script_versions(REPO).values()}))
+        assert check(f"v{version}-rc1", REPO) == []
+
+
+class TestCheckCatchesMistakes:
+    def test_reports_a_tag_that_no_script_matches(self):
+        problems = check("v9.9.9", REPO)
+        assert problems
+        assert any("declares VERSION" in p for p in problems)
+
+    def test_reports_a_missing_changelog_section(self):
+        problems = check("v9.9.9", REPO)
+        assert any("no section" in p for p in problems)
+
+
+class TestSignificance:
+    """Drives the TrueNAS update alert: what is worth bothering a human about.
+
+    The rule: a release whose CHANGELOG only has a "### Docs" section changed no
+    code, and nobody should get an alert because a README was reworded.
+    """
+
+    TEXT = """\
+# Changelog
+
+## v0.4.2 — 2026-07-13
+
+### Docs
+
+- reworded the README
+
+## v0.4.1 — 2026-07-13
+
+### Fixed
+
+- a real bug
+
+## v0.4.0 — 2026-07-13
+
+### Added
+
+- a feature
+
+## v0.3.3 — 2026-07-13
+
+### Security
+
+- keep a password out of argv
+
+## v0.3.2 — 2026-07-13
+
+### Fixed
+
+- something
+"""
+
+    def test_docs_only_release_does_not_alert(self):
+        level, versions, _ = significance(self.TEXT, "0.4.1", "0.4.2")
+        assert level == "docs"
+        assert versions == ["0.4.2"]
+
+    def test_a_real_fix_alerts(self):
+        level, _v, _h = significance(self.TEXT, "0.4.0", "0.4.1")
+        assert level == "notable"
+
+    def test_security_in_range_escalates(self):
+        level, _v, _h = significance(self.TEXT, "0.3.2", "0.3.3")
+        assert level == "security"
+
+    def test_security_wins_even_when_the_newest_release_is_docs_only(self):
+        # A docs-only v0.4.2 sitting on top of a security-fixing v0.3.3 must still
+        # be reported as security — classify the whole span, not just the tip.
+        level, versions, _ = significance(self.TEXT, "0.3.2", "0.4.2")
+        assert level == "security"
+        assert set(versions) == {"0.3.3", "0.4.0", "0.4.1", "0.4.2"}
+
+    def test_same_version_is_never_notable(self):
+        level, versions, _ = significance(self.TEXT, "0.4.2", "0.4.2")
+        assert level == "docs"
+        assert versions == []
+
+    def test_range_is_exclusive_of_current_inclusive_of_latest(self):
+        _l, versions, _h = significance(self.TEXT, "0.4.0", "0.4.2")
+        assert "0.4.0" not in versions
+        assert "0.4.2" in versions
+
+    def test_version_tuple_orders_correctly(self):
+        assert version_tuple("v0.10.0") > version_tuple("v0.9.9")
+        assert version_tuple("0.4.2") > version_tuple("0.4.1")
+        # Pre-release suffixes are dropped, not ranked above the release.
+        assert version_tuple("v0.5.0-rc1") == version_tuple("v0.5.0")
+
+
+class TestCandidateNotesResolveToTheBaseVersion:
+    """`notes v0.6.0-rc1` must return v0.6.0's section.
+
+    A candidate ships the same code as the release it is a candidate for, and the
+    CHANGELOG only ever has the one section. Without this, the release workflow cut
+    v0.6.0-rc1, passed every gate, and then died extracting the body -- so the tag
+    existed but nothing was ever published. Caught in an rc, which is the entire
+    point of having them.
+    """
+
+    CHANGELOG = "# C\n\n## v0.6.0 — 2026-07-13\n\n### Added\n- the thing\n\n## v0.5.1 — 2026-07-13\n\n- older\n"
+
+    def test_an_rc_resolves_to_its_base_version(self):
+        body = extract_notes(self.CHANGELOG, "v0.6.0-rc1")
+        assert "the thing" in body
+        assert "older" not in body
+
+    def test_rc10_too(self):
+        assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0-rc10")
+
+    def test_the_plain_version_still_works(self):
+        assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0")
+
+    def test_a_genuinely_missing_section_still_raises(self):
+        with pytest.raises(KeyError):
+            extract_notes(self.CHANGELOG, "v9.9.9-rc1")

tests/test_truecloud_nested.py

@@ -0,0 +1,611 @@
+"""Tests for nested-dataset snapshot staging.
+
+Two rules are under test above all else:
+
+1. A tree that cannot be staged completely must fail LOUDLY. A silently
+   incomplete backup is the exact failure that stock TrueNAS's "no further
+   nesting" guard exists to prevent.
+
+2. Every snapshot we cause to exist must be cleaned up. ``zfs.snapshot.delete``
+   is non-recursive by default and stock calls it with no options, so a
+   recursive snapshot would otherwise orphan one snapshot per descendant dataset
+   on EVERY run.
+"""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "patch"))
+
+from truecloud_nested import (  # noqa: E402
+    StagingError,
+    apply_plan,
+    cleanup_all,
+    cleanup_task,
+    current_mounts_under,
+    delete_snapshot_tree,
+    plan_staging,
+    sidecar_for,
+    snapshot_tree_names,
+    staging_root_for,
+    teardown,
+    verify_staged,
+)
+
+SNAP = "cloud_backup-5-20260712030000"
+ROOT = "/run/truecloud-nested/cloud_backup-5"
+
+
+def ds(name, mountpoint, mounted="yes"):
+    return {
+        "name": name,
+        "properties": {
+            "mountpoint": {"value": mountpoint},
+            "mounted": {"value": mounted},
+        },
+    }
+
+
+# Mirrors the real layout: apps are datasets, several with their own children.
+DATASETS = [
+    ds("Tap", "/mnt/Tap"),
+    ds("Tap/apps", "/mnt/Tap/apps"),
+    ds("Tap/apps/lidarr", "/mnt/Tap/apps/lidarr"),
+    ds("Tap/apps/lidarr/config", "/mnt/Tap/apps/lidarr/config"),
+    ds("Tap/apps/immich", "/mnt/Tap/apps/immich"),
+    ds("Tap/apps/immich/pgdata", "/mnt/Tap/apps/immich/pgdata"),
+]
+
+
+def yes(_path):
+    return "ok"
+
+
+def plan(datasets=DATASETS, base_dataset="Tap", base_mp="/mnt/Tap",
+         path="/mnt/Tap", probe=yes):
+    return plan_staging(base_dataset, base_mp, path, SNAP, datasets, ROOT, probe=probe)
+
+
+class TestPlanStaging:
+    def test_stages_every_descendant_dataset(self):
+        mounts, skipped = plan()
+        assert skipped == []
+        assert len(mounts) == 6  # root + 5 descendants
+        assert mounts[0] == (f"/mnt/Tap/.zfs/snapshot/{SNAP}", ROOT)
+
+        by_target = {t: s for s, t in mounts}
+        assert by_target[f"{ROOT}/apps"] == f"/mnt/Tap/apps/.zfs/snapshot/{SNAP}"
+        assert by_target[f"{ROOT}/apps/immich/pgdata"] == (
+            f"/mnt/Tap/apps/immich/pgdata/.zfs/snapshot/{SNAP}"
+        )
+
+    def test_parents_are_mounted_before_children(self):
+        # A child's mountpoint dir only exists inside its parent's snapshot, so
+        # mounting a child first would fail.
+        mounts, _ = plan()
+        seen = set()
+        for _src, target in mounts:
+            if target != ROOT:
+                assert os.path.dirname(target) in seen
+            seen.add(target)
+
+    def test_backup_path_below_dataset_root(self):
+        mounts, _ = plan(base_dataset="Tap/apps", base_mp="/mnt/Tap/apps",
+                         path="/mnt/Tap/apps")
+        assert mounts[0] == (f"/mnt/Tap/apps/.zfs/snapshot/{SNAP}", ROOT)
+        targets = [t for _s, t in mounts]
+        assert f"{ROOT}/lidarr" in targets
+        assert f"{ROOT}/apps/lidarr" not in targets
+
+    def test_base_dataset_is_not_a_descendant_of_itself(self):
+        mounts, _ = plan(datasets=[ds("Tap", "/mnt/Tap")])
+        assert len(mounts) == 1
+
+
+class TestScoping:
+    def test_unrelated_datasets_are_ignored_silently(self):
+        # Regression: scoping by mountpoint first dragged in every
+        # mountpoint-less dataset on the box (all of Tank/.system/*), burying the
+        # warnings that actually matter.
+        noisy = DATASETS + [
+            ds("Tank/.system", "none"),
+            ds("Tank/.system/cores", "legacy"),
+            ds("Tank/backups", "/mnt/Tank/backups"),
+        ]
+        mounts, skipped = plan(datasets=noisy)
+        assert len(mounts) == 6
+        assert skipped == [], "datasets outside the base dataset must not be reported"
+
+    def test_in_scope_dataset_without_mountpoint_is_reported(self):
+        datasets = DATASETS + [ds("Tap/apps/weird", "none")]
+        _mounts, skipped = plan(datasets=datasets)
+        assert ("Tap/apps/weird", "mountpoint is none") in skipped
+
+    def test_unmounted_dataset_is_skipped_but_never_silently(self):
+        datasets = DATASETS + [ds("Tap/apps/vault", "/mnt/Tap/apps/vault", mounted="no")]
+        mounts, skipped = plan(datasets=datasets)
+        assert f"{ROOT}/apps/vault" not in [t for _s, t in mounts]
+        assert ("Tap/apps/vault", "dataset is not mounted (locked/encrypted?)") in skipped
+
+    def test_descendant_mounted_outside_the_path_is_not_an_omission(self):
+        datasets = DATASETS + [ds("Tap/elsewhere", "/mnt/other")]
+        mounts, skipped = plan(datasets=datasets)
+        assert len(mounts) == 6
+        assert skipped == []
+
+
+class TestSilentOmissionGuard:
+    """The whole point of the feature. These are the tests that matter."""
+
+    @staticmethod
+    def _missing_pgdata(path):
+        return "missing" if "/mnt/Tap/apps/immich/pgdata/" in path else "ok"
+
+    @staticmethod
+    def _denied_pgdata(path):
+        if "/mnt/Tap/apps/immich/pgdata/" in path:
+            return "cannot be read (Permission denied)"
+        return "ok"
+
+    def test_missing_snapshot_on_descendant_raises(self):
+        with pytest.raises(StagingError, match="incomplete tree"):
+            plan(probe=self._missing_pgdata)
+
+    def test_error_names_the_offending_dataset(self):
+        with pytest.raises(StagingError, match="Tap/apps/immich/pgdata"):
+            plan(probe=self._missing_pgdata)
+
+    def test_missing_and_unreadable_are_reported_differently(self):
+        # os.path.isdir() collapses both into False, which would report a
+        # permission problem as "has no snapshot" and send you hunting for a
+        # snapshot that is sitting right there. Both abort -- but say which.
+        with pytest.raises(StagingError, match="has no snapshot"):
+            plan(probe=self._missing_pgdata)
+        with pytest.raises(StagingError, match="Permission denied"):
+            plan(probe=self._denied_pgdata)
+
+
+class TestSnapshotTreeNames:
+    """zfs.snapshot.delete is non-recursive; we must sweep children ourselves."""
+
+    ALL = [
+        "Tap@cloud_backup-5-20260712030000",
+        "Tap/apps@cloud_backup-5-20260712030000",
+        "Tap/apps/lidarr/config@cloud_backup-5-20260712030000",
+        "Tap@auto-2026-07-12_03-00",            # unrelated periodic snapshot
+        "Tap/apps@cloud_backup-9-20260712030000",  # another task
+        "Tank/backups@cloud_backup-5-20260712030000",  # different pool
+    ]
+
+    def test_returns_parent_and_all_children(self):
+        got = snapshot_tree_names("Tap@cloud_backup-5-20260712030000", self.ALL)
+        assert set(got) == {
+            "Tap@cloud_backup-5-20260712030000",
+            "Tap/apps@cloud_backup-5-20260712030000",
+            "Tap/apps/lidarr/config@cloud_backup-5-20260712030000",
+        }
+
+    def test_never_touches_periodic_or_other_tasks_or_other_pools(self):
+        got = snapshot_tree_names("Tap@cloud_backup-5-20260712030000", self.ALL)
+        assert "Tap@auto-2026-07-12_03-00" not in got
+        assert "Tap/apps@cloud_backup-9-20260712030000" not in got
+        assert "Tank/backups@cloud_backup-5-20260712030000" not in got
+
+    def test_malformed_snapshot_name_yields_nothing(self):
+        assert snapshot_tree_names("Tap", self.ALL) == []
+
+
+class FakeMiddleware:
+    """middlewared as this module actually uses it: `call_sync`, from a thread.
+
+    The module is synchronous on purpose -- see the orchestration note in
+    truecloud_nested.py. TrueNAS <= 25.10 reaches it through
+    `await middleware.run_in_thread(...)` and TrueNAS 26 calls it directly, but the
+    logic below the boundary is the same code either way, so it is tested once.
+    """
+
+    def __init__(self, snapshots=None):
+        self.snapshots = list(snapshots or [])
+        self.calls = []
+        self.logger = None
+
+    def call_sync(self, method, *args):
+        self.calls.append((method, args))
+        if method == "zfs.snapshot.query":
+            return [{"name": n} for n in self.snapshots]
+        if method == "zfs.snapshot.delete":
+            name = args[0]
+            opts = args[1] if len(args) > 1 else {}
+            if name not in self.snapshots:
+                raise RuntimeError("does not exist")
+            if opts.get("recursive"):
+                # Real `zfs destroy -r` takes the parent and every child snapshot.
+                for n in snapshot_tree_names(name, list(self.snapshots)):
+                    self.snapshots.remove(n)
+            else:
+                self.snapshots.remove(name)
+            return True
+        raise AssertionError(f"unexpected call {method}")
+
+
+def stub_core(monkeypatch, tn, *, plan=None, order=None, plan_raises=None):
+    """Replace the blocking core (plan/apply/verify/teardown) with recorders.
+
+    stage_nested calls these directly now, so they are patched by NAME rather than
+    intercepted at a `run_in_thread` boundary that no longer exists.
+    """
+    def record(name, result):
+        def fn(*args, **kwargs):
+            if order is not None:
+                order.append(name)
+            if name == "plan_staging" and plan_raises is not None:
+                raise plan_raises
+            return result() if callable(result) else result
+        fn.__name__ = name
+        return fn
+
+    real_write = tn._write_sidecar
+
+    def write_sidecar(*args, **kwargs):
+        if order is not None:
+            order.append("_write_sidecar")
+        return real_write(*args, **kwargs)
+
+    monkeypatch.setattr(tn, "_write_sidecar", write_sidecar)
+    monkeypatch.setattr(tn, "plan_staging", record("plan_staging", plan or ([], [])))
+    monkeypatch.setattr(tn, "apply_plan", record("apply_plan", True))
+    monkeypatch.setattr(tn, "verify_staged", record("verify_staged", True))
+    monkeypatch.setattr(tn, "teardown", record("teardown", []))
+
+
+class TestDeleteSnapshotTree:
+    def test_deletes_parent_and_every_child(self):
+        mw = FakeMiddleware([
+            "Tap@snap", "Tap/apps@snap", "Tap/apps/lidarr@snap", "Tap@keepme",
+        ])
+        delete_snapshot_tree(mw, "Tap@snap")
+        assert mw.snapshots == ["Tap@keepme"]
+
+    def test_is_idempotent_when_stock_already_removed_the_parent(self):
+        # Stock's finally can win the race once our mounts are released.
+        mw = FakeMiddleware(["Tap/apps@snap", "Tap/apps/lidarr@snap"])
+        delete_snapshot_tree(mw, "Tap@snap")
+        assert mw.snapshots == []
+
+    def test_uses_a_single_recursive_delete_not_252_individual_ones(self):
+        # 252 sequential deletes are slow AND not atomic: a run killed part-way
+        # through leaves exactly the orphans this function exists to prevent.
+        mw = FakeMiddleware(["Tap@snap", "Tap/apps@snap", "Tap/apps/lidarr@snap"])
+        delete_snapshot_tree(mw, "Tap@snap")
+        assert mw.snapshots == []
+        deletes = [a for m, a in mw.calls if m == "zfs.snapshot.delete"]
+        assert len(deletes) == 1, "should be ONE recursive call, not one per snapshot"
+        assert deletes[0][1] == {"recursive": True}
+        assert not [m for m, _a in mw.calls if m == "zfs.snapshot.query"], (
+            "no enumeration needed on the fast path"
+        )
+
+    def test_survives_recursive_and_query_failure_by_deleting_the_parent(self):
+        class Broken(FakeMiddleware):
+            def call_sync(self, method, *args):
+                if method == "zfs.snapshot.query":
+                    raise RuntimeError("boom")
+                if method == "zfs.snapshot.delete" and len(args) > 1:
+                    raise RuntimeError("recursive delete unavailable")
+                return super().call_sync(method, *args)
+
+        mw = Broken(["Tap@snap"])
+        delete_snapshot_tree(mw, "Tap@snap")
+        assert mw.snapshots == []
+
+    def test_leaves_unrelated_snapshots_alone_when_the_tree_is_gone(self):
+        mw = FakeMiddleware(["Tap@unrelated"])
+        delete_snapshot_tree(mw, "Tap@snap")
+        assert mw.snapshots == ["Tap@unrelated"]
+
+
+class TestStageNestedOrdering:
+    def test_sidecar_is_written_before_anything_is_mounted(self, tmp_path, monkeypatch):
+        # middlewared can die at any moment. If the snapshot were recorded only
+        # after apply_plan, a crash in that window would orphan a 160-snapshot
+        # tree -- the precise failure the sidecar exists to prevent.
+        import truecloud_nested as tn
+
+        monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
+        order = []
+        stub_core(monkeypatch, tn, order=order,
+                  plan=([("/src", str(tmp_path / "cloud_backup-5"))], []))
+
+        tn.stage_nested(
+            FakeMiddleware(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap",
+            "cloud_backup-5", DATASETS,
+        )
+
+        assert order.index("_write_sidecar") < order.index("apply_plan")
+
+    def test_reclaims_the_snapshot_tree_left_by_a_crashed_run(self, tmp_path,
+                                                              monkeypatch):
+        # teardown() reclaims the crashed run's MOUNTS, but nothing else would
+        # ever reclaim its SNAPSHOTS -- and we are about to overwrite the only
+        # record of them. One crash would orphan 160+ snapshots permanently.
+        import truecloud_nested as tn
+
+        monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
+        root = tn.staging_root_for("cloud_backup-5")
+        os.makedirs(os.path.dirname(root), exist_ok=True)
+        with open(sidecar_for(root), "w", encoding="utf-8") as fh:
+            fh.write("Tap@old-crashed-run")
+
+        mw = FakeMiddleware(["Tap@old-crashed-run", "Tap/apps@old-crashed-run"])
+        stub_core(monkeypatch, tn, plan=([("/src", root)], []))
+
+        tn.stage_nested(
+            mw, "/mnt/Tap", "Tap@new", "Tap", "/mnt/Tap",
+            "cloud_backup-5", DATASETS,
+        )
+
+        assert mw.snapshots == [], "the crashed run's snapshot tree must be reclaimed"
+
+    def test_sidecar_is_removed_when_staging_fails(self, tmp_path, monkeypatch):
+        import truecloud_nested as tn
+
+        monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
+        root = tn.staging_root_for("cloud_backup-5")
+        stub_core(monkeypatch, tn, plan_raises=StagingError("boom"))
+
+        with pytest.raises(StagingError):
+            tn.stage_nested(
+                FakeMiddleware(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap",
+                "cloud_backup-5", DATASETS,
+            )
+
+        assert not os.path.exists(sidecar_for(root))
+
+
+class TestCleanupTask:
+    def test_recovers_snapshot_from_sidecar_after_middlewared_restart(self, tmp_path,
+                                                                      monkeypatch):
+        # The sidecar is the ONLY record of the pinned snapshot, precisely so a
+        # middlewared restart cannot orphan the tree.
+        import truecloud_nested as tn
+
+        monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
+        root = tn.staging_root_for("cloud_backup-5", base=str(tmp_path))
+        os.makedirs(root, exist_ok=True)
+        with open(sidecar_for(root), "w", encoding="utf-8") as fh:
+            fh.write("Tap@snap")
+
+        mw = FakeMiddleware(["Tap@snap", "Tap/apps@snap"])
+        monkeypatch.setattr(tn, "teardown", lambda *_a, **_k: [])
+
+        cleanup_task(mw, "cloud_backup-5")
+
+        assert mw.snapshots == []
+        assert not os.path.exists(sidecar_for(root))
+
+    def test_is_a_noop_when_never_staged(self, tmp_path, monkeypatch):
+        import truecloud_nested as tn
+
+        monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path / "nope"))
+        mw = FakeMiddleware(["Tap@snap"])
+        cleanup_task(mw, "cloud_backup-5")
+        assert mw.calls == []
+        assert mw.snapshots == ["Tap@snap"]
+
+
+class TestVerifyStaged:
+    """Anti-regression guard: proves the staged tree is real before we back it up."""
+
+    def test_passes_when_every_target_is_mounted_and_root_non_empty(self):
+        mounts = [("/src", ROOT), ("/src/a", f"{ROOT}/a")]
+        assert verify_staged(mounts, ismount=lambda p: True, listdir=lambda p: ["apps"])
+
+    def test_raises_when_a_target_is_not_actually_mounted(self):
+        # This is the case that would produce a silently-empty backup.
+        mounts = [("/src", ROOT), ("/src/a", f"{ROOT}/a")]
+        with pytest.raises(StagingError, match="not a mountpoint"):
+            verify_staged(mounts, ismount=lambda p: p == ROOT, listdir=lambda p: ["apps"])
+
+    def test_raises_when_staging_root_is_empty(self):
+        with pytest.raises(StagingError, match="empty"):
+            verify_staged([("/src", ROOT)], ismount=lambda p: True, listdir=lambda p: [])
+
+    def test_raises_on_empty_plan(self):
+        with pytest.raises(StagingError):
+            verify_staged([])
+
+
+class FakeRunner:
+    def __init__(self, fail_on=None):
+        self.fail_on = fail_on
+        self.calls = []
+
+    def __call__(self, cmd):
+        self.calls.append(cmd)
+
+        class R:
+            returncode = 0
+            stderr = ""
+
+        if self.fail_on and cmd[0] == "mount" and cmd[2] == self.fail_on:
+            R.returncode = 32
+            R.stderr = "mount failed"
+        return R
+
+
+class TestApplyPlanRollback:
+    def test_rolls_back_mounts_when_one_fails(self, tmp_path):
+        # A half-built tree must never reach the backup tool.
+        root = str(tmp_path / "root")
+        mounts = [("/src", root), ("/src/a", root + "/a"), ("/src/b", root + "/b")]
+
+        runner = FakeRunner(fail_on="/src/b")
+        with pytest.raises(StagingError, match="bind-mount"):
+            apply_plan(mounts, runner=runner, isdir=lambda _p: True)
+
+        umounts = [c[-1] for c in runner.calls if c[0] == "umount"]
+        assert umounts == [root + "/a", root]
+
+    def test_raises_when_target_missing(self, tmp_path):
+        root = str(tmp_path / "root")
+        with pytest.raises(StagingError, match="does not exist"):
+            apply_plan(
+                [("/src", root), ("/src/a", root + "/a")],
+                runner=FakeRunner(),
+                isdir=lambda p: p == root,
+            )
+
+
+class TestTeardown:
+    def test_unmounts_deepest_first(self, tmp_path):
+        mounts_file = tmp_path / "mounts"
+        mounts_file.write_text(
+            f"tmpfs {ROOT} tmpfs rw 0 0\n"
+            f"tmpfs {ROOT}/apps tmpfs rw 0 0\n"
+            f"tmpfs {ROOT}/apps/lidarr/config tmpfs rw 0 0\n"
+            f"tmpfs {ROOT}/apps/lidarr tmpfs rw 0 0\n"
+            "tmpfs /somewhere/else tmpfs rw 0 0\n"
+        )
+        runner = FakeRunner()
+        teardown(ROOT, runner=runner, mounts_file=str(mounts_file))
+
+        order = [c[-1] for c in runner.calls if c[0] == "umount"]
+        assert order == [
+            f"{ROOT}/apps/lidarr/config",
+            f"{ROOT}/apps/lidarr",
+            f"{ROOT}/apps",
+            ROOT,
+        ]
+        assert "/somewhere/else" not in order
+
+    def test_is_idempotent_when_nothing_mounted(self, tmp_path):
+        mounts_file = tmp_path / "mounts"
+        mounts_file.write_text("tmpfs /somewhere/else tmpfs rw 0 0\n")
+        runner = FakeRunner()
+        assert teardown(ROOT, runner=runner, mounts_file=str(mounts_file)) == []
+        assert runner.calls == []
+
+    def test_falls_back_to_lazy_umount(self, tmp_path):
+        mounts_file = tmp_path / "mounts"
+        mounts_file.write_text(f"tmpfs {ROOT} tmpfs rw 0 0\n")
+
+        class Busy(FakeRunner):
+            def __call__(self, cmd):
+                self.calls.append(cmd)
+
+                class R:
+                    returncode = 0 if "-l" in cmd else 32
+                    stderr = "target is busy"
+
+                return R
+
+        runner = Busy()
+        assert teardown(ROOT, runner=runner, mounts_file=str(mounts_file)) == []
+        assert ["umount", "-l", ROOT] in runner.calls
+
+
+class TestCleanupAll:
+    """uninstall.sh and recover.sh call this instead of reimplementing teardown."""
+
+    def test_reports_orphan_snapshots_before_deleting_their_sidecars(self, tmp_path):
+        # The sidecar is the only record that an interrupted run's snapshot tree
+        # is still on disk. Deleting it without naming the snapshot orphans the
+        # whole tree silently.
+        base = tmp_path / "stage"
+        base.mkdir()
+        (base / "cloud_backup-5.snapshot").write_text("Tap@interrupted")
+
+        mounts_file = tmp_path / "mounts"
+        mounts_file.write_text("")
+
+        lines, errors = cleanup_all(
+            base=str(base), runner=FakeRunner(), mounts_file=str(mounts_file)
+        )
+        assert errors == []
+        assert any("Tap@interrupted" in ln for ln in lines)
+        assert any("zfs destroy -r" in ln for ln in lines)
+        # Sidecar cleared only after being reported.
+        assert not (base / "cloud_backup-5.snapshot").exists()
+
+    def test_unmounts_everything_under_the_base_deepest_first(self, tmp_path):
+        base = tmp_path / "stage"
+        base.mkdir()
+        mounts_file = tmp_path / "mounts"
+        mounts_file.write_text(
+            f"tmpfs {base} tmpfs rw 0 0\n"
+            f"tmpfs {base}/cloud_backup-5 tmpfs rw 0 0\n"
+            f"tmpfs {base}/cloud_backup-5/apps tmpfs rw 0 0\n"
+        )
+        runner = FakeRunner()
+        _lines, errors = cleanup_all(
+            base=str(base), runner=runner, mounts_file=str(mounts_file)
+        )
+        assert errors == []
+        order = [c[-1] for c in runner.calls if c[0] == "umount"]
+        assert order == [
+            f"{base}/cloud_backup-5/apps",
+            f"{base}/cloud_backup-5",
+            str(base),
+        ]
+
+    def test_keeps_sidecars_when_an_unmount_failed(self, tmp_path):
+        # If a mount is stuck, the snapshot is still pinned — so the record of it
+        # must survive for the next run (or the operator) to act on.
+        base = tmp_path / "stage"
+        base.mkdir()
+        (base / "cloud_backup-5.snapshot").write_text("Tap@stuck")
+        mounts_file = tmp_path / "mounts"
+        mounts_file.write_text(f"tmpfs {base}/cloud_backup-5 tmpfs rw 0 0\n")
+
+        class Stuck(FakeRunner):
+            def __call__(self, cmd):
+                self.calls.append(cmd)
+
+                class R:
+                    returncode = 32
+                    stderr = "target is busy"
+
+                return R
+
+        _lines, errors = cleanup_all(
+            base=str(base), runner=Stuck(), mounts_file=str(mounts_file)
+        )
+        assert errors, "a stuck unmount must be reported"
+        assert (base / "cloud_backup-5.snapshot").exists()
+
+    def test_is_a_noop_on_a_clean_system(self, tmp_path):
+        mounts_file = tmp_path / "mounts"
+        mounts_file.write_text("")
+        lines, errors = cleanup_all(
+            base=str(tmp_path / "absent"), runner=FakeRunner(),
+            mounts_file=str(mounts_file),
+        )
+        assert errors == []
+        assert lines == ["  None active."]
+
+
+class TestCurrentMountsUnder:
+    def test_matches_only_the_staging_subtree(self, tmp_path):
+        mounts_file = tmp_path / "mounts"
+        # "cloud_backup-50" must NOT match "cloud_backup-5".
+        mounts_file.write_text(
+            f"tmpfs {ROOT} tmpfs rw 0 0\n"
+            "tmpfs /run/truecloud-nested/cloud_backup-50 tmpfs rw 0 0\n"
+        )
+        assert current_mounts_under(ROOT, mounts_file=str(mounts_file)) == [ROOT]
+
+
+class TestStagingRootFor:
+    def test_stable_per_task(self):
+        assert staging_root_for("cloud_backup-5") == "/run/truecloud-nested/cloud_backup-5"
+
+    def test_sanitises_path_separators(self):
+        assert "/" not in staging_root_for("evil/name").rsplit("/", 1)[-1]
+
+    @pytest.mark.parametrize("name", ["..", ".", "...", "/", ""])
+    def test_dot_components_cannot_escape_the_staging_base(self, name):
+        # os.path.join(BASE, "..") normalises to /run — teardown would rmdir it.
+        root = staging_root_for(name)
+        assert os.path.normpath(root).startswith("/run/truecloud-nested/")

tests/test_workflows.py

@@ -0,0 +1,107 @@
+"""Guards on the CI workflows themselves.
+
+The workflows run on TWO forges -- Gitea (canonical) and GitHub (mirror), because
+Gitea reads .github/workflows too -- and they hold tokens. A mistake here is not a
+failed build, it is a bug report nobody files or a command nobody meant to run.
+"""
+
+import os
+import re
+
+import pytest
+
+WORKFLOWS = os.path.join(os.path.dirname(__file__), "..", ".github", "workflows")
+
+
+def workflow_files():
+    return [
+        os.path.join(WORKFLOWS, f)
+        for f in sorted(os.listdir(WORKFLOWS))
+        if f.endswith((".yml", ".yaml"))
+    ]
+
+
+def run_bodies(path):
+    """Every `run:` block's text, with its line number."""
+    with open(path, encoding="utf-8") as fh:
+        lines = fh.readlines()
+
+    out = []
+    i = 0
+    while i < len(lines):
+        m = re.match(r"^(\s*)run:\s*\|", lines[i])
+        if not m:
+            i += 1
+            continue
+        indent = len(m.group(1))
+        start = i + 1
+        body = []
+        i += 1
+        while i < len(lines):
+            line = lines[i]
+            if line.strip() and (len(line) - len(line.lstrip())) <= indent:
+                break
+            body.append(line)
+            i += 1
+        out.append((start + 1, "".join(body)))
+    return out
+
+
+class TestNoExpressionInterpolationIntoShell:
+    """`${{ ... }}` inside a `run:` body is spliced into the SCRIPT TEXT.
+
+    This is not theoretical. `echo "${{ steps.report.outputs.body }}"` in the compat
+    workflow pasted the report -- which is full of backticks -- straight into bash,
+    which promptly ran `create-snapshot`, `def` and `async` as commands. And because
+    that report is built from iX's middleware source, anything landing in their tree
+    would have executed on our runner.
+
+    The rule: files for data, `env:` for scalars. `env:` is safe because the runner
+    sets the variable rather than pasting it into the script.
+    """
+
+    @pytest.mark.parametrize("path", workflow_files(), ids=os.path.basename)
+    def test_no_github_expression_in_a_run_body(self, path):
+        offenders = []
+        for lineno, body in run_bodies(path):
+            for m in re.finditer(r"\$\{\{[^}]*\}\}", body):
+                offenders.append(f"{os.path.basename(path)}:~{lineno}: {m.group(0)}")
+        assert not offenders, (
+            "GitHub/Gitea expressions interpolate into the shell script text, so "
+            "backticks and $() in the value EXECUTE. Pass data via a file, or a "
+            "scalar via `env:`.\n  " + "\n  ".join(offenders)
+        )
+
+
+class TestBothForges:
+    """Gitea is canonical; GitHub is a mirror. Both run these files."""
+
+    def test_release_publishes_on_each_forge_exactly_once(self):
+        with open(os.path.join(WORKFLOWS, "release.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        # One step gated ON github.com, one gated OFF it. Without the pair, a release
+        # either double-publishes or silently never publishes on the canonical host.
+        assert "if: ${{ contains(github.server_url, 'github.com') }}" in src
+        assert "if: ${{ !contains(github.server_url, 'github.com') }}" in src
+
+    def test_compat_files_an_issue_on_each_forge(self):
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        assert "file a bug report (GitHub)" in src
+        assert "file a bug report (Gitea)" in src
+
+
+class TestCompatCannotSilentlyPass:
+    def test_the_exit_code_is_captured_not_swallowed(self):
+        # Actions runs `bash -e`: `cmd > out` followed by `echo $?` never reaches the
+        # echo, so the "a shipped release is broken" signal would be lost and the job
+        # would go green while users were broken.
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        assert "|| rc=$?" in src
+        assert "shipped_broken=$rc" in src
+
+    def test_a_broken_shipped_release_fails_the_job(self):
+        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
+            src = fh.read()
+        assert "steps.check.outputs.shipped_broken != '0'" in src

tools/compat.py

@@ -0,0 +1,993 @@
+#!/usr/bin/env python3
+"""What this patch assumes about middlewared -- written down, and checkable.
+
+WHY THIS EXISTS
+---------------
+This patch appends code to middlewared's own modules. middlewared has no stability
+contract: it is internal API, and iX may reshape it in any release. When they do,
+the patch does not politely decline -- it breaks a backup, possibly silently, which
+is the worst thing a backup tool can do.
+
+It has already happened. TrueNAS 26 rewrites the whole cloud_backup path from
+async to synchronous:
+
+    25.10:  async def create_snapshot(...)   /  await create_snapshot(...)
+    26.0:         def create_snapshot(...)   /        create_snapshot(...)
+
+Every block the nested module injects is an `async def` wrapping an `await`ed
+original. On 26 that unpacks a coroutine object instead of a tuple. Nobody would
+have found out until a restore failed.
+
+So the assumptions are written down here, once, and checked in two places:
+
+  * .github/workflows/compat.yml runs `--ref` against TrueNAS's *unreleased*
+    branches (master, the newest BETA/RC) on a schedule, and opens a bug report
+    the day iX breaks us -- while it is still a beta, not after it ships.
+
+  * patch/apply.sh runs `--tree` against the middlewared *actually installed*, at
+    every boot, and REFUSES to patch a module whose assumptions no longer hold.
+    That is the guarantee: an unpatched module means stock TrueNAS (Storj only,
+    but working). A patched-anyway module means broken backups. Declining is
+    always the better failure.
+
+The two modules are checked independently, because they fail independently: on 26
+the providers module (B2/S3) only touches synchronous symbols and survives, while
+the nested module does not.
+
+    python3 tools/compat.py --tree /usr/lib/python3/dist-packages/middlewared
+    python3 tools/compat.py --ref release/26.0.0-BETA.3
+    python3 tools/compat.py --ref master --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import json
+import os
+import sys
+import urllib.request
+
+PROVIDERS = "providers"
+NESTED = "nested"
+
+RAW = "https://raw.githubusercontent.com/truenas/middleware/{ref}/src/middlewared/middlewared/{path}"
+
+_TIMEOUT = 30
+
+
+class Assumption:
+    """One thing that must be true of middlewared, or a module cannot be applied.
+
+    `is_async=None` means "do not care". Everywhere else it is stated explicitly,
+    because asyncness is exactly the axis TrueNAS 26 changed and a checker that
+    ignored it would have passed a build that breaks every backup.
+    """
+
+    def __init__(self, ident, module, path, symbol, *, kind="function",
+                 is_async=None, params=None, forwards=False, why=""):
+        self.id = ident
+        self.module = module
+        self.path = path
+        self.symbol = symbol
+        self.kind = kind
+        self.is_async = is_async
+        #: The positional parameters the patch passes, in order.
+        self.params = params or []
+        #: True if the wrapper takes *args/**kwargs and forwards the rest. Then a
+        #: trailing parameter that iX adds or removes is harmless, and only the
+        #: leading `params` must still match.
+        self.forwards = forwards
+        self.why = why
+
+
+#: Everything patch/apply.sh's injected blocks depend on. Derived from the blocks
+#: themselves -- if you add a block, add its assumptions here or the checker is
+#: decoration.
+ASSUMPTIONS = [
+    # ── providers (B2/S3). Touches only synchronous symbols. ──────────────────
+    Assumption(
+        "b2-remote-class", PROVIDERS, "rclone/remote/b2.py", "B2RcloneRemote",
+        kind="class",
+        why="B2_BLOCK sets .get_restic_config and .restic on this class",
+    ),
+    Assumption(
+        "restic-config-fn", PROVIDERS, "plugins/cloud_backup/restic.py",
+        "get_restic_config", is_async=False, params=["cloud_backup"],
+        why="RESTIC_BLOCK wraps it to rewrite the repo URL; it calls the original "
+            "WITHOUT await, so it must stay synchronous",
+    ),
+    Assumption(
+        "restic-config-class", PROVIDERS, "plugins/cloud_backup/restic.py",
+        "ResticConfig", kind="class",
+        why="RESTIC_BLOCK does dataclasses.replace(result, cmd=...) on what "
+            "get_restic_config returns",
+    ),
+
+    # ── nested snapshots. Every block here is an async wrapper. ───────────────
+    # is_async is deliberately NOT asserted on these three. The patch now injects an
+    # async OR a sync wrapper to match whichever the installed middleware declares
+    # (TrueNAS <= 25.10 is async; 26 rewrote them synchronous), so asyncness is a
+    # thing to DETECT, not a thing to require -- see async_flavour(). What must still
+    # hold is the shape: same name, same leading positional parameters.
+    Assumption(
+        "create-snapshot", NESTED, "plugins/cloud/snapshot.py", "create_snapshot",
+        params=["middleware", "path", "name"],
+        why="SNAPSHOT_BLOCK wraps it and returns (snapshot, staging_root) instead of "
+            "(snapshot, snap_path)",
+    ),
+    Assumption(
+        "crud-mixin-validate", NESTED, "plugins/cloud/crud.py",
+        "CloudTaskServiceMixin._validate",
+        kind="method", params=["self", "app", "verrors", "name", "data"],
+        why="CRUD_BLOCK wraps it to drop the no-further-nesting error",
+    ),
+    Assumption(
+        # SYNC_BLOCK's wrapper is (middleware, job, cloud_backup, *args, **kwargs) and
+        # forwards the rest, precisely because iX keeps changing the tail: 24.10 and
+        # 25.04 have `(…, dry_run)`, 25.10 added `rate_limit`. Only the leading three
+        # are named by the patch, so only they have to hold.
+        "restic-backup", NESTED, "plugins/cloud_backup/sync.py", "restic_backup",
+        forwards=True,
+        params=["middleware", "job", "cloud_backup"],
+        why="SYNC_BLOCK wraps it to tear down bind mounts in a finally",
+    ),
+]
+
+
+class MiddlewareCall:
+    """A middlewared METHOD the injected code calls at runtime.
+
+    THIS CLASS OF ASSUMPTION IS WHY THE CHECKER EXISTS, AND IT WAS THE ONE MISSING.
+
+    The manifest above records the symbols the patch *wraps*. It said nothing about
+    the methods the patch *calls* -- and that gap hid two separate TrueNAS 26 breaks
+    that both pass every other check:
+
+      * `get_dataset_recursive()` was deleted from plugins/cloud/snapshot.py, and the
+        injected block called it out of the host module's namespace (now vendored).
+      * plugins/zfs_/dataset.py and plugins/zfs_/snapshot.py were DELETED outright,
+        taking `zfs.dataset.query`, `zfs.snapshot.query` and `zfs.snapshot.delete`
+        with them. 26 uses filesystem.statfs and zfs.resource.* instead.
+
+    Nothing about the five cloud_backup files reveals that. The patch would apply
+    perfectly, and then the FIRST BACKUP would fail -- or, far worse, succeed at
+    snapshotting and fail at `zfs.snapshot.delete`, orphaning one snapshot per
+    descendant dataset (250 on a real pool) on every single run, forever.
+
+    A method is present when some plugin file declares its namespace AND defines it.
+    If iX merely MOVES a method to a different file we report BROKEN wrongly, and the
+    module declines to apply -- costing a feature, not a backup. That asymmetry is
+    the whole design: declining is always the cheaper mistake.
+    """
+
+    def __init__(self, ident, module, method, path, why=""):
+        self.id = ident
+        self.module = module
+        self.method = method              # "zfs.snapshot.delete"
+        self.path = path                  # plugin file that declares it
+        self.why = why
+
+    @property
+    def namespace(self):
+        return self.method.rsplit(".", 1)[0]
+
+    @property
+    def name(self):
+        return self.method.rsplit(".", 1)[1]
+
+
+#: Every middlewared method the nested module calls at runtime.
+MIDDLEWARE_CALLS = [
+    MiddlewareCall(
+        "call-zfs-dataset-query", NESTED, "zfs.dataset.query",
+        "plugins/zfs_/dataset.py",
+        why="SNAPSHOT_BLOCK enumerates FILESYSTEM datasets to build the staging plan",
+    ),
+    MiddlewareCall(
+        "call-zfs-snapshot-delete", NESTED, "zfs.snapshot.delete",
+        "plugins/zfs_/snapshot.py",
+        why="delete_snapshot_tree() sweeps the recursive snapshot. Without it every "
+            "run orphans one snapshot per descendant dataset (250 on a real pool)",
+    ),
+    MiddlewareCall(
+        "call-zfs-snapshot-query", NESTED, "zfs.snapshot.query",
+        "plugins/zfs_/snapshot.py",
+        why="delete_snapshot_tree()'s fallback sweep enumerates the tree by name",
+    ),
+]
+
+
+def check_call(c: MiddlewareCall, src: str | None) -> tuple[str, str | None]:
+    """Is `c.method` still registered by middlewared?"""
+    if src is None:
+        return "broken", (
+            f"{c.path} no longer exists, so `{c.method}` is gone"
+        )
+
+    try:
+        tree = ast.parse(_stock(src))
+    except SyntaxError as e:
+        return "unknown", f"{c.path} does not parse: {e}"
+
+    # namespace = 'zfs.snapshot' on some Service class in this file...
+    namespaces = {
+        n.value.value
+        for n in ast.walk(tree)
+        if isinstance(n, ast.Assign)
+        and isinstance(n.value, ast.Constant)
+        and isinstance(n.value.value, str)
+        and any(isinstance(t, ast.Name) and t.id == "namespace" for t in n.targets)
+    }
+    if c.namespace not in namespaces:
+        return "broken", (
+            f"{c.path} no longer declares namespace {c.namespace!r} "
+            f"(found: {sorted(namespaces) or 'none'}), so `{c.method}` is gone"
+        )
+
+    # ...and it defines the method.
+    #
+    # A CRUDService exposes `create`/`update`/`delete` from methods NAMED
+    # `do_create`/`do_update`/`do_delete`. Both spellings are live right now:
+    # 24.10 and 25.04 declare `do_delete`, 25.10 renamed it to `delete`, and all
+    # three answer to `zfs.snapshot.delete`. Accepting only the literal name reported
+    # the two older releases as broken -- a false BROKEN that would have switched off
+    # nested snapshots on boxes where they work.
+    defined = {
+        n.name for n in ast.walk(tree)
+        if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef)
+    }
+    if c.name not in defined and f"do_{c.name}" not in defined:
+        return "broken", f"{c.path} no longer defines `{c.method}`"
+
+    return "ok", None
+
+
+#: Things that mean iX has done the job themselves and the module should RETIRE,
+#: not break. Absence of the nesting guard = nested snapshots went native.
+#: `restic = True` already on B2RcloneRemote = B2 restic support went native.
+NATIVE_PROBES = {
+    NESTED: (
+        "plugins/cloud/crud.py",
+        "no further nesting",
+        False,  # native when the phrase is ABSENT
+    ),
+    PROVIDERS: (
+        "rclone/remote/b2.py",
+        "restic = True",
+        True,   # native when the phrase is PRESENT
+    ),
+}
+
+
+def _squash(text: str) -> str:
+    """Drop whitespace and quotes, so a phrase split across string literals matches.
+
+    Stock middleware writes the guard as an implicitly-concatenated literal:
+
+        verrors.add(f"{name}.snapshot", "This option is only available for "
+                                        "datasets that have no further nesting")
+
+    A naive `"no further nesting" in source` is therefore FALSE on a version that
+    very much has the guard -- and this probe's False means "TrueNAS supports it
+    natively, retire the module". That is a silent, catastrophic misread: it would
+    disable nested snapshots on every box that currently depends on them.
+
+    apply.sh already learned this the hard way and normalises the same way. Both
+    now call this one function, which is the only reason they cannot drift apart
+    again.
+    """
+    return text.translate(str.maketrans("", "", " \t\n\r\"'"))
+
+
+# ── AST lookups ──────────────────────────────────────────────────────────────
+
+_DEFS = (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)
+
+
+def _defs_in(body):
+    """Definitions in `body`, descending into if/try/else/with.
+
+    A module-level `def` is not always at module level:
+
+        try:
+            from .fast import create_snapshot
+        except ImportError:
+            async def create_snapshot(...): ...
+
+    Scanning only `tree.body` would say "no longer defines create_snapshot" -- a
+    false BROKEN. And a false BROKEN is not a harmless over-caution here: it makes a
+    module decline to apply on a box where it works perfectly.
+    """
+    for node in body:
+        if isinstance(node, _DEFS):
+            yield node
+        elif isinstance(node, ast.If | ast.Try | ast.With | ast.AsyncWith):
+            yield from _defs_in(node.body)
+            yield from _defs_in(getattr(node, "orelse", []))
+            yield from _defs_in(getattr(node, "finalbody", []))
+            for h in getattr(node, "handlers", []):
+                yield from _defs_in(h.body)
+
+
+def _imports(tree, name):
+    """True if `name` is bound by an import -- i.e. re-exported from elsewhere."""
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import | ast.ImportFrom):
+            for alias in node.names:
+                if (alias.asname or alias.name.split(".")[0]) == name:
+                    return True
+    return False
+
+
+def _find(tree, symbol):
+    """The def node for `name` or `Class.method`, or None."""
+    if "." in symbol:
+        cls_name, meth = symbol.split(".", 1)
+        for node in _defs_in(tree.body):
+            if isinstance(node, ast.ClassDef) and node.name == cls_name:
+                for sub in _defs_in(node.body):
+                    if isinstance(sub, ast.FunctionDef | ast.AsyncFunctionDef) \
+                            and sub.name == meth:
+                        return sub
+        return None
+
+    for node in _defs_in(tree.body):
+        if node.name == symbol:
+            return node
+    return None
+
+
+def _positional(node):
+    a = node.args
+    return [p.arg for p in (*a.posonlyargs, *a.args)]
+
+
+def _signature_problem(node, symbol, want, forwards=False):
+    """Why `symbol`'s signature no longer supports how the patch calls it.
+
+    The injected blocks call the original POSITIONALLY and with a fixed arg list:
+
+        await _tc_orig_create_snapshot(middleware, path, name)
+        _tc_orig_get_restic_config(cloud_backup)
+
+    So a name-subset test ("are these names still in there somewhere?") is not
+    enough, and that is what this used to be. It passed a reorder, a keyword-only
+    conversion, and an added required parameter -- each of which is a TypeError or,
+    worse, silently correct-looking with the arguments swapped.
+
+    The realistic one is not hypothetical: on `master`, iX already renamed
+    get_restic_config's parameter and added a second. That function is rebound
+    module-wide by RESTIC_BLOCK, so a wrong wrapper there kills EVERY TrueCloud
+    task -- Storj included, for users who never wanted this patch's features.
+    """
+    have = _positional(node)
+    n = len(want)
+
+    if have[:n] != want:
+        return (
+            f"{symbol}{tuple(have)} — positional parameters changed; the patch "
+            f"calls it as ({', '.join(want)})"
+        )
+
+    # Extra parameters are fine only if they are optional -- the patch will not pass
+    # them -- OR if the wrapper forwards *args/**kwargs, in which case whatever the
+    # caller supplied is handed straight through. A new REQUIRED one that we neither
+    # pass nor forward is a TypeError at the first backup.
+    args = node.args
+    required = len(have) - len(args.defaults)
+    if required > n and not forwards:
+        return (
+            f"{symbol} now requires {', '.join(have[n:required])} — the patch does "
+            f"not pass it"
+        )
+
+    req_kwonly = [
+        k.arg for k, d in zip(args.kwonlyargs, args.kw_defaults, strict=False)
+        if d is None
+    ]
+    if req_kwonly and not forwards:
+        return (
+            f"{symbol} now requires keyword-only {', '.join(req_kwonly)} — the "
+            f"patch does not pass it"
+        )
+
+    return None
+
+
+def check_source(a: Assumption, src: str | None) -> tuple[str, str | None]:
+    """("ok"|"broken"|"unknown", detail).
+
+    "unknown" exists so that "I cannot inspect this" is never reported as "this is
+    broken". Only "broken" makes a module decline to apply, and declining wrongly
+    breaks a box that was working.
+    """
+    if src is None:
+        return "broken", f"{a.path} does not exist"
+
+    try:
+        tree = ast.parse(src)
+    except SyntaxError as e:
+        return "unknown", f"{a.path} does not parse: {e}"
+
+    node = _find(tree, a.symbol)
+    if node is None:
+        root = a.symbol.split(".", 1)[0]
+        if _imports(tree, root):
+            # Re-exported: `from ._impl import get_restic_config`. The name is still
+            # there and the patch's rebinding still works; we simply cannot see the
+            # signature from here. Refusing to apply over a refactor that changed
+            # nothing would be worse than not checking.
+            return "unknown", (
+                f"{a.path} re-exports {root} from another module; "
+                f"cannot verify its signature here"
+            )
+        return "broken", f"{a.path} no longer defines {a.symbol}"
+
+    if a.kind == "class":
+        if not isinstance(node, ast.ClassDef):
+            return "broken", f"{a.symbol} is no longer a class"
+        return "ok", None
+
+    if isinstance(node, ast.ClassDef):
+        return "broken", f"{a.symbol} is a class, expected a function"
+
+    got_async = isinstance(node, ast.AsyncFunctionDef)
+    if a.is_async is not None and got_async != a.is_async:
+        want = "async def" if a.is_async else "def"
+        got = "async def" if got_async else "def"
+        return "broken", (
+            f"{a.symbol} is now `{got}`, the patch requires `{want}` ({a.path})"
+        )
+
+    problem = _signature_problem(node, a.symbol, a.params, a.forwards)
+    return ("broken", problem) if problem else ("ok", None)
+
+
+# ── sources ──────────────────────────────────────────────────────────────────
+
+class Unreadable(Exception):
+    """The source could not be READ. That is not the same as it not existing.
+
+    Folding these together is how a network blip becomes "iX deleted six files",
+    which becomes "both modules are broken", which becomes a bug report, a red
+    support matrix pushed to the README, and -- on a real box -- a module declining
+    to apply. A transient failure must never be able to say anything about
+    middleware.
+    """
+
+
+def _fetch(ref: str, path: str) -> str | None:
+    """Source at `ref`, None if iX genuinely does not have that file (404).
+
+    Raises Unreadable for anything else: rate limits (the matrix makes ~30
+    unauthenticated requests per run and 429 is a real outcome), DNS, timeouts.
+    """
+    url = RAW.format(ref=ref, path=path)
+    try:
+        with urllib.request.urlopen(url, timeout=_TIMEOUT) as r:  # noqa: S310
+            if r.status == 404:
+                return None
+            if r.status != 200:
+                raise Unreadable(f"{url} -> HTTP {r.status}")
+            return r.read().decode("utf-8", "replace")
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return None          # the file really is gone
+        raise Unreadable(f"{url} -> HTTP {e.code}") from e
+    except Unreadable:
+        raise
+    except Exception as e:
+        raise Unreadable(f"{url} -> {e!r}") from e
+
+
+def _read(root: str, path: str) -> str | None:
+    full = os.path.join(root, *path.split("/"))
+    try:
+        with open(full, encoding="utf-8") as fh:
+            return fh.read()
+    except FileNotFoundError:
+        return None                       # genuinely absent
+    except OSError as e:
+        raise Unreadable(f"{full} -> {e!r}") from e   # permissions, I/O, ...
+
+
+def _stock(text: str) -> str:
+    """Only the part of the file that iX wrote.
+
+    Our own blocks are appended after the MARKER, and they quote the very strings
+    the probes look for -- B2_BLOCK literally writes `B2RcloneRemote.restic = True`
+    into b2.py, and CRUD_BLOCK quotes the "no further nesting" message it filters
+    on. Scanning the whole file on an already-patched box therefore finds OUR text
+    and concludes TrueNAS went native, i.e. "retire the module". apply.sh has always
+    cut at the marker for exactly this reason; compat.py did not, so the one command
+    its own docstring recommends for a live box (`--tree /usr/lib/.../middlewared`)
+    reported providers as native on every patched machine.
+    """
+    return text.split("\n# TRUECLOUD_PATCH", 1)[0]
+
+
+def check(loader, modules=None) -> dict:
+    """Check every assumption. `loader(path) -> source|None`, may raise Unreadable.
+
+    Returns {module: {"ok", "native", "unknown", "problems"}}.
+
+    `unknown` means the sources could not be READ -- a rate limit, a timeout, an
+    unreadable tree. It is NOT `ok` and it is emphatically NOT `broken`: nothing may
+    act on a verdict derived from a failed download.
+    """
+    modules = modules or [PROVIDERS, NESTED]
+    cache = {}
+
+    def src(path):
+        if path not in cache:
+            cache[path] = loader(path)
+        return cache[path]
+
+    out = {
+        m: {"ok": True, "native": False, "unknown": False, "problems": []}
+        for m in modules
+    }
+
+    for a in ASSUMPTIONS:
+        if a.module not in out:
+            continue
+        try:
+            text = src(a.path)
+        except Unreadable as e:
+            out[a.module]["unknown"] = True
+            out[a.module]["problems"].append({
+                "id": a.id, "detail": f"could not read {a.path}: {e}", "why": a.why,
+            })
+            continue
+
+        status, detail = check_source(a, text if text is None else _stock(text))
+        if status == "broken":
+            out[a.module]["ok"] = False
+            out[a.module]["problems"].append({
+                "id": a.id, "detail": detail, "why": a.why,
+            })
+        elif status == "unknown":
+            out[a.module]["unknown"] = True
+            out[a.module]["problems"].append({
+                "id": a.id, "detail": detail, "why": a.why,
+            })
+
+    # The methods the injected code CALLS, not just the symbols it wraps.
+    for c in MIDDLEWARE_CALLS:
+        if c.module not in out:
+            continue
+        try:
+            text = src(c.path)
+        except Unreadable as e:
+            out[c.module]["unknown"] = True
+            out[c.module]["problems"].append({
+                "id": c.id, "detail": f"could not read {c.path}: {e}", "why": c.why,
+            })
+            continue
+
+        status, detail = check_call(c, text)
+        if status == "broken":
+            out[c.module]["ok"] = False
+            out[c.module]["problems"].append({
+                "id": c.id, "detail": detail, "why": c.why,
+            })
+        elif status == "unknown":
+            out[c.module]["unknown"] = True
+            out[c.module]["problems"].append({
+                "id": c.id, "detail": detail, "why": c.why,
+            })
+
+    for module, (path, phrase, native_when_present) in NATIVE_PROBES.items():
+        if module not in out:
+            continue
+        try:
+            text = src(path)
+        except Unreadable:
+            out[module]["unknown"] = True
+            continue
+        if text is None:
+            continue
+        present = _squash(phrase) in _squash(_stock(text))
+        out[module]["native"] = (present == native_when_present)
+
+    # `ok` is cleared ONLY by a definite violation, so "unknown" never needs to
+    # repair it -- and must not: a module with one unreadable file AND one proven
+    # broken assumption is broken, not unknown.
+    return out
+
+
+#: The three stock symbols the nested module wraps. TrueNAS <= 25.10 declares them
+#: `async def`; TrueNAS 26 rewrote them synchronous. apply.sh injects the wrapper
+#: that matches, so this is the question it has to answer at every boot.
+NESTED_WRAPPED = [
+    ("plugins/cloud/snapshot.py", "create_snapshot"),
+    ("plugins/cloud/crud.py", "CloudTaskServiceMixin._validate"),
+    ("plugins/cloud_backup/sync.py", "restic_backup"),
+]
+
+
+def async_flavour(loader) -> bool | None:
+    """Is the installed cloud_backup path async? True, False, or None if unclear.
+
+    None means "do not patch": either a symbol is missing, or -- the case worth
+    naming -- the three DISAGREE. A middleware caught half-converted is one this
+    patch has never seen, and guessing a flavour there means injecting an `async def`
+    that a synchronous caller unpacks as a tuple. Declining costs a feature; guessing
+    costs a backup.
+    """
+    flavours = set()
+    for path, symbol in NESTED_WRAPPED:
+        try:
+            src = loader(path)
+        except Unreadable:
+            return None
+        if src is None:
+            return None
+        try:
+            tree = ast.parse(_stock(src))
+        except SyntaxError:
+            return None
+
+        node = _find(tree, symbol)
+        if not isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef):
+            return None
+        flavours.add(isinstance(node, ast.AsyncFunctionDef))
+
+    return flavours.pop() if len(flavours) == 1 else None
+
+
+def async_flavour_tree(root: str) -> bool | None:
+    return async_flavour(lambda p: _read(root, p))
+
+
+def check_ref(ref: str, modules=None) -> dict:
+    return check(lambda p: _fetch(ref, p), modules)
+
+
+def check_tree(root: str, modules=None) -> dict:
+    return check(lambda p: _read(root, p), modules)
+
+
+# ── which TrueNAS versions to check ──────────────────────────────────────────
+
+REPO = "https://github.com/truenas/middleware"
+
+#: TrueCloud Backup -- the restic-based cloud_backup this patch extends -- was
+#: introduced in 24.10. In 24.04 the modules simply do not exist (404), which the
+#: checker would otherwise report as three separate "broken assumptions" for a
+#: feature that was never there.
+OLDEST = (24, 10)
+
+#: BETA < RC < shipped. Without this, "26.0.0-BETA.1" and "26.0.0-BETA.3" both
+#: reduce to (26,0,0) and the matrix silently reports whichever was seen first --
+#: which is how it first showed BETA.1 while BETA.3 was the one to worry about.
+_STAGE = {"BETA": 0, "RC": 1}
+_SHIPPED = 2
+
+
+def _version_of(name: str):
+    """Sortable version of 'release/26.0.0-BETA.3' or 'TS-25.10.4'. None if junk.
+
+    Returns ((major, minor, ...), stage_rank, stage_number).
+    """
+    tail = name.split("/", 1)[1] if "/" in name else name
+    tail = tail.removeprefix("TS-")
+
+    core, _, suffix = tail.partition("-")
+    try:
+        version = tuple(int(p) for p in core.split("."))
+    except ValueError:
+        return None
+    if len(version) < 2:
+        return None
+
+    if not suffix:
+        return version, _SHIPPED, 0
+
+    stage, _, num = suffix.partition(".")
+    rank = _STAGE.get(stage.upper())
+    if rank is None:
+        return None                      # not a release line we understand
+    return version, rank, int(num) if num.isdigit() else 0
+
+
+def _newest_per_line(names):
+    """Newest name on each (major, minor) line."""
+    best = {}
+    for name in names:
+        v = _version_of(name)
+        if not v or v[0][:2] < OLDEST:
+            continue
+        key = v[0][:2]
+        if key not in best or v > best[key][0]:
+            best[key] = (v, name)
+    return [n for _, n in sorted(best.values())]
+
+
+def _ls_remote(remote, what):
+    import subprocess
+
+    out = subprocess.run(
+        ["git", "ls-remote", what, "--refs", remote],
+        capture_output=True, text=True, check=True, timeout=60,
+    ).stdout
+    prefix = "refs/tags/" if what == "--tags" else "refs/heads/"
+    return [
+        line.split(prefix, 1)[1].strip()
+        for line in out.splitlines() if prefix in line
+    ]
+
+
+def discover_refs(remote: str = REPO) -> list[str]:
+    """What to check: every shipped TrueNAS line, everything unreleased, and master.
+
+    Two sources, because they are authoritative for different things:
+
+      * SHIPPED comes from the `TS-*` TAGS. Those are what iX actually released.
+        The `release/*` branches include mistakes -- `release/25.20.2.2` exists and
+        25.20 is not a TrueNAS version -- and a typo branch in the matrix reads as
+        a real supported release that we are silently broken on.
+
+      * UNRELEASED comes from the BRANCHES, because that is where a beta appears
+        first: `release/26.0.0-BETA.3` had no tag yet while it was the newest beta.
+        Catching breakage here, before it ships, is the whole point of this file.
+    """
+    tags = _ls_remote(remote, "--tags")
+    heads = _ls_remote(remote, "--heads")
+
+    shipped = _newest_per_line([
+        t for t in tags if t.startswith("TS-") and "-BETA" not in t and "-RC" not in t
+    ])
+
+    # A prerelease of a line that has ALREADY shipped is history, not a warning:
+    # release/24.10-RC.2 still exists, and the nested module does not apply to it,
+    # but 24.10 shipped long ago and TS-24.10.2.4 is fine. Reporting it would be a
+    # standing red row in the matrix for a version nobody can install.
+    shipped_lines = {_version_of(t)[0][:2] for t in shipped}
+    upcoming = [
+        h for h in _newest_per_line([
+            h for h in heads
+            if h.startswith("release/") and ("-BETA" in h or "-RC" in h)
+        ])
+        if _version_of(h)[0][:2] not in shipped_lines
+    ]
+
+    return [*shipped, *upcoming, "master"]
+
+
+def is_unreleased(ref: str) -> bool:
+    """master and any BETA/RC. Breakage here is early warning, not an outage."""
+    return ref == "master" or "-BETA" in ref or "-RC" in ref
+
+
+def matrix(refs=None, remote: str = REPO) -> list[dict]:
+    """Check every release line. Returns one row per ref."""
+    rows = []
+    for ref in (refs or discover_refs(remote)):
+        result = check(lambda p, r=ref: _fetch(r, p))
+        rows.append({
+            "ref": ref,
+            "unreleased": is_unreleased(ref),
+            "modules": result,
+        })
+    return rows
+
+
+def _verdict(r: dict) -> str:
+    """BROKEN outranks native, which outranks unknown.
+
+    "native" used to win outright, which meant a module that was BOTH broken and
+    apparently-native rendered as good news: green CI, no bug report, and a README
+    row telling users the feature went native while it was in fact broken. A proven
+    violation is the strongest signal here and must never be masked by a weaker one
+    -- and the native probe is only a substring match on iX's source, so it is
+    exactly the weaker one.
+    """
+    if not r["ok"]:
+        return "BROKEN"
+    if r["native"]:
+        return "native"
+    if r["unknown"]:
+        return "unknown"
+    return "ok"
+
+
+def is_broken(r: dict) -> bool:
+    return not r["ok"]
+
+
+#: Versions a human has actually run a backup on, with real data, on real hardware.
+#: This is NOT automatable and must never be inferred: everything else in this file
+#: is static analysis of iX's source, which proves the patch's assumptions hold --
+#: a strictly weaker claim than "a restore worked". Add a row only after doing it.
+HARDWARE_VERIFIED = {
+    "25.10.4": "nested + providers; 252-snapshot recursive backup of /mnt/Tap, 18m",
+}
+
+_LEGEND = """
+| verdict | meaning |
+| --- | --- |
+| **ok** | Every assumption the patch makes about middleware still holds. |
+| **BROKEN** | middleware changed underneath the patch. `apply.sh` **refuses to apply that module** on this version and leaves TrueNAS stock, so backups keep working — without the module's feature. |
+| **native** | TrueNAS does this itself now. The module retires; it is not a failure. |
+
+"ok" means *the patch's assumptions hold*, checked automatically against iX's
+source. It does not mean a human ran a backup on it — that is the
+**Hardware-verified** column, which is filled in by hand and only by doing it.
+"""
+
+
+#: The README's matrix lives between these. CI regenerates it daily, so a table
+#: claiming the patch works on a TrueNAS that iX has since changed cannot survive
+#: for longer than a day -- a stale support matrix is not a stale doc, it is a lie
+#: to somebody deciding whether to trust this with their backups.
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+README = os.path.join(ROOT_DIR, "README.md")
+BEGIN = "<!-- BEGIN COMPAT MATRIX (generated by tools/compat.py --matrix --markdown) -->"
+END = "<!-- END COMPAT MATRIX -->"
+
+
+def update_readme(rows: list[dict], path: str = README) -> bool:
+    """Rewrite the README's matrix block. True if it changed.
+
+    Refuses if ANY row could not be fully checked. The published matrix is what a
+    stranger reads before trusting this with their backups, and CI pushes it
+    automatically -- so a rate limit or a DNS blip must never be able to repaint it.
+    A stale-but-true table beats a fresh-but-invented one.
+    """
+    unknown = [
+        r["ref"] for r in rows
+        if any(m["unknown"] for m in r["modules"].values())
+    ]
+    if unknown:
+        raise Unreadable(
+            "not rewriting the matrix: could not fully check " + ", ".join(unknown)
+        )
+
+    with open(path, encoding="utf-8") as fh:
+        text = fh.read()
+
+    i, j = text.find(BEGIN), text.find(END)
+    if i == -1 or j == -1:
+        raise ValueError(f"{path} has no COMPAT MATRIX markers")
+
+    new = f"{BEGIN}\n{render_markdown(rows).rstrip()}\n{END}"
+    old = text[i:j + len(END)]
+    if old == new:
+        return False
+
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write(text[:i] + new + text[j + len(END):])
+    return True
+
+
+def render_markdown(rows: list[dict]) -> str:
+    """The matrix, for the README."""
+    out = [
+        "| TrueNAS | B2/S3 providers | Nested snapshots | Hardware-verified |",
+        "| --- | --- | --- | --- |",
+    ]
+    for row in rows:
+        m = row["modules"]
+        ref = row["ref"]
+        label = ref.removeprefix("TS-").removeprefix("release/")
+        if row["unreleased"]:
+            label = f"{label} _(unreleased)_"
+
+        cells = []
+        for mod in (PROVIDERS, NESTED):
+            v = _verdict(m[mod])
+            cells.append({
+                "ok": "ok",
+                "BROKEN": "**BROKEN**",
+                "native": "native",
+                "unknown": "unknown",
+            }[v])
+
+        version = ref.removeprefix("TS-")
+        hw = HARDWARE_VERIFIED.get(version)
+        out.append(f"| {label} | {cells[0]} | {cells[1]} | {hw or '—'} |")
+
+    return "\n".join(out) + "\n" + _LEGEND
+
+
+def render_matrix(rows: list[dict]) -> str:
+    """A support table.
+
+    Says "assumptions hold", not "works" -- this is static analysis of iX's source,
+    which is a strictly weaker claim than having run a backup on the hardware. The
+    hardware-verified column lives in COMPATIBILITY.md and is maintained by hand,
+    because nothing else can honestly fill it in.
+    """
+    w = max((len(r["ref"]) for r in rows), default=10)
+    lines = [
+        f"{'TrueNAS'.ljust(w)}  {'providers':<10}  {'nested':<10}",
+        f"{'-' * w}  {'-' * 10}  {'-' * 10}",
+    ]
+    for row in rows:
+        m = row["modules"]
+        lines.append(
+            f"{row['ref'].ljust(w)}  "
+            f"{_verdict(m[PROVIDERS]):<10}  {_verdict(m[NESTED]):<10}"
+        )
+    return "\n".join(lines)
+
+
+# ── reporting ────────────────────────────────────────────────────────────────
+
+def render(label: str, result: dict) -> str:
+    lines = [f"TrueNAS middleware @ {label}", ""]
+    for module, r in sorted(result.items()):
+        if r["native"]:
+            lines.append(
+                f"  [NATIVE]  {module}: TrueNAS appears to support this natively "
+                f"now — the module should be retired, not fixed."
+            )
+        elif r["ok"]:
+            lines.append(f"  [ok]      {module}: all assumptions hold")
+        else:
+            lines.append(f"  [BROKEN]  {module}:")
+            for p in r["problems"]:
+                lines.append(f"              - {p['detail']}")
+                lines.append(f"                why it matters: {p['why']}")
+    return "\n".join(lines)
+
+
+def main(argv):
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    g = ap.add_mutually_exclusive_group(required=True)
+    g.add_argument("--ref", help="a truenas/middleware git ref, e.g. master")
+    g.add_argument("--tree", help="path to an installed middlewared package")
+    g.add_argument("--matrix", action="store_true",
+                   help="check every TrueNAS release line, newest of each")
+    ap.add_argument("--module", action="append", choices=[PROVIDERS, NESTED],
+                    help="check only this module (repeatable)")
+    ap.add_argument("--json", action="store_true")
+    ap.add_argument("--markdown", action="store_true",
+                    help="with --matrix: emit the table as markdown")
+    ap.add_argument("--update-readme", action="store_true",
+                    help="with --matrix: rewrite the README's matrix block in place")
+    args = ap.parse_args(argv[1:])
+
+    if args.matrix:
+        rows = matrix()
+        if args.json:
+            print(json.dumps(rows, indent=2))
+        elif args.markdown:
+            print(render_markdown(rows))
+        elif args.update_readme:
+            changed = update_readme(rows)
+            print("README.md updated" if changed else "README.md already current")
+        else:
+            print(render_matrix(rows))
+        # A broken UNRELEASED line (master, -BETA, -RC) is a warning, not a build
+        # failure -- it is exactly what we want to know early, and it is iX's tree
+        # to change. compat.yml turns it into a bug report. A broken SHIPPED line
+        # is a genuine failure: users are on it right now.
+        shipped_broken = [
+            r["ref"] for r in rows
+            if not r["unreleased"]
+            and any(is_broken(m) for m in r["modules"].values())
+        ]
+        if shipped_broken:
+            print(f"\nBROKEN on shipped releases: {', '.join(shipped_broken)}",
+                  file=sys.stderr)
+            return 1
+        return 0
+
+    label = args.ref or args.tree
+    result = (check_ref(args.ref, args.module) if args.ref
+              else check_tree(args.tree, args.module))
+
+    if args.json:
+        print(json.dumps({"ref": label, "modules": result}, indent=2))
+    else:
+        print(render(label, result))
+
+    # Exit 1 if any module is broken. "Native" is not broken -- it is good news.
+    return 1 if any(is_broken(r) for r in result.values()) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

tools/release_gate.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""The barrier: a stable release must have been a release candidate first.
+
+CHECKS
+------
+release_notes.py checks the *content* of the tree (versions agree, CHANGELOG has a
+non-empty section, nothing stranded under Unreleased). This module checks the
+*provenance* of the commit: was this exact code ever a release candidate, and did
+that candidate pass CI?
+
+WHY
+---
+This repo cut twelve releases in a single day, several of them "fix the thing the
+last release broke". With an update alert live on every user's box, that is not
+iteration, it is nagging -- and it teaches people to ignore the alert that will one
+day carry a real security fix.
+
+The rule that makes the bad path impossible:
+
+    A stable vX.Y.Z tag is only publishable if a vX.Y.Z-rcN tag points at the SAME
+    commit, and that candidate's CI run passed.
+
+Release candidates are invisible to users: update.sh and the alert source both take
+the newest plain vX.Y.Z tag, so an rc is never offered as an update. Debugging
+therefore happens across rc1, rc2, rc3 -- where it costs nobody anything -- instead
+of across v0.5.0, v0.5.1, v0.5.2, where it costs everybody an alert.
+
+The commit must be *identical*, not merely an ancestor. "The rc passed, then I
+pushed one more little fix" is exactly the habit this exists to break.
+
+    python3 tools/release_gate.py v0.6.0        # exit 1 if not promotable
+    python3 tools/release_gate.py v0.6.0 --next-rc   # -> the rc tag to cut next
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+import sys
+
+from release_notes import base_version, is_prerelease, normalise
+
+
+def _git(*args: str, cwd: str | None = None) -> str:
+    return subprocess.run(
+        ["git", *args],
+        cwd=cwd, capture_output=True, text=True, check=True,
+    ).stdout.strip()
+
+
+def rc_tags(version: str, cwd: str | None = None) -> list[str]:
+    """Every rc tag for this version, oldest first (rc2 sorts after rc1).
+
+    The glob is only a prefilter; the anchored regex decides. `v1.0.0-rc*` also
+    matches `v1.0.0-rc1-hotfix`, which _rc_number reads as 0 -- so a tag the
+    numbering logic does not understand could satisfy the barrier while never having
+    been a release candidate.
+    """
+    want = re.escape(normalise(base_version(version)))
+    exact = re.compile(rf"^v{want}-rc\d+$")
+
+    out = _git("tag", "--list", f"v{normalise(base_version(version))}-rc*", cwd=cwd)
+    tags = [t.strip() for t in out.splitlines() if exact.match(t.strip())]
+    return sorted(tags, key=_rc_number)
+
+
+def _rc_number(tag: str) -> int:
+    m = re.search(r"-rc(\d+)$", tag)
+    return int(m.group(1)) if m else 0
+
+
+def next_rc(version: str, cwd: str | None = None) -> str:
+    """The next rc tag to cut: v0.6.0-rc1, then -rc2, ..."""
+    existing = rc_tags(version, cwd=cwd)
+    n = max((_rc_number(t) for t in existing), default=0) + 1
+    return f"v{normalise(base_version(version))}-rc{n}"
+
+
+def commit_for(ref: str, cwd: str | None = None) -> str | None:
+    try:
+        return _git("rev-list", "-n", "1", ref, cwd=cwd)
+    except subprocess.CalledProcessError:
+        return None
+
+
+def check_promotable(version: str, cwd: str | None = None) -> list[str]:
+    """Every reason v<version> may not be cut as a stable release.
+
+    Empty list means the barrier is satisfied.
+
+    The commit under test is the tag's if it exists, and HEAD otherwise. Both are
+    real: CI runs this AFTER the tag is pushed, and release.sh runs it BEFORE
+    creating the tag -- which is the whole point, since refusing after the tag
+    exists is too late to be a gate. Requiring the tag unconditionally made
+    `release.sh --promote` impossible: it dies if the tag already exists, and the
+    gate died if it did not, so the only way through was to hand-tag and bypass
+    every check this file exists to enforce.
+    """
+    if is_prerelease(version):
+        return []  # candidates are what the barrier exists to encourage
+
+    want = normalise(version)
+    tag = f"v{want}"
+
+    target = commit_for(tag, cwd=cwd)
+    if target is None:
+        target = commit_for("HEAD", cwd=cwd)
+    if target is None:
+        return ["cannot resolve a commit to release (no HEAD?)"]
+
+    candidates = rc_tags(want, cwd=cwd)
+    if not candidates:
+        return [
+            f"{tag} was never a release candidate. Cut one first:\n"
+            f"    bash release.sh {want} --rc\n"
+            f"Candidates are invisible to users -- debug there, not in a release."
+        ]
+
+    matching = [c for c in candidates if commit_for(c, cwd=cwd) == target]
+    if not matching:
+        newest = candidates[-1]
+        return [
+            f"{tag} points at {target[:12]}, but no release candidate does.\n"
+            f"    Candidates: {', '.join(candidates)}\n"
+            f"    Newest ({newest}) is at "
+            f"{(commit_for(newest, cwd=cwd) or '?')[:12]}.\n"
+            f"Code changed after the last candidate. That change is untested as a\n"
+            f"release: cut {next_rc(want, cwd=cwd)} and promote THAT commit."
+        ]
+
+    return []
+
+
+def main(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("version", help="e.g. v0.6.0")
+    ap.add_argument("--next-rc", action="store_true",
+                    help="print the next rc tag to cut, and exit")
+    ap.add_argument("-C", dest="cwd", default=None, help="run git in this directory")
+    args = ap.parse_args(argv[1:])
+
+    if args.next_rc:
+        print(next_rc(args.version, cwd=args.cwd))
+        return 0
+
+    problems = check_promotable(args.version, cwd=args.cwd)
+    for p in problems:
+        print(f"::error::{p}")
+    if problems:
+        return 1
+
+    print(f"v{normalise(args.version)} was a release candidate and may be promoted")
+    return 0
+
+
+if __name__ == "__main__":
+    # Running `python3 tools/release_gate.py` already puts tools/ on sys.path[0],
+    # which is what makes the `release_notes` import above resolve.
+    sys.exit(main(sys.argv))

tools/release_notes.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""Extract one version's section from CHANGELOG.md, and check version consistency.
+
+Used by .github/workflows/release.yml so a release's body is always the changelog
+entry -- there is no second place to write release notes, and therefore no second
+place for them to be wrong.
+
+    python3 tools/release_notes.py notes v0.3.0     # -> the section body
+    python3 tools/release_notes.py version          # -> version per the scripts
+    python3 tools/release_notes.py check v0.3.0     # -> exit 1 on any mismatch
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+CHANGELOG = os.path.join(ROOT, "CHANGELOG.md")
+
+# Everything that announces a version must agree with everything else. They drifted
+# to three different values once (0.0.4 / 0.2.1) before anything checked them --
+# and create_task.py's __version__ then sat at 0.2.0 through three more releases,
+# because the first version of this check only looked at VERSION= in shell scripts.
+VERSIONED_FILES = [
+    "install.sh",
+    "uninstall.sh",
+    "recover.sh",
+    os.path.join("patch", "apply.sh"),
+    os.path.join("patch", "create_task.py"),   # exposes `--version` to users
+    "update.sh",
+]
+
+# `VERSION="x"` (shell) or `__version__ = "x"` (python).
+_VERSION_RE = re.compile(r'^(?:VERSION=|__version__\s*=\s*)"([^"]+)"', re.M)
+_HEADING_RE = re.compile(r"^##\s+v?(\d+\.\d+\.\d+[^\s]*)", re.M)
+
+
+#: Work in progress lives here until a release promotes it. Batching through this
+#: section is what stops "tag, find bug, tag again" from becoming twelve releases.
+UNRELEASED = "Unreleased"
+
+_UNRELEASED_RE = re.compile(r"^##\s+Unreleased\s*$", re.M | re.I)
+_RC_RE = re.compile(r"-(rc|beta|alpha)\d*$", re.I)
+
+
+def normalise(v: str) -> str:
+    return v.strip().lstrip("v")
+
+
+def is_prerelease(tag: str) -> bool:
+    """True for v1.2.3-rc1 / -beta / -alpha. Those never reach users."""
+    return bool(_RC_RE.search(tag.strip()))
+
+
+def base_version(tag: str) -> str:
+    """v1.2.3-rc2 -> 1.2.3"""
+    return _RC_RE.sub("", normalise(tag))
+
+
+def unreleased_body(text: str) -> str:
+    """Content under `## Unreleased`, or "" if the section is absent/empty."""
+    m = _UNRELEASED_RE.search(text)
+    if not m:
+        return ""
+    rest = text[m.end():]
+    nxt = _HEADING_RE.search(rest)
+    return (rest[:nxt.start()] if nxt else rest).strip()
+
+
+def promote(text: str, version: str, date: str) -> str:
+    """Rename `## Unreleased` to `## vX.Y.Z — date`.
+
+    Refuses if the section is missing or empty: a release with nothing in it is a
+    release nobody needed, and cutting one only trains people to ignore alerts.
+    """
+    if not unreleased_body(text):
+        raise ValueError(
+            "CHANGELOG.md has no `## Unreleased` content — nothing to release. "
+            "Add your changes there first."
+        )
+    m = _UNRELEASED_RE.search(text)
+    return text[:m.start()] + f"## v{normalise(version)} — {date}" + text[m.end():]
+
+
+def script_versions(root: str = ROOT) -> dict[str, str]:
+    """VERSION= as declared by each script."""
+    found = {}
+    for rel in VERSIONED_FILES:
+        path = os.path.join(root, rel)
+        try:
+            with open(path, encoding="utf-8") as fh:
+                m = _VERSION_RE.search(fh.read())
+        except OSError:
+            continue
+        if m:
+            found[rel] = m.group(1)
+    return found
+
+
+def changelog_versions(text: str) -> list[str]:
+    """Versions with a section in the changelog, newest first."""
+    return [normalise(v) for v in _HEADING_RE.findall(text)]
+
+
+def extract_notes(text: str, version: str) -> str:
+    """The body of one version's section, without its heading.
+
+    A release candidate resolves to its BASE version: v0.6.0-rc2 ships the same code
+    as v0.6.0 and therefore the same notes, and the CHANGELOG only ever has the one
+    section. Without this, the release workflow cut the tag, passed every gate, and
+    then died extracting the body -- so the candidate existed but was never published.
+
+    Raises KeyError if the version has no section -- a release with an empty or
+    wrong body is worse than a failed release.
+    """
+    want = base_version(version)
+    lines = text.splitlines()
+
+    start = None
+    for i, line in enumerate(lines):
+        m = _HEADING_RE.match(line)
+        if m and normalise(m.group(1)) == want:
+            start = i + 1
+            break
+    if start is None:
+        raise KeyError(f"CHANGELOG.md has no section for v{want}")
+
+    end = len(lines)
+    for i in range(start, len(lines)):
+        if _HEADING_RE.match(lines[i]):
+            end = i
+            break
+
+    return "\n".join(lines[start:end]).strip()
+
+
+# ── significance ──────────────────────────────────────────────────────────────
+# Used by the TrueNAS update alert to decide whether a release is worth bothering
+# anyone about. The CHANGELOG's own section headings are the signal: a release that
+# only has "### Docs" changed no code, and nobody should get an alert for a README.
+
+_SECTION_RE = re.compile(r"^###\s+(.+?)\s*$", re.M)
+
+#: Headings that mean "nothing about the running system changed".
+QUIET_SECTIONS = {"docs", "documentation"}
+
+
+def version_tuple(v: str) -> tuple:
+    """Sortable version. Pre-release suffixes are dropped, not ranked."""
+    return tuple(int(x) for x in normalise(v).split("-")[0].split("."))
+
+
+def section_headings(body: str) -> list[str]:
+    """The `### ...` headings inside one version's body, lowercased."""
+    return [h.strip().lower() for h in _SECTION_RE.findall(body)]
+
+
+def significance(text: str, current: str, latest: str):
+    """How much does upgrading `current` -> `latest` actually matter?
+
+    Returns ``(level, versions, headings)`` where level is one of:
+
+      "security"  a release in the range has a Security section  -> alert loudly
+      "notable"   something about the system changed             -> alert quietly
+      "docs"      only documentation changed                     -> DO NOT alert
+
+    Considers every release in the range, not just the newest: a docs-only v0.4.2
+    on top of a security-fixing v0.4.1 must still be reported as security.
+    """
+    cur, lat = version_tuple(current), version_tuple(latest)
+
+    versions = [
+        v for v in changelog_versions(text)
+        if cur < version_tuple(v) <= lat
+    ]
+
+    headings = []
+    for v in versions:
+        try:
+            headings.extend(section_headings(extract_notes(text, v)))
+        except KeyError:
+            continue
+
+    if any(h.startswith("security") for h in headings):
+        return "security", versions, headings
+    if [h for h in headings if h not in QUIET_SECTIONS]:
+        return "notable", versions, headings
+    return "docs", versions, headings
+
+
+def check(version: str, root: str = ROOT) -> list[str]:
+    """Every reason this version is not releasable. Empty list means it is.
+
+    `version` may be a release candidate (v1.2.3-rc2); the scripts and CHANGELOG
+    are checked against its BASE version, since an rc ships the same code.
+    """
+    want = base_version(version)
+    problems = []
+
+    versions = script_versions(root)
+    for rel, got in sorted(versions.items()):
+        if normalise(got) != want:
+            problems.append(f"{rel} declares VERSION={got!r}, tag is v{want}")
+    missing = [r for r in VERSIONED_FILES if r not in versions]
+    for rel in missing:
+        problems.append(f"{rel} has no VERSION= line")
+
+    try:
+        with open(os.path.join(root, "CHANGELOG.md"), encoding="utf-8") as fh:
+            text = fh.read()
+    except OSError as e:
+        problems.append(f"cannot read CHANGELOG.md: {e}")
+        return problems
+
+    try:
+        body = extract_notes(text, want)
+    except KeyError as e:
+        problems.append(str(e))
+    else:
+        if not body:
+            problems.append(f"CHANGELOG.md section for v{want} is empty")
+
+    # A stable release must not leave work stranded under `## Unreleased`. If it is
+    # finished enough to ship, it belongs in the release; if it is not, the release
+    # is premature. (An rc may legitimately have more work queued behind it.)
+    if not is_prerelease(version) and unreleased_body(text):
+        problems.append(
+            "CHANGELOG.md still has content under `## Unreleased` — either include "
+            "it in this release, or do not cut the release yet"
+        )
+
+    return problems
+
+
+def main(argv):
+    if len(argv) < 2:
+        print(__doc__, file=sys.stderr)
+        return 2
+
+    cmd = argv[1]
+
+    if cmd == "version":
+        versions = set(map(normalise, script_versions().values()))
+        if len(versions) != 1:
+            print(f"scripts disagree on version: {sorted(versions)}", file=sys.stderr)
+            return 1
+        print(versions.pop())
+        return 0
+
+    if len(argv) < 3:
+        print(f"usage: {argv[0]} {cmd} <version>", file=sys.stderr)
+        return 2
+    version = argv[2]
+
+    if cmd == "notes":
+        # An explicit path lets update.sh show the notes from the CHANGELOG of the
+        # version it is about to install (`git show <tag>:CHANGELOG.md`), not the
+        # one already checked out.
+        path = argv[3] if len(argv) > 3 else CHANGELOG
+        with open(path, encoding="utf-8") as fh:
+            print(extract_notes(fh.read(), version))
+        return 0
+
+    if cmd == "check":
+        problems = check(version)
+        for p in problems:
+            print(f"::error::{p}")
+        if problems:
+            return 1
+        print(f"v{normalise(version)} is consistent across scripts and CHANGELOG")
+        return 0
+
+    print(f"unknown command: {cmd}", file=sys.stderr)
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

uninstall.sh

@@ -3,7 +3,7 @@
 
 set -euo pipefail
 
-VERSION="0.0.4"
+VERSION="0.6.0"
 
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 _HOOK_COMMENT='TrueCloud provider patch (S3/B2)'
@@ -91,6 +91,41 @@ if [ "$_ov_found" -eq 0 ]; then
 fi
 echo ""
 
+# ── Revert file-level patches ─────────────────────────────────────────────────
+# Unmounting the overlay is what normally reverts everything — the lower layer is
+# the untouched /usr. But apply.sh only mounts an overlay when the directory is
+# read-only; on a writable /usr it patches the real files in place. Uninstall
+# would then remove the boot hook and report success while leaving every patch
+# applied. Strip our appended blocks explicitly.
+
+# Same implementation apply.sh uses (patch/mw_patch.py) — a second shell copy of
+# this would be the untested one.
+echo "Reverting any file-level patches ..."
+python3 "$PATCH_DIR/patch/mw_patch.py" revert-all || \
+    echo "  WARNING: could not revert file-level patches."
+echo ""
+
+# ── Unmount nested-snapshot staging trees ─────────────────────────────────────
+# These bind mounts pin their ZFS snapshots, so they must go before anything
+# tries to destroy those snapshots. Deepest first.
+
+# Delegated to the patch module rather than reimplemented here: the depth
+# ordering and lazy-umount fallback are fiddly, and a shell copy would be the
+# untested one.
+echo "Unmounting nested-snapshot staging trees (if any) ..."
+if ! python3 "$PATCH_DIR/patch/truecloud_nested.py" cleanup; then
+    echo "  WARNING: staging mounts remain. Unmount them manually; until you do,"
+    echo "           the ZFS snapshots they pin cannot be destroyed."
+fi
+
+# The opt-in marker lives in the repo dir; remove it so a later re-install
+# starts from the safe default (feature off).
+if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]; then
+    rm -f "$PATCH_DIR/nested_snapshots_enabled"
+    echo "  Removed nested-snapshot opt-in marker."
+fi
+echo ""
+
 if [ "$_restore_failed" -eq 1 ]; then
     echo ""
     echo "ERROR: One or more UI bundle backups could not be restored." >&2

update.sh

@@ -0,0 +1,293 @@
+#!/bin/bash
+# update.sh — fetch a newer release of truecloud-patch and apply it.
+#
+# ── RUN THIS BY HAND. NEVER FROM CRON OR A SYSTEMD TIMER. ─────────────────────
+#
+# This patch injects Python into middlewared and re-applies itself at every boot.
+# An unattended pull would let any bad upstream commit reach your box with no
+# human in the loop, and take effect on the next reboot. That is not theoretical:
+# v0.0.4 shipped a boot-time bug that took every app on the box down.
+#
+# The manual step IS the safety gate. Keep it.
+#
+# By default this updates to the newest RELEASE TAG, not to main. main can be
+# mid-refactor; a tag is the tested artifact. Use --main only if you know why.
+#
+#   bash update.sh              # to the newest release, with a confirmation
+#   bash update.sh --check      # show what would happen; change nothing
+#   bash update.sh --rollback   # undo the last update
+
+set -euo pipefail
+
+VERSION="0.6.0"
+
+PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
+_PREV_FILE="$PATCH_DIR/.update_previous"
+
+_target=""
+_use_main=0
+_assume_yes=0
+_check_only=0
+_rollback=0
+
+usage() {
+    cat <<USAGE
+Usage: bash update.sh [options]
+
+Options:
+  --to <ref>    Update to a specific tag or commit (default: newest release tag)
+  --main        Update to origin/main — UNRELEASED code, no guarantees
+  --check       Show what an update would do and exit; changes nothing
+  --rollback    Return to the revision recorded before the last update
+  --yes, -y     Skip the confirmation prompt
+  -h, --help    Show this help
+
+Updating preserves your nested-snapshot opt-in setting either way.
+USAGE
+}
+
+# An UNTRACKED file that the target tracks makes `git checkout` abort. The dirty-
+# tree check deliberately ignores untracked files, so this slips past it and the
+# checkout then dies mid-operation. Not hypothetical: a hand-copied
+# patch/wait_restart.sh blocked a pull on a real box exactly this way.
+#
+# Used by BOTH the update and the rollback path -- rolling back moves the tree too,
+# and would hit the identical failure.
+_abort_if_untracked_blockers() {
+    local ref="$1" blocking
+
+    # Set intersection of {untracked, not ignored} and {tracked by the target}. Two
+    # git calls, not one `ls-files --error-unmatch` per file in the target tree.
+    # --exclude-standard is deliberate: git silently overwrites *ignored* files on
+    # checkout, so those are not blockers — only untracked-and-not-ignored ones are.
+    blocking="$(comm -12 \
+        <(git ls-files --others --exclude-standard | sort) \
+        <(git ls-tree -r --name-only "$ref" | sort) \
+        | sed 's/^/    /')"
+
+    [ -n "$blocking" ] || return 0
+
+    echo "ERROR: these untracked files would be overwritten:" >&2
+    printf '%s\n\n' "$blocking" >&2
+    echo "  They exist here but git does not track them — most likely hand-copied" >&2
+    echo "  or scp'd in. Move or delete them, then re-run." >&2
+
+    # "Delete update.sh, then re-run update.sh" is impossible. If the script itself
+    # is a blocker, it was hand-copied in to bootstrap; the honest answer is to
+    # bootstrap with git instead, which installs it properly.
+    case "$blocking" in
+        *update.sh*)
+            echo "" >&2
+            echo "  update.sh itself is untracked here — you copied it in to bootstrap." >&2
+            echo "  Do that with git instead, once; it installs update.sh properly:" >&2
+            echo "" >&2
+            echo "    rm -f $PATCH_DIR/update.sh" >&2
+            echo "    git -C $PATCH_DIR checkout $ref" >&2
+            echo "    bash $PATCH_DIR/install.sh" >&2
+            echo "" >&2
+            echo "  Every later update is then just: bash update.sh" >&2
+            ;;
+    esac
+    exit 1
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --to)
+            if [ -z "${2:-}" ]; then
+                echo "ERROR: --to needs a tag, branch, or commit." >&2
+                exit 1
+            fi
+            _target="$2"; shift ;;
+        --main)     _use_main=1 ;;
+        --check)    _check_only=1 ;;
+        --rollback) _rollback=1 ;;
+        --yes|-y)   _assume_yes=1 ;;
+        -h|--help)  usage; exit 0 ;;
+        *) echo "ERROR: unknown option: $1" >&2; echo "" >&2; usage >&2; exit 1 ;;
+    esac
+    shift
+done
+
+echo "=== TrueNAS TrueCloud Provider Patch — Update (v${VERSION}) ==="
+echo ""
+
+# ── Preflight ─────────────────────────────────────────────────────────────────
+
+if [ "$(id -u)" -ne 0 ]; then
+    echo "ERROR: must be run as root (install.sh needs it)." >&2
+    exit 1
+fi
+
+cd "$PATCH_DIR"
+
+if ! git rev-parse --git-dir >/dev/null 2>&1; then
+    echo "ERROR: $PATCH_DIR is not a git clone — nothing to update." >&2
+    echo "  Re-clone from https://github.com/sudolulo/truenas-truecloud-patch" >&2
+    exit 1
+fi
+
+# Past `sudo git pull`s can leave root-owned objects in .git that then break any
+# non-root git command. We run as root, so we would only make that worse.
+_owner="$(stat -c '%U' "$PATCH_DIR")"
+if [ -n "$_owner" ] && [ "$_owner" != "root" ]; then
+    chown -R "$_owner" "$PATCH_DIR/.git" 2>/dev/null || true
+fi
+
+# A dirty tree means someone edited or scp'd files in place; merging over that
+# silently loses their changes, or conflicts halfway through.
+if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+    echo "ERROR: the working tree has uncommitted changes:" >&2
+    git status --short --untracked-files=no >&2
+    echo "" >&2
+    echo "  Refusing to update over them. Commit, stash, or discard them first:" >&2
+    echo "    git -C $PATCH_DIR checkout -- ." >&2
+    exit 1
+fi
+
+# ── Rollback ──────────────────────────────────────────────────────────────────
+
+if [ "$_rollback" -eq 1 ]; then
+    if [ ! -f "$_PREV_FILE" ]; then
+        echo "ERROR: no previous revision recorded — nothing to roll back to." >&2
+        exit 1
+    fi
+    _prev="$(cat "$_PREV_FILE")"
+    if ! git rev-parse --verify --quiet "${_prev}^{commit}" >/dev/null; then
+        echo "ERROR: recorded revision '$_prev' is not a valid commit." >&2
+        echo "  The history may have been rewritten. Pick a target explicitly:" >&2
+        echo "    bash update.sh --to <tag>" >&2
+        exit 1
+    fi
+    _abort_if_untracked_blockers "$_prev"
+    echo "Rolling back to $_prev ..."
+    git checkout -q --detach "$_prev"
+    echo "Reverted. Re-applying ..."
+    echo ""
+    bash "$PATCH_DIR/install.sh"
+    exit 0
+fi
+
+# ── Work out where we are and where we are going ──────────────────────────────
+
+echo "Fetching ..."
+git fetch --quiet --tags --prune origin
+
+_current="$(git rev-parse HEAD)"
+_current_desc="$(git describe --tags --always 2>/dev/null || echo "$_current")"
+
+if [ -n "$_target" ]; then
+    :
+elif [ "$_use_main" -eq 1 ]; then
+    _target="origin/main"
+else
+    # Newest release tag by VERSION order, not by tag date. Date order is only
+    # correct while tags are created in ascending version order; it breaks the
+    # moment a hotfix is tagged out of band (a v0.3.6 released after v0.4.0 would
+    # sort as "newest" by date and silently downgrade the box).
+    #
+    # Filter to PLAIN vX.Y.Z: git's version sort ranks `v0.5.0-rc1` ABOVE `v0.5.0`
+    # (verified), so without this a release candidate would be installed as though
+    # it were the newest release. The release workflow deliberately supports
+    # rc/beta/alpha tags, so they will exist.
+    _target="$(git tag -l 'v*' --sort=-version:refname \
+                 | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1)"
+    if [ -z "$_target" ]; then
+        echo "ERROR: no release tags found; use --main to track unreleased code." >&2
+        exit 1
+    fi
+fi
+
+if ! _target_sha="$(git rev-parse --verify --quiet "${_target}^{commit}")"; then
+    echo "ERROR: '$_target' is not a valid tag, branch, or commit." >&2
+    exit 1
+fi
+
+echo "  current: $_current_desc"
+echo "  target:  $_target ($(git rev-parse --short "$_target_sha"))"
+echo ""
+
+if [ "$_current" = "$_target_sha" ]; then
+    echo "Already up to date. Nothing to do."
+    exit 0
+fi
+
+_abort_if_untracked_blockers "$_target_sha"
+
+# ── Show what is coming ───────────────────────────────────────────────────────
+
+echo "Commits you do not have yet:"
+git log --oneline --no-decorate "$_current..$_target_sha" | sed 's/^/  /' || true
+echo ""
+
+# Reuse tools/release_notes.py rather than re-implementing the extractor here —
+# a second copy would be the untested one. Read the CHANGELOG *of the target*, so
+# the notes describe what you are about to install.
+if [ -f "$PATCH_DIR/tools/release_notes.py" ] && [ "$_use_main" -eq 0 ] \
+       && [ -z "${_target##v*}" ]; then
+    _cl="$(mktemp)"
+    if git show "$_target_sha:CHANGELOG.md" > "$_cl" 2>/dev/null && [ -s "$_cl" ]; then
+        echo "Release notes for $_target:"
+        python3 "$PATCH_DIR/tools/release_notes.py" notes "$_target" "$_cl" \
+            2>/dev/null | sed 's/^/  /' || echo "  (no notes for $_target)"
+        echo ""
+    fi
+    rm -f "$_cl"
+fi
+
+if [ "$_use_main" -eq 1 ]; then
+    echo "NOTE: --main tracks UNRELEASED code. It has passed CI, but it is not a"
+    echo "      tested release, and apply.sh runs at every boot."
+    echo ""
+fi
+
+if [ "$_check_only" -eq 1 ]; then
+    echo "--check given; nothing changed."
+    exit 0
+fi
+
+# ── Confirm ───────────────────────────────────────────────────────────────────
+
+if [ "$_assume_yes" -eq 0 ]; then
+    printf "Apply this update and restart middlewared? [y/N] "
+    read -r _answer </dev/tty || _answer=""
+    case "$_answer" in
+        y|Y|yes|YES) ;;
+        *) echo "Aborted. Nothing changed."; exit 0 ;;
+    esac
+    echo ""
+fi
+
+# ── Apply ─────────────────────────────────────────────────────────────────────
+
+# Record where we were BEFORE moving, so --rollback works even if install.sh dies.
+echo "$_current" > "$_PREV_FILE"
+
+echo "Checking out $_target ..."
+git checkout -q --detach "$_target_sha"
+echo "  now at $(git describe --tags --always)"
+echo ""
+
+echo "Applying (this preserves your nested-snapshot setting) ..."
+echo ""
+if ! bash "$PATCH_DIR/install.sh"; then
+    echo ""
+    echo "ERROR: install.sh failed after updating." >&2
+    echo "  Roll back with:  bash $PATCH_DIR/update.sh --rollback" >&2
+    echo "  Or disable the patch entirely:  bash $PATCH_DIR/recover.sh" >&2
+    exit 1
+fi
+
+echo ""
+echo "=== Update complete ==="
+echo "  $_current_desc  ->  $(git describe --tags --always)"
+echo ""
+if ! git symbolic-ref -q HEAD >/dev/null; then
+    echo "NOTE: the checkout is now pinned to a release tag (detached HEAD), which is"
+    echo "      what you want for a deployment. Plain \`git pull\` will not work here —"
+    echo "      use \`bash update.sh\` from now on."
+    echo ""
+fi
+echo "If anything looks wrong:"
+echo "  bash $PATCH_DIR/update.sh --rollback    # back to $_current_desc"
+echo "  bash $PATCH_DIR/recover.sh              # kill switch + restart"