docs: refresh the TrueNAS compatibility matrix

Run CI as a single job so concurrent jobs cannot race
Two CI failures shared one cause: four jobs starting together on the self-hosted runner. act caches each action as one shared clone under /root/.cache/act/<hash> and re-pulls it per job, so concurrent jobs fight over that directory and the loser dies with "lstat .../<file>: no such file or directory" before any test runs — a different victim each push. And the runner force-pulls its base image per job, so four jobs meant four anonymous Docker Hub pulls per push; that hit 429 Too Many Requests and every job started failing before it began, including the shell job nothing had touched. Installing uv without an action only shrank the surface, since every job still used actions/checkout. Concurrency is the ingredient, so this removes it: one job cannot race itself whatever actions it uses, and one job is one pull. The version sweep moves inside the job and still runs every version after one fails, preserving what fail-fast: false bought.
2026-08-27 06:17:44 +00:00 · 2026-08-26 05:34:11 +00:00 · 2026-08-26 05:28:26 +00:00 · 2026-08-26 05:20:23 +00:00 · 2026-08-26 04:57:33 +00:00 · 2026-08-26 04:37:50 +00:00
30 changed files with 4676 additions and 442 deletions
@@ -0,0 +1,2 @@
 github: sudolulo
 ko_fi: sudolulo
@@ -9,9 +9,31 @@ on:
 permissions:
  contents: read
 # ONE job, deliberately. This was four (shell + a 3-way python matrix) and they
 # started within the same second on the self-hosted Gitea runner, which is what
 # made CI unreliable in two separate ways:
 #
 #   1. The act action-cache race. `act` caches each ACTION as a single shared git
 #      clone under /root/.cache/act/<hash> and re-pulls it per job, so concurrent
 #      jobs using the same action fight over that directory and the loser dies
 #      with `lstat /root/.cache/act/<hash>/<file>: no such file or directory` --
 #      a red `main` with zero suite output, and a different victim each push
 #      (3.12 on one, 3.11 on the next). Dropping one action only shrank the
 #      surface: every job still used actions/checkout. Concurrency is the actual
 #      ingredient, so removing it removes the whole class -- a single job cannot
 #      race itself, no matter which actions it uses.
 #
 #   2. Docker Hub 429s. The runner force-pulls its base image per job, so four
 #      jobs meant four anonymous pulls per push. A few pushes and re-runs in an
 #      afternoon exhausted the anonymous limit and every job failed before it
 #      started -- including the shell job, which nothing had touched. One job is
 #      one pull.
 #
 # The cost is wall-clock parallelism, and this repo does not need it: the suite
 # is ~1.5s, so container start and interpreter downloads dominate either way.
 jobs:
-  shell:
+  ci:
-    name: shell (shellcheck + syntax)
+    name: ci (shell + python 3.11-3.13)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
@@ -31,32 +53,57 @@ jobs:
        env:
          SHELLCHECK_OPTS: -S warning -e SC1091
-  python:
+      # uv-managed interpreters instead of actions/setup-python: the prebuilt-CPython
-    name: python ${{ matrix.python }}
+      # download path setup-python relies on does not work on the self-hosted Gitea
-    runs-on: ubuntu-latest
+      # runner (all three matrix jobs failed at setup there while passing on GitHub);
-    strategy:
+      # uv works identically on both.
-      fail-fast: false
+      #
-      matrix:
+      # Installed by a plain `run:` step rather than astral-sh/setup-uv: one fewer
-        # TrueNAS SCALE middleware runs 3.11+; keep the patch importable across
+      # action is one fewer thing to go wrong, and the action was only ever
-        # the versions it may be injected into.
+      # fetching a binary -- the interpreter is chosen per command by `uvx
-        python: ["3.11", "3.12", "3.13"]
+      # --python`, never by the action.
-    steps:
+      #
-      - uses: actions/checkout@v4
+      # Pinned for the same reason ruff is pinned below: an unpinned uv means any
-
+      # upstream release can turn main red with no code change here.
-      - uses: actions/setup-python@v5
+      - name: install uv
-        with:
+        env:
-          python-version: ${{ matrix.python }}
+          UV_VERSION: "0.11.21"
-
+        run: |
-      - name: install dev deps
+          curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | sh
-        run: python -m pip install --upgrade pip pytest ruff
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
      - name: ruff
-        run: ruff check patch tests tools
+        # Pinned: an unpinned ruff means any upstream release can turn main red
        # with no code change.
        run: uvx ruff@0.16.1 check patch tests tools
      # TrueNAS SCALE middleware runs 3.11+; keep the patch importable across the
      # versions it may be injected into. Every version runs even after one
      # fails -- that is what `fail-fast: false` bought when this was a matrix,
      # and losing it would mean a 3.11 break hides whether 3.12 and 3.13 are
      # fine, which is exactly the information you want at that moment.
      - name: pytest
-        run: pytest tests -v
+        env:
          PYTHONS: "3.11 3.12 3.13"
        run: |
          fail=0
          for v in $PYTHONS; do
            echo "::group::pytest on python $v"
            uvx --python "$v" pytest tests -v \
              || { echo "::error::suite failed on python $v"; fail=1; }
            echo "::endgroup::"
          done
          exit $fail
      - name: verify injected middleware blocks compile
        # Belt-and-braces: the *_BLOCK strings are appended into live middlewared
        # modules. A syntax error there would break the box at boot.
-        run: pytest tests/test_apply_blocks.py -v
+        env:
          PYTHONS: "3.11 3.12 3.13"
        run: |
          fail=0
          for v in $PYTHONS; do
            uvx --python "$v" pytest tests/test_apply_blocks.py -v \
              || { echo "::error::injected blocks failed to compile on python $v"; fail=1; }
          done
          exit $fail
@@ -24,10 +24,18 @@ on:
    paths:
      # The manifest itself changed -- re-check immediately rather than waiting a day.
      - "tools/compat.py"
      # ...and so did the thing that PUBLISHES the finding. This was missing, and it
      # showed: the commit teaching the bot to refresh a stale report body touched only
      # compat_publish.py, so no run fired, and the report stayed stale until the next
      # scheduled one. A fix nobody runs is a fix nobody has.
      - "tools/compat_publish.py"
      - ".github/workflows/compat.yml"
 permissions:
-  contents: read
+  # write, because the matrix refresh pushes a branch and opens a PR. It does NOT get
  # to move `main` -- that is the whole reason it is a PR. See the refresh step below.
  contents: write
  pull-requests: write
  issues: write
 jobs:
@@ -130,17 +138,27 @@ jobs:
      - name: matrix
        run: cat /tmp/matrix.md
-      # Keep the README's table true. A support matrix that quietly goes stale is not
+      # Keep the README's table true — as a PULL REQUEST, on the CANONICAL forge.
      # a stale doc -- it is a false promise to somebody deciding whether to trust
      # this with their backups.
      #
-      # Only ever touches the block between the COMPAT MATRIX markers, and only on
+      # Two things this gets right that the obvious version gets wrong:
-      # the canonical host (Gitea) so the two forges cannot race each other. The
+      #
-      # `paths:` trigger above does not include README.md, so this cannot re-trigger
+      #  1. It is a PR, not a push to main. This used to `git push origin HEAD:main`
-      # itself; and a README change is documentation-only, which by design raises no
+      #     from CI. An unattended write to main is exactly what the release barrier
-      # update alert on anyone's box.
+      #     exists to prevent — a bot that can move main can move it somewhere nobody
-      - name: refresh the README matrix
+      #     looked. Nothing lands by itself.
      #
      #  2. It runs on GITEA, not GitHub. GitHub is a one-way MIRROR: a PR merged there
      #     would be silently clobbered by the next `fleet-repos mirror` push from Gitea.
      #     A bot opening PRs against a mirror is a bot doing nothing, slowly.
      #
      # A stale support matrix is not a stale doc — it is a false promise to somebody
      # deciding whether to trust this with their backups. So it is refreshed daily; it
      # just asks first.
      - name: refresh the README matrix (PR on the canonical forge)
        if: ${{ github.event_name == 'schedule' && !contains(github.server_url, 'github.com') }}
        env:
          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
        run: |
          python3 - <<'PY'
          import json, sys
@@ -151,87 +169,73 @@ jobs:
          print("changed" if compat.update_readme(rows) else "unchanged")
          PY
-          if ! git diff --quiet -- README.md; then
+          git diff --quiet -- README.md && { echo "matrix unchanged — nothing to propose"; exit 0; }
            git config user.name  "truecloud-patch bot"
            git config user.email "bot@onetick.ninja"
            git add README.md
            git commit -m "docs: refresh the TrueNAS compatibility matrix"
            git push origin HEAD:main
          fi
-      # A broken SHIPPED release is an outage: users are on it right now.
+          git config user.name  "truecloud-patch bot"
          git config user.email "bot@onetick.ninja"
          BRANCH=bot/compat-matrix
          git checkout -B "$BRANCH"
          git add README.md
          git commit -m "docs: refresh the TrueNAS compatibility matrix"
          git push -f origin "$BRANCH"
          # ONE long-lived PR, force-pushed in place — not a new one every morning.
          # (A daily PR is the same mistake as a daily comment, wearing a hat.)
          BRANCH="$BRANCH" python3 - <<'PY'
          import json, os, urllib.error, urllib.request
          api, token, branch = os.environ["API"], os.environ["TOKEN"], os.environ["BRANCH"]
          h = {"Authorization": f"token {token}", "Content-Type": "application/json"}
          def call(url, method="GET", data=None):
              r = urllib.request.Request(
                  url, method=method, headers=h,
                  data=json.dumps(data).encode() if data else None)
              with urllib.request.urlopen(r) as resp:  # noqa: S310
                  return json.load(resp) if resp.length != 0 else {}
          existing = [
              p for p in call(f"{api}/pulls?state=open")
              if p["head"]["ref"] == branch
          ]
          if existing:
              print(f"PR #{existing[0]['number']} already open; the force-push updated it")
          else:
              pr = call(f"{api}/pulls", "POST", {
                  "head": branch, "base": "main",
                  "title": "docs: refresh the TrueNAS compatibility matrix",
                  "body": (
                      "The daily compatibility check found that the support matrix in "
                      "the README no longer matches iXsystems' actual middleware.\n\n"
                      "This only touches the block between the `COMPAT MATRIX` markers. "
                      "It is regenerated by `tools/compat.py --matrix --update-readme` "
                      "and force-pushed, so it always reflects the latest run."
                  ),
              })
              print(f"opened PR #{pr['number']}")
          PY
      # ONE bug report, kept in sync. It is edited in place when the findings change and
      # says NOTHING when they do not.
      #
      # The first version commented on every run and left 11 identical 3,000-character
      # comments on one issue in a single day. A bot that repeats itself daily gets
      # muted, and then the next real finding is scrolled past — which defeats the whole
      # reason for building it.
      #
      # Runs on whichever forge it lands on; compat_publish.py handles both, so the two
      # cannot drift.
      - name: file / update / close the bug report
        env:
          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
          API: ${{ contains(github.server_url, 'github.com') && 'https://api.github.com' || format('{0}/api/v1', github.server_url) }}/repos/${{ github.repository }}
        run: python3 tools/compat_publish.py --api "$API" --token "$TOKEN" --matrix /tmp/matrix.json
      # A broken SHIPPED release is an outage: users are on it right now. Fails LAST, so
      # the report is filed before the job goes red.
      - name: fail if a shipped release is broken
        if: ${{ steps.check.outputs.shipped_broken != '0' }}
        run: |
          echo "::error::The patch is broken on a SHIPPED TrueNAS release."
          exit 1
      - name: file a bug report (GitHub)
        if: ${{ steps.report.outputs.broken == '1' && contains(github.server_url, 'github.com') }}
        env:
          GH_TOKEN: ${{ github.token }}
          TITLE: "TrueNAS compatibility: the patch's assumptions no longer hold"
        run: |
          # One issue per set of broken refs, reopened/updated rather than duplicated
          # daily -- a bot that files the same issue every morning gets muted, and
          # then it is not a warning system any more.
          # Lowest-numbered match, for the same reason as the Gitea step below.
          existing="$(gh issue list --state all --search "$TITLE" \
                        --json number,title \
                        --jq '[.[] | select(.title == env.TITLE) | .number] | min // empty')"
          if [ -n "$existing" ]; then
            gh issue comment "$existing" --body-file /tmp/issue.md
            gh issue reopen "$existing" 2>/dev/null || true
          else
            gh issue create --title "$TITLE" --body-file /tmp/issue.md
          fi
      - name: file a bug report (Gitea)
        if: ${{ steps.report.outputs.broken == '1' && !contains(github.server_url, 'github.com') }}
        env:
          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
          TITLE: "TrueNAS compatibility: the patch's assumptions no longer hold"
        run: |
          # python3, not jq: jq is not guaranteed on a self-hosted runner, and a bug
          # report that dies on a missing tool is a warning system that does not warn.
          python3 - <<'PY'
          import json, os, urllib.error, urllib.request
          api, token, title = os.environ["API"], os.environ["TOKEN"], os.environ["TITLE"]
          with open("/tmp/issue.md", encoding="utf-8") as fh:
              body = fh.read()
          headers = {"Authorization": f"token {token}",
                     "Content-Type": "application/json"}
          def call(url, method, data=None):
              req = urllib.request.Request(
                  url, method=method, headers=headers,
                  data=json.dumps(data).encode() if data else None)
              with urllib.request.urlopen(req) as r:  # noqa: S310
                  return json.load(r) if r.length != 0 else {}
          # Same title => same issue. Comment on it rather than filing a new one every
          # morning: a bot that duplicates itself daily gets muted, and then it is not
          # a warning system any more.
          # LOWEST-numbered match, not "whichever the API returns first". Two issues
          # with the same title already existed once (the old title embedded the ref
          # list, so the identity changed when that set changed), and an
          # order-dependent pick would have alternated between them, reopening one and
          # commenting on the other. Lowest number is stable no matter what the API
          # sorts by.
          issues = call(f"{api}/issues?state=all&type=issues", "GET")
          matches = sorted((i for i in issues if i["title"] == title),
                           key=lambda i: i["number"])
          match = matches[0] if matches else None
          if match:
              n = match["number"]
              call(f"{api}/issues/{n}/comments", "POST", {"body": body})
              call(f"{api}/issues/{n}", "PATCH", {"state": "open"})
              print(f"commented on and reopened issue #{n}")
          else:
              made = call(f"{api}/issues", "POST", {"title": title, "body": body})
              print(f"filed issue #{made['number']}")
          PY
@@ -124,7 +124,7 @@ jobs:
          echo "--- release body ---"
          cat /tmp/notes.md
-      # This repo is canonically hosted on Gitea (git.onetick.ninja) and mirrored to
+      # This repo is canonically hosted on Gitea (git.arch.fyi) and mirrored to
      # GitHub, and BOTH run this workflow -- Gitea reads .github/workflows too. So
      # the publish step has to work on whichever forge it lands on. Everything
      # above is forge-agnostic; only the "create a release" API differs.
@@ -3,6 +3,10 @@
 /apply.log.1
 /apply.log.2
 /hook_status.json
 # The resolved middlewared directory, recorded by apply.sh so wait_restart.sh can
 # check whether the patched modules are still on the live path without
 # re-deriving site-packages.
 /.mw_dir
 /disabled
 /nested_snapshots_enabled
 # Written by apply.sh when a module's assumptions no longer fit the installed
@@ -6,6 +6,320 @@ is deliberate: see [Releasing](docs/releasing.md). Twelve releases were cut on
 live, every one of those interrupts every user. An alert people learn to ignore is
 worse than no alert, because one day it carries a security fix.
 ## v0.8.0 — 2026-08-26
 ### Changed
 - **CI's python matrix is green on the self-hosted Gitea runner again.** The real
  failure was that the Gitea runner image executes jobs as root, and the two
  unreadable-sidecar tests build their scenario with `chmod(0)` — which cannot make
  a file unreadable for root (`CAP_DAC_OVERRIDE`). Those two tests now skip as root
  with that reason; GitHub's non-root runner still exercises them. The matrix also
  moved to uv-managed interpreters (one toolchain across both runners) and ruff is
  pinned to 0.16.1 so an upstream ruff release can't turn `main` red without a code
  change.
 - **README badges point at the public GitHub mirror** (workflow status and
  releases) instead of the private forge. The release badge had also been reading
  the stale Gitea v0.6.1 release instead of the current v0.7.0 on GitHub.
 - **`master` is now labelled `27-dev`, because it is not the next release.** iX
  branches each major onto its own `release/` line and master rolls straight on to the
  one after — on 2026-07-14 every recent commit on master targeted `27.0.0-BETA.1`
  while 26 was still in beta. So a **BROKEN** master row, rendered as
  "master _(unreleased)_", read as *"the version you are about to install is broken"*
  when the breakage was a major release away on a line nobody can download. In a table
  whose entire job is helping somebody decide whether to trust this with their backups,
  that is a false alarm in the worst possible place. The label is derived from the
  newest major in the matrix plus one, so it rolls over to `28-dev` by itself once 27
  branches.
  For the record, the breakage is `NAS-141498` (2026-06-24), "Convert cloud_backup
  plugin to the typesafe pattern": it re-signatures `restic_backup` and
  `get_restic_config`, splitting `entry`/`credentials` out of the `cloud_backup` dict.
  It is deliberately not being chased while the 27 line is still churning.
 ### Fixed
 - **CI turned `main` red on two of three pushes without running a single test,
  then stopped running at all.** Two failures, one cause: four concurrent jobs on
  one self-hosted runner.
  `act`, the engine behind the Gitea runner, caches each *action* as one shared
  git clone under `/root/.cache/act/<hash>` and re-pulls it per job. Jobs
  starting in the same second fight over that directory, and the loser dies with
  `lstat /root/.cache/act/<hash>/.npmrc: no such file or directory` — before any
  suite output exists, with a different victim each push (3.12 on one, 3.11 on
  the next). Separately, the runner force-pulls its base image per job, so four
  jobs meant four anonymous Docker Hub pulls per push; a few pushes and re-runs
  in one afternoon hit `429 Too Many Requests` and *every* job began failing
  before it started — including the shell job, which nothing had touched.
  CI is now a single job. Dropping one action (uv is installed by a `run:` step
  rather than `astral-sh/setup-uv`, which was only ever fetching a binary) only
  shrank the surface, because every job still used `actions/checkout`.
  Concurrency is the actual ingredient, so removing it removes the whole class:
  one job cannot race itself whatever actions it uses, and one job is one pull.
  The Python sweep moved inside that job and still runs every version after one
  fails — that is what `fail-fast: false` bought, and losing it would mean a 3.11
  break hides whether 3.12 and 3.13 are fine. The cost is wall-clock
  parallelism, which this repo does not need: the suite is ~1.5s, so container
  start and interpreter downloads dominate either way.
  A red gate that is usually noise is worse than no gate, because the one time
  it means something, nobody looks.
 - **The patch survived being applied and then silently stopped existing, because
  something else remounted `/usr` four seconds later.** On a box running
  TrueNAS 25.10.6 the boot of 2026-08-19 went: 16:41:56 `apply.sh` mounts its
  overlay on `/usr/lib/python3/dist-packages`, patches `b2.py`/`restic.py`, logs
  every step `OK`; **16:42:00** a second PREINIT hook runs `systemd-sysext
  refresh` over `/usr` — `Unmerged '/usr'.` / `Merged extensions into '/usr'.` —
  and our overlay, which lives *inside* that hierarchy, is torn off with it;
  16:47:24 our own deferred restart fires exactly as designed and middlewared
  imports the **stock** modules. Every B2 TrueCloud Backup task then failed with
  `NotImplementedError` from stock `rclone/base.py` for nineteen hours, across
  four scheduled runs, while `apply.log` and `hook_status.json` both said the
  patch was active.
  Nothing in the patch was wrong, which is the point: applying at PREINIT and
  restarting later is only sound if the patched files are still on the live path
  when middlewared re-imports them, and **that is not something PREINIT can
  guarantee**. Init scripts run sequentially in id order, so any hook registered
  after ours always wins. Worse, hook ordering cannot fix it either —
  middlewared's own `docker.configure_nvidia` merges a sysext over `/usr` at
  *runtime*, long after every PREINIT hook is finished.
  So the deferred restart no longer trusts the PREINIT pass. `wait_restart.sh`
  now re-applies immediately before it restarts middlewared — after boot has
  settled, which is also after every sysext merge and docker nvidia
  configuration — and verifies the marker is genuinely on the live path before
  restarting. It is no longer `exec systemctl try-restart middlewared`, because
  something has to run afterwards.
  What runs afterwards deliberately does **not** restart again. `try-restart`
  returns as soon as middlewared is READY, and middlewared then brings docker up
  — `docker.configure_nvidia` merges the stock nvidia sysext over `/usr` at that
  point, detaching the overlay *after* the patched modules have already been
  imported. A disk check there reports "missing" on a perfectly healthy system,
  and restarting on that signal would restart a correctly-patched middlewared
  straight back into the same race. So the overlay is re-mounted for the benefit
  of the next restart, and the question of whether *this* middlewared actually
  holds the patch is left to the one thing that can answer it exactly — the
  in-process alert below. That re-mount preserves `hook_status.json`'s
  `patched_at`: `create_task.py verify` decides "loaded" by comparing
  middlewared's start time against that stamp, so a re-apply running *after* the
  restart would have made the stamp newer than the process which correctly
  imported the patch, and `verify` would have reported FAIL forever on every
  boot where the sysext merge detaches the overlay. Caught on hardware while
  validating the candidate — a new lying status introduced by the fix for a
  lying status.
  Two supporting fixes fell out of the same failure. `_ensure_writable` treated
  "one of our overlays is listed on this directory" as "already done" — but it
  only ever reaches that check when the directory is **not** writable, and a live
  overlay of ours always is. A shadowed overlay was therefore indistinguishable
  from a healthy one; it is now detached and re-mounted, reusing the same
  upperdir so everything patched earlier in the boot reappears intact, with a
  fresh workdir because overlayfs refuses one left behind by a detached mount.
 - **middlewared now says so when it is running stock.** The gap that let this
  cost nineteen hours was not the remount, it was that nothing could tell the
  difference between "patched on disk" and "patched in the running process".
  `apply.log` can only ever report the first. A new CRITICAL alert asks the
  second question from inside middlewared, hourly, where it is exact: the patch
  stamps the objects it replaces, so a missing stamp means this interpreter
  imported stock code. It checks both halves — `restic.py`'s `_truecloud_patched`
  marker and whether `B2RcloneRemote.get_restic_config` is still the base class's
  — since either can go missing alone. It stays quiet when the kill switch is
  set or the providers module has been retired as native, and it is deliberately
  **not** silenced by `update_alerts_disabled`: that mutes release notifications,
  not a broken backup path.
  Boot-time diagnosis also no longer depends on the journal. `wait_restart.sh`
  logged only to the journal, and journald retention on a busy box is easily
  shorter than the interval between reboots — the 2026-08-19 boot had already
  rotated away by the time it was investigated. It now writes to `apply.log`
  alongside everything else.
 - **The next maintenance release was never checked, and it is the one that reaches
  users.** Shipped versions were discovered from `TS-*` tags and unreleased ones from
  `release/*` branches carrying `-BETA`/`-RC`. A branched-but-untagged *maintenance*
  release is neither: `release/25.10.5` has no tag, and its line has already shipped,
  so the "a prerelease of a shipped line is history" filter discarded it. It was
  invisible — and it is precisely what a 25.10.4 box gets on its next update. A break
  there would have reached real users before the daily check ever looked at it, on the
  only line anybody is actually running.
  A plain `release/X.Y.Z` branch is now checked when its line **has** shipped and it
  sorts **newer** than that line's newest tag. Both things that must stay out fall out
  of the same rule: `release/24.10-RC.2` sorts older than `TS-24.10.2.4` (history, not
  a warning), and iX's typo branch `release/25.20.2.2` is on a line that has no tag at
  all, so it is not a release line. This immediately surfaced two refs that had never
  been checked — `release/25.10.5` and `release/24.10.2.5` — both of which pass.
  `is_unreleased()` now keys off where a ref came from (branch = not yet shipped)
  rather than looking for `-BETA`/`-RC` in its name. Otherwise `release/25.10.5` would
  count as shipped and a break in it would fail the build as a live outage — on a
  version nobody is running yet.
 - **An unchanged fingerprint froze the bug report's body, not just its comments.** Two
  questions were sharing one answer. *Have the findings changed?* gates **comments** —
  they notify, and a daily "still broken, same as yesterday" is what teaches everyone
  to ignore the one that finally matters. *Is the body still true?* gates the **body** —
  and editing an issue body notifies nobody on either forge, so keeping it honest is
  free. Conflated, the report could never be corrected while the findings held steady,
  and the fingerprint deliberately ignores everything that moves on its own — healthy
  rows, the hardware-verified column, point releases, and how a row is labelled. The
  `master` → `27-dev` relabel above would have reached the README and never the issue
  anybody actually opens. The body is now rewritten whenever it is out of date (after
  normalising line endings, so a forge round-tripping `\r\n` does not cause a rewrite
  every run) and comments remain strictly a changelog of real changes.
 - **A change to the publisher did not re-run the check.** `compat.yml`'s `push:` paths
  listed `tools/compat.py` but not `tools/compat_publish.py` — so the very commit that
  taught the bot to refresh a stale report body triggered no run, and the report stayed
  stale until the next scheduled one. A fix nobody runs is a fix nobody has.
 - **The compatibility bot filed a new duplicate bug report on every Gitea run.**
  `find_issue()` skipped pull requests by testing for the *presence* of the
  `pull_request` key. GitHub omits that key on a plain issue; Gitea sends it as
  `null`. So on Gitea every issue was discarded as a PR, the lookup always came back
  empty, and the bot took the "nothing filed yet" branch and opened a fresh report
  each run — **nine copies on the canonical forge**, four of them filed *after* the
  commit that was meant to stop precisely this. The mirror was fine, which is why it
  went unnoticed: GitHub's payload shape is the one the filter was written against.
  It is the same failure the anti-spam fix was written to prevent, moved from
  comments to issues, and it survived because `find_issue` was the only function in
  `compat_publish.py` with no test. It now has one, per forge, and the daily cron —
  which had not yet run once — no longer accumulates a report a day.
  The issue list is also requested with **both** paging parameters (`per_page` for
  GitHub, `limit` for Gitea). Each forge ignores the other's, and Gitea's default page
  is 30, so the lookup would have started missing the report again once the pile it
  was creating grew past one page.
 ## v0.7.0 — 2026-07-14
 ### Added
 - **TrueNAS 26 support, verified on a real TrueNAS 26 install.** 26 deletes
  `plugins/zfs_/` outright, taking the private `zfs.dataset.query`,
  `zfs.snapshot.query` and `zfs.snapshot.delete` with it. Every one of those was on
  the nested module's critical path, so nested snapshots were **BROKEN** on 26 and
  `apply.sh` correctly refused to apply the module there.
  Snapshot **deletion** now resolves its namespace at runtime — `pool.snapshot` on
  25.10 and 26, `zfs.snapshot` on 24.10 and 25.04, because no single namespace spans
  every supported release. `tools/compat.py` checks the same list the runtime uses,
  so what CI verifies and what runs cannot drift apart.
  Hardware-verified on TrueNAS 26.0.0-BETA.1: a 274-snapshot recursive backup of a
  292-dataset pool, then a **byte-identical restore of a four-level-deep child
  dataset**.
 ### Fixed
 - **Enumeration no longer trusts middleware's dataset and snapshot queries — they
  are filtered.** This is the important one, and it is the bug that a test VM caught
  and no amount of source analysis ever could have.
  The obvious port of the deleted private `zfs.dataset.query` was the public
  `pool.dataset.query`. It exists, it is documented, it is covered by iX's
  deprecation policy — and it is **not a like-for-like replacement**. It applies a
  *visibility policy*: it hides the datasets TrueNAS considers its own — `ix-apps/*`,
  `.system/*`, `.ix-virt/*`. On a real pool that is **84 of 270 datasets**, and
  `ix-apps` holds **live application data**.
  Staging from that view would have silently omitted every one of them. Worse,
  `plan_staging()` would never have seen them, so they would not have appeared in its
  `skipped` list either — no warning, no failure, just a green backup quietly missing
  data. That is precisely the failure this module exists to prevent. The snapshot
  query lies the same way (205 of 274), so the sweep would have orphaned one snapshot
  per hidden dataset, on every run, forever.
  The module now **reads the truth from ZFS and makes changes through middleware**:
  enumeration is `zfs list`, which no policy can filter and which behaves identically
  on every release; mutation stays a middleware call, so TrueNAS's own bookkeeping
  stays consistent. A failing `zfs list` raises rather than returning an empty list —
  "no datasets" and "the command broke" must never look the same.
  **No shipped release is affected.** v0.6.1 and earlier call the *private*
  `zfs.dataset.query`, which returns all 270 datasets. The bug existed only in the
  unreleased TrueNAS 26 port.
 - **The patch now owns the snapshot sweep even when it does not stage anything.**
  Stock decides whether to take a *recursive* snapshot by its own rule, and on
  TrueNAS 26 that rule stopped being ours.
  Up to 25.10, stock's `create_snapshot` called `get_dataset_recursive()` — the same
  function this module vendors — so "stock went recursive" and "we have something to
  stage" were the *same question*, and stock's non-recursive delete was correct for
  everything the patch declined to stage. TrueNAS 26 uses `filesystem.statfs`:
  `recursive = (path == the dataset's mountpoint)`. The two rules now disagree for a
  dataset whose only descendants are **ZVOLs** or **legacy/none-mountpoint** datasets
  — stock snapshots it recursively, while the patch sees nothing to stage.
  The patch then handed the snapshot back to stock, which destroys the parent only.
  With no staging tree there was no sidecar, and the garbage collector only ever ran
  from the staging path — so nothing on the box would ever have found the children.
  Reproduced on the test VM: one orphaned snapshot per zvol, on every run, forever,
  with the backup reporting success. Ownership of the sweep is no longer conditional
  on staging.
 - **The runtime resolved a *namespace*; the checker verified a *method*.** Those are
  different questions, and the gap is a false "ok". `get_service()` only proves a
  namespace is registered — it says nothing about whether `delete` still exists on it.
  So if iX guts the method while keeping the service (they have already done exactly
  that to `pool.snapshot.do_update` on master), `tools/compat.py` would fall through
  to `zfs.snapshot`, report the box healthy, and let the patch apply — while the
  runtime picked `pool.snapshot` and failed *every* delete, orphaning the whole tree.
  Both sides now ask the same question, and a test binds the two lists together.
 - `query_filesystems()` **dropped malformed `zfs list` rows silently** — the last
  remaining silent-omission path, and a direct contradiction of this module's cardinal
  rule. It raises now. A missing `zfs` binary raised `FileNotFoundError` rather than
  `ZfsError`; also fixed.
 - The snapshot retry loop **discarded the delete error** and reported every survivor
  as "(still busy?)" — naming the one cause that is benign and self-healing, and
  hiding the ones that are permanent. It keeps and reports the real error.
 - The staging-failure handler could **lose the original exception** if its own cleanup
  sweep raised. An error handler must not be able to lose the error.
 - **A snapshot delete that returns cleanly is not proof that anything was deleted.**
  The recursive sweep's fast path took the call's word for it and returned "no
  survivors" — so `cleanup_task` read that as a clean sweep and removed the sidecar,
  the only record the tree ever existed. Roughly 250 snapshots would have been orphaned
  on every run, with nothing left able to find them, and the backup reporting success.
  This is not a hypothetical about a well-behaved API: iX has already gutted
  `pool.snapshot.do_update` on master into a no-op whose body is commented out and
  which returns `None`. A source check still sees the `def`; a runtime check still sees
  a callable method. Only asking ZFS can tell. The sweep now confirms against ZFS, and
  where it *cannot* confirm it keeps owning the tree rather than claiming success — a
  false survivor self-heals on the next run, a lost record never does.
 - `_write_sidecar` **swallowed `OSError`**. The sidecar is the only thing that survives
  a middlewared restart; failing to write it is not fatal, but it must never be
  invisible. `_read_sidecar` had the mirror bug — it conflated "there is no sidecar"
  with "I could not read the sidecar", and `cleanup_task` then took the empty branch
  and **unlinked the only record** of a tree it had failed to read.
 - **A dataset from another tree, mounted inside the backup path, was omitted
  silently.** The staging plan scopes by dataset *name*, which is correct — a dataset
  with no mountpoint cannot be scoped by path at all. But ZFS lets any dataset mount
  anywhere, so one from an unrelated tree can sit inside the path:
      Tank/photos   mountpoint=/mnt/Tap/apps/photos
  It holds data inside the backed-up path, and `zfs snapshot -r Tap@…` does **not**
  cover it: recursion follows the dataset tree, not the directory tree. So there is no
  snapshot of it to stage, and no way to capture it consistently with the rest. It fell
  out of the name filter and vanished — not staged, not in `skipped`, no error, backup
  green. Stock has the same blind spot, but stock also refuses the nested config
  outright; this patch is what relaxes that guard, so the hole is this patch's to close.
  It now refuses, and names the offending datasets.
 ## v0.6.1 — 2026-07-13
 ### Fixed
@@ -0,0 +1,4 @@
 truenas-truecloud-patch
 Parts of this project were written with AI assistance (Claude); all of it is
 reviewed and tested before release.
@@ -1,5 +1,7 @@
 # truenas-truecloud-patch
 [![CI](https://git.arch.fyi/flan/truenas-truecloud-patch/actions/workflows/ci.yml/badge.svg)](https://git.arch.fyi/flan/truenas-truecloud-patch/actions)
 Extends TrueNAS SCALE's **TrueCloud Backup** to:
 - back up to **Backblaze B2 and any S3-compatible provider**, not just Storj;
@@ -59,9 +61,11 @@ If something is wrong, the reason is in `apply.log` — start at
 | --- | --- | --- | --- |
 | 24.10.2.4 | ok | ok | — |
 | 25.04.2.6 | ok | ok | — |
-| 25.10.4 | ok | ok | nested + providers; 252-snapshot recursive backup of /mnt/Tap, 18m |
+| 25.10.6 | ok | ok | — |
-| 26.0.0-BETA.3 _(unreleased)_ | ok | **BROKEN** | — |
+| 24.10.2.5 _(unreleased)_ | ok | ok | — |
-| master _(unreleased)_ | **BROKEN** | **BROKEN** | — |
+| 25.10.7 _(unreleased)_ | ok | ok | — |
 | 26.0.0-BETA.3 _(unreleased)_ | ok | ok | — |
 | master _(27-dev)_ | **BROKEN** | **BROKEN** | — |
 | verdict | meaning |
 | --- | --- |
@@ -72,17 +76,44 @@ If something is wrong, the reason is in `apply.log` — start at
 "ok" means *the patch's assumptions hold*, checked automatically against iX's
 source. It does not mean a human ran a backup on it — that is the
 **Hardware-verified** column, which is filled in by hand and only by doing it.
 **`master` is not the next release.** iX branches each major off to its own
 `release/` line and master rolls straight on to the one after — so master is
 `27-dev` while 26 is still in beta. A **BROKEN** master means iX has changed
 something that will reach users *a major release from now*, not in the version you
 are about to install. Read the numbered rows for that.
 A row like `25.10.5 _(unreleased)_` is the next maintenance release: branched by iX,
 not tagged yet, and the very next thing a 25.10.4 box gets. It is checked precisely
 because it is the one unshipped ref that reaches real users without warning.
 <!-- END COMPAT MATRIX -->
 The table is **regenerated daily by CI** against iXsystems' actual middleware source
 — it is not a claim somebody typed once and forgot.
-**TrueNAS 26: nested snapshots are not supported yet, and upgrading will not break
+It is also **static analysis**: it proves the patch's assumptions still hold, which is
-you.** 26 rewrites `cloud_backup` and deletes the ZFS methods this module calls. On
+a weaker claim than "a backup ran and a restore came back". For what has actually been
-26 `apply.sh` finds that the assumptions no longer hold and **does not apply the
+run — which tasks, on which hardware, and the md5 of the file that came back — see
-module**: TrueNAS is left stock, B2/S3 keeps working, nested datasets are simply not
+[docs/verification.md](docs/verification.md).
-covered, and the reason is named in `apply.log`. A broken backup is worse than a
+
-missing feature. Details: [How it works](docs/how-it-works.md#truenas-26).
+**TrueNAS 26 is supported** as of v0.7.0, and was verified on a real
 **26.0.0-BETA.1** install: a 274-snapshot recursive backup of a 292-dataset pool, and a
 byte-identical restore of a four-level-deep child dataset. The *Hardware-verified*
 column tracks the newest beta iX has tagged (currently BETA.3), so it does not carry
 that mark — a build nobody has actually run a backup on does not get credit for one.
 26 rewrites `cloud_backup` from async to synchronous and deletes the private ZFS
 methods this module used to call, so getting there took real work: the patch now
 injects the wrapper flavour that matches the installed middleware, reads dataset and
 snapshot lists **from ZFS rather than middleware** (whose queries hide TrueNAS's own
 datasets — 84 of 270 on a real pool, including live app data), and owns the snapshot
 sweep even when it stages nothing (26 decides `recursive` by a rule this patch does not
 share, and would otherwise orphan one snapshot per zvol on every run).
 **And if a future TrueNAS breaks it, you get a missing feature, not a broken backup.**
 `apply.sh` re-checks the patch's assumptions at every boot and **refuses to apply a
 module whose assumptions no longer hold** — TrueNAS is left stock, and the reason is
 named in `apply.log`. Details: [How it works](docs/how-it-works.md#truenas-26).
 ---
@@ -125,6 +156,17 @@ alert, which both take the newest plain `vX.Y.Z` tag. That is what lets debuggin
 happen in `-rc` tags instead of in your notification bell — see
 [Releasing](docs/releasing.md).
 **A second alert reports the patch not being loaded**, and this one you cannot
 turn off with `--no-update-alerts` — it is CRITICAL, hourly, and it means B2/S3
 backup tasks are about to fail. Being patched *on disk* and being patched *in the
 running middlewared* are different facts, and only middlewared can answer the
 second one: the patch stamps the objects it replaces, so a missing stamp means
 the process imported stock code. It fires if something detaches the patch overlay
 (a `systemd-sysext` merge over `/usr`, for instance) and the self-healing re-apply
 in the deferred restart could not put it back. `bash install.sh` clears it. It
 stays quiet when the kill switch is set, or when the providers module has been
 retired because TrueNAS went native.
 The changelog is read from whichever forge `origin` points at, derived from the
 remote rather than hard-coded. That is not cosmetic: when the changelog cannot be
 read, the alert deliberately fires **anyway** rather than risk hiding a security
@@ -207,5 +249,7 @@ and restores the original UI bundle from backup.
 - Filing a TrueNAS bug? **Remove the patch first** and reproduce on a stock system.
 - Provided as-is, no warranty. See LICENSE.
-Parts of this project were written with AI assistance (Claude); all of it is reviewed
+## Support
-and tested before release. Bugs are mine.
+
 If truenas-truecloud-patch is useful to you, consider supporting development via
 [GitHub Sponsors](https://github.com/sponsors/sudolulo) or [Ko-fi](https://ko-fi.com/sudolulo).
@@ -74,22 +74,46 @@ Two different things must survive two different events:
   and creates a transient systemd unit (`truecloud-mw-restart`, via
   `systemd-run --no-block`) running `patch/wait_restart.sh` — detached so it
   cannot disrupt the remainder of the boot sequence.
-5. **Once boot has settled, middlewared restarts once** and imports the
+5. **Once boot has settled, the patch is re-applied and middlewared restarts
-   patched modules from the overlay. `wait_restart.sh` holds the restart until
+   once**, importing the patched modules from the overlay. `wait_restart.sh`
-   the systemd boot job queue has drained (so in-flight `ix-*` units like
+   holds the restart until the systemd boot job queue has drained (so in-flight
-   `ix-reporting` finish first) *and* middlewared's docker/apps startup has
+   `ix-*` units like `ix-reporting` finish first) *and* middlewared's docker/apps
-   reached a terminal state — plain unit ordering cannot see either, and
+   startup has reached a terminal state — plain unit ordering cannot see either,
-   restarting middlewared while they run kills apps and dashboard reporting
+   and restarting middlewared while they run kills apps and dashboard reporting
   for the whole boot. S3/B2 backup support is then active until the next
   reboot, when the cycle repeats.
   The **re-apply** in that sentence is load-bearing, not a safety blanket. The
   overlay from step 3 sits *inside* `/usr`, so anything that remounts that
   hierarchy detaches it, and two ordinary things do exactly that after our hook
   has finished: another PREINIT script running `systemd-sysext merge`/`refresh`
   over `/usr` (an out-of-tree nvidia driver, say), and middlewared's own
   `docker.configure_nvidia` when it brings docker up. Init scripts run
   sequentially in id order, so a hook registered after ours always wins — and
   ordering them differently would still not help, because `docker.configure_nvidia`
   fires at runtime. `wait_restart.sh` therefore re-runs `apply.sh` at the point
   where boot has settled and every such remount is behind it, re-mounting the
   overlay if it was torn off (same upper layer, so files patched in step 3
   reappear intact), then verifies the patch is really on the live path, restarts,
   and verifies again — retrying once if it was lost in between.
   This is the failure that made it necessary: on 2026-08-19 the overlay was
   mounted at 16:41:56 and a sysext refresh unmerged and remerged `/usr` four
   seconds later. The restart at 16:47:24 loaded stock modules, and every B2
   backup failed for nineteen hours while `apply.log` said `OK` — because
   `apply.log` can only report what was written to disk, never what the restart
   imported. That second question is now asked from inside middlewared by an
   hourly CRITICAL alert (see [Update alerts](../README.md#update-alerts)).
 What you will observe: one middlewared restart shortly after every boot (a
 brief web UI/API blip; running services are unaffected). Between steps 3
 and 5 there is a short window — typically well under a minute — where the UI
 already shows S3/B2 (the JS bundle is read from disk per request) but the
 backend is still stock. A backup job that fires inside that window fails once
 with `NotImplementedError` and succeeds on its next run; see
-[Troubleshooting](recovery.md) if it persists beyond boot.
+[Troubleshooting](recovery.md) if it persists beyond boot. If the backend is
 still stock an hour after boot, middlewared raises the "installed but NOT
 loaded" alert rather than leaving you to notice via a failed backup.
 Manual runs of `bash patch/apply.sh` never trigger the restart — that only
 happens in boot context. `install.sh` and `recover.sh` perform their own
@@ -187,23 +211,71 @@ backup-breaking**, and none of them is visible from the `cloud_backup` files:
 | `get_dataset_recursive()` **deleted** from `plugins/cloud/snapshot.py` | `NameError` — the injected block called it out of the host module's namespace |
 | `plugins/zfs_/dataset.py` and `zfs_/snapshot.py` **deleted** | `zfs.dataset.query`, `zfs.snapshot.query` and `zfs.snapshot.delete` all vanish. 26 uses `filesystem.statfs` and `zfs.resource.*` |
-The first two are fixed: the patch reads which flavour of `cloud_backup` your box
+All three are fixed as of **v0.7.0**, and 26 is supported.
 declares and injects the wrapper that matches (one implementation of the real logic,
 two thin wrappers), and it carries its own copy of the deleted helper.
-The third is **not** fixed, and is why 26 reports BROKEN. Porting it means rewriting
+The first two were straightforward: the patch reads which flavour of `cloud_backup`
-the module's ZFS calls onto 26's new API, and no single API spans 24.10 through 26 —
+your box declares and injects the wrapper that matches (one implementation of the real
-so it needs a real 26 box to verify against, not a plausible-looking diff. Shipping a
+logic, two thin wrappers), and it carries its own copy of the deleted helper.
 port nobody has run is exactly the failure this project exists to avoid.
-It is also the row that would have hurt most. `zfs.snapshot.delete` is what sweeps the
+The third was not, and it is the one that would have hurt most — `zfs.snapshot.delete`
-recursive snapshot; without it, **every run would orphan one snapshot per descendant
+is what sweeps the recursive snapshot, and without it **every run would orphan one
-dataset — 250 on a real pool — forever.** The compatibility check caught it only
+snapshot per descendant dataset (250 on a real pool), forever, while reporting
-because it now asserts the middleware *methods the patch calls*, not just the symbols
+success.**
 it wraps.
-`master` (development after 26) reports BROKEN too: iXsystems are still reshaping
+The obvious port is to the public `pool.dataset.query` / `pool.snapshot.query`. Those
 methods exist, are documented, and are covered by iX's deprecation policy — and they
 are **not like-for-like replacements**. They apply a *visibility policy*: they hide the
 datasets TrueNAS considers its own (`ix-apps/*`, `.system/*`, `.ix-virt/*`). On a real
 pool that is **84 of 270 datasets, including live application data.** Staging from that
 view would have omitted every one of them from the backup — and the planner would never
 have seen them, so they would not have appeared in its "skipped" list either. A green
 backup, quietly missing data. The snapshot query lies the same way, so the sweep would
 have orphaned one snapshot per hidden dataset.
 No source analysis could have caught that. The methods are all present and correctly
 shaped. Only running it could, which is why it took a real 26 box.
 So the module now follows one rule:
 > **Read the truth from ZFS. Make changes through middleware.**
 Enumeration is `zfs list` — no policy can filter it, and it behaves identically on every
 release, which also means one code path instead of a version conditional. Mutation stays
 a middleware call, so TrueNAS's own bookkeeping stays consistent; an exact-name delete
 works fine even on a dataset the query hides. It is only enumeration that lies.
 The snapshot *delete* still needs a namespace, and no single one spans every release —
 24.10 and 25.04 have `zfs.snapshot`, 26 has only `pool.snapshot`, 25.10 has both. So it
 is resolved at runtime, by asking whether the namespace can actually delete. `tools/compat.py`
 asks the identical question against iX's source, and a test binds the two lists together,
 so what CI verifies and what runs cannot drift apart.
 ### The one 26 changed that nothing warned about
 Stock decides whether to take a **recursive** snapshot by its own rule, and on 26 that
 rule stopped being ours:
 | | decides `recursive` by |
 | --- | --- |
 | stock ≤ 25.10 | `get_dataset_recursive()` — the same function this patch vendors |
 | **stock 26** | `filesystem.statfs`: `recursive = (path == the dataset's mountpoint)` |
 | this patch | `get_dataset_recursive()` — is a mounted *filesystem* child under the path? |
 Up to 25.10 those were the *same question*, so a snapshot the patch declined to stage
 provably had no children and stock's non-recursive delete was correct. On 26 they
 disagree: a dataset whose only descendants are **zvols** or **legacy-mountpoint**
 datasets gets a recursive snapshot, while the patch sees nothing to stage. Stock then
 destroys the parent only — and with no staging tree there was no sidecar, and the
 garbage collector only ever ran from the staging path. Nothing on the box would ever
 have found the children.
 It was reproduced on a 26 VM (one orphan per zvol, every run, backup green) and closed:
 **ownership of the sweep is no longer conditional on staging.**
 `master` (development after 26) **does** report BROKEN: iXsystems are still reshaping
 these functions there, renaming `middleware` → `context` and `cloud_backup` → `entry`
 and adding a required `credentials` parameter. That is a moving target and is
 deliberately not chased; the check keeps reporting it until it settles into a beta,
-which is when it becomes worth fixing.
+which is when it becomes worth fixing. Until then, a box running master would simply
 not get the modules — `apply.sh` refuses to apply a module whose assumptions no longer
 hold, and says why in `apply.log`.
@@ -120,8 +120,9 @@ If a module shows `[FAIL]`:
 The traceback ends in `rclone/base.py` → `raise NotImplementedError` and
 contains no `_tc_` frames: the running middlewared is executing stock code.
-Either the deferred restart never fired, or the patch never landed on disk
+Either the deferred restart never fired, the patch never landed on disk this
-this boot. Diagnose in this order:
+boot, or it landed and was then torn off before the restart. Diagnose in this
 order:
 ```bash
 # Did apply.sh run this boot, at which version, and did it schedule the restart?
@@ -130,11 +131,21 @@ tail -40 /mnt/tank/truenas-truecloud-patch/apply.log
 # Full check — compares the running process against the patch timestamp
 python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
-# Did the deferred restart unit run, fail, or never get created?
+# What the deferred restart did -- re-apply, restart, and what it verified.
-systemctl status truecloud-mw-restart.service
+# apply.log is the durable record; journald retention on a busy box is often
 # shorter than the gap between reboots, so the journal may have nothing left.
 grep wait_restart /mnt/tank/truenas-truecloud-patch/apply.log | tail -20
 journalctl -u truecloud-mw-restart.service --no-pager | tail -20
 # Did something remount /usr and detach the patch overlay?
 systemd-sysext status
 findmnt -o TARGET,SOURCE /usr/lib/python3/dist-packages
 ```
 `systemctl status truecloud-mw-restart.service` reporting *"could not be
 found"* is **normal** — the unit is transient and is collected once it exits.
 It is not evidence that the restart was skipped.
 - `verify` reports the process started **before** the patch → the restart
  didn't happen. `systemctl restart middlewared` fixes it immediately; the
  journal output above tells you why it was missed.
@@ -145,6 +156,22 @@ journalctl -u truecloud-mw-restart.service --no-pager | tail -20
 - `apply.log` header shows `[v0.0.3]` or older → update:
  `git pull && bash install.sh` (v0.0.4 fixed patches not loading after
  reboot).
 - `apply.log` says the patch applied, but `findmnt` shows no `truecloud-mw`
  overlay on the dist-packages path → something remounted `/usr` after our
  PREINIT hook and detached it. `systemd-sysext status` names the culprit if it
  is a sysext (the `SINCE` column will sit a few seconds *after* the `apply.log`
  timestamp). Releases from 2026-08-26 on re-apply and verify immediately before
  the restart, so this should self-heal; if you are seeing it, update first.
 **TrueNAS raises "truecloud-patch is installed but NOT loaded"**
 The definitive symptom, and it does not depend on a backup failing first: the
 running middlewared has stock cloud_backup modules even though the patch is
 installed and its providers module is meant to be active. `bash install.sh`
 re-applies and restarts. The alert clears within the hour. It is silent when the
 kill switch is set or the providers module has been retired as native, and it is
 deliberately not muted by `update_alerts_disabled` — that silences release
 notifications, not a broken backup path.
 **Apply log** (check after each reboot or install):
 ```bash
@@ -25,7 +25,7 @@ are worth calling out, because nothing else would catch what they catch:
  newest CHANGELOG entry. `VERSION=` had silently drifted to three different
  values across the scripts before anything checked.
-The project is hosted on **Gitea** (`git.onetick.ninja/flan/truenas-truecloud-patch`)
+The project is hosted on **Gitea** (`git.arch.fyi/flan/truenas-truecloud-patch`)
 and mirrored to GitHub. Both run the same workflows — Gitea reads
 `.github/workflows/` too — so a change is checked twice, on two independent runners.
@@ -0,0 +1,77 @@
 # What has actually been run
 The support matrix in the README is **static analysis**: it proves the patch's
 assumptions about middlewared still hold. That is a strictly weaker claim than "a
 backup ran and a restore came back". This file is the stronger claim, and it is
 maintained by hand, because the only way to fill it in is to do it.
 If you are deciding whether to trust this with your backups, read this file, not the
 matrix.
 ---
 ## v0.7.0 — TrueNAS 25.10.4 (production hardware)
 Six live TrueCloud tasks, all `snapshot = true`, backing up to Backblaze B2. Three were
 exercised end to end, chosen to cover the three shapes the code handles differently:
 | Task | Path | Shape | Result |
 | --- | --- | --- | --- |
 | 5 | `/mnt/Tap` | 191 nested datasets staged, 282-snapshot recursive tree | SUCCESS |
 | 7 | `/mnt/Tank/backups` | 215 filesystems **+ 2 zvols** | SUCCESS |
 | 9 | `/mnt/Tank/flan` | **no** nested filesystem children | SUCCESS |
 After every run: **0 orphaned snapshots, 0 leaked bind mounts, 0 stale sidecars.**
 **The restore.** `apps/vaultwarden/data/config.json` — a file inside a *child* dataset,
 which is exactly what stock TrueNAS cannot capture — was restored from B2 and compared
 against the live file:
    live      f809df6ba231986b1ba824044228a03a   1808 bytes
    restored  f809df6ba231986b1ba824044228a03a   1808 bytes
    => byte-identical
 **The collector earned its keep on real data.** The pool was already carrying an orphan:
 `Tap/apps/prometheus@cloud_backup-5-20260713202355`, left behind by an earlier run when
 ZFS's automount held the snapshot busy past all four retries. The first v0.7.0 run found
 it by name, reclaimed it, and the pool's snapshot count went 2148 → 2147. That is the
 garbage collector doing the job it was written for, against a leak that was already
 there and that nothing else would ever have found.
 **Boot path.** `apply.sh` is registered as a PREINIT `initshutdownscript`; it was
 re-run against the live middleware and left exactly one `TRUECLOUD_PATCH` marker in
 each patched module (a second copy stacked into a live middlewared module would break
 the box at boot). It correctly detected the box as **async** (`cloud_backup is async
 (TrueNAS <= 25.10)`) and injected the matching wrappers.
 **Upgrade path.** `update.sh` was used to move the box from the release candidate to
 the stable tag, in detached HEAD at `v0.7.0`, which is how a user's box actually
 upgrades.
 ## v0.7.0 — TrueNAS 26.0.0-BETA.1 (VM)
 A throwaway VM whose pool reproduces the production pool's *shape* — 292 datasets, 26
 `legacy` mountpoints, nesting five deep — because every bug found on the real box came
 from the shape of the pool, not the bytes in it. MinIO was not used; `rclone serve s3`
 (already on the box) provided the S3 target, so no real B2 credential ever entered the
 VM.
 * 274-snapshot recursive backup of the 292-dataset pool. 0 orphans, 0 leaked mounts.
 * Restored `ix-apps/app_mounts/vaultwarden/pgData` — **four levels deep, and a dataset
  that middleware's own `pool.dataset.query` hides from itself** — byte-identical.
 * The zvol-orphan case was **reproduced with the fix disabled** (one orphan per zvol,
  every run, backup green), then **closed with it enabled**. See the CHANGELOG entry
  for why TrueNAS 26 decides `recursive` by a different rule than this patch decides
  `nested`.
 ## What is NOT covered
 * **24.10 and 25.04** are `ok` in the matrix — the assumptions hold, checked against
  iX's source — but nobody has run a backup on them. The matrix says so.
 * **master** is BROKEN, and correctly reports so: iX renamed the leading parameters of
  `get_restic_config` and `restic_backup`. It is not a shipped release; the daily
  compatibility bot files it, and `apply.sh` would refuse to apply the modules on a box
  running it.
 * A **reboot** of the production box has not been done on v0.7.0. `apply.sh` was
  re-executed by hand against the live middleware, which exercises the same code path,
  but the PREINIT ordering itself has only been proven on earlier versions.
@@ -18,7 +18,7 @@
 set -euo pipefail
-VERSION="0.6.1"
+VERSION="0.8.0"
 # The directory containing install.sh is the permanent install location.
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -21,6 +21,7 @@ the way a `git fetch` from middlewared (running as root) would.
 import datetime
 import importlib.util
 import json
 import logging
 import os
 import re
@@ -47,8 +48,8 @@ _VERSION_RE = re.compile(r'^VERSION="([^"]+)"', re.M)
 #: owner/repo out of any of:
 #:   git@github.com:sudolulo/repo.git
 #:   https://github.com/sudolulo/repo.git
-#:   ssh://git@git.onetick.ninja:55214/flan/repo.git
+#:   ssh://git@git.arch.fyi:55214/flan/repo.git
-#:   https://git.onetick.ninja/flan/repo.git
+#:   https://git.arch.fyi/flan/repo.git
 #: The SSH port is deliberately not captured: it is not the web port.
 _REMOTE_RE = re.compile(
    r"^(?:\w+://)?(?:[^@/]+@)?([^:/]+)(?::\d+)?[:/]([^/]+)/([^/]+?)(?:\.git)?/?$"
@@ -77,6 +78,107 @@ class TrueCloudPatchSecurityUpdateAlertClass(AlertClass):
    )
 class TrueCloudPatchNotLoadedAlertClass(AlertClass):
    category = AlertCategory.SYSTEM
    level = AlertLevel.CRITICAL
    title = "truecloud-patch is installed but NOT loaded"
    text = (
        "truecloud-patch patched middlewared on disk, but this middlewared is "
        "running the STOCK cloud_backup modules -- B2 and S3 TrueCloud Backup "
        "tasks will fail with NotImplementedError. Something remounted /usr "
        "after the patch was applied (a systemd-sysext merge, or "
        "docker.configure_nvidia), detaching the patch overlay. Re-apply with:  "
        "bash %(dir)s/install.sh"
    )
 class TrueCloudPatchNotLoadedAlertSource(ThreadedAlertSource):
    """Does the middlewared running this check actually have the patch in it?
    This is the one question apply.log cannot answer. apply.sh reports what it
    wrote to disk; whether the restart that followed imported those files is a
    separate fact, and on 2026-08-19 the two disagreed silently for nineteen
    hours while every B2 backup task failed. Asking from inside the process is
    exact -- the patch stamps the objects it replaces, so a missing stamp means
    this interpreter imported stock code.
    Deliberately NOT silenced by the update-alert marker: that mutes release
    notifications, not a broken backup path. Only the patch's own kill switch
    (the `disabled` file, meaning the operator turned the patch off) stops it.
    """
    schedule = IntervalSchedule(datetime.timedelta(hours=1))
    run_on_backup_node = False
    def check_sync(self):
        try:
            return self._check()
        except Exception:
            # An alert source must never take middlewared down with it.
            logger.debug("truecloud-patch loaded check failed", exc_info=True)
            return None
    # -- internals ------------------------------------------------------------
    def _check(self):
        if os.path.exists(os.path.join(PATCH_DIR, "disabled")):
            return None
        # Only the providers module puts B2/S3 on the restic path. If it was
        # never applied here, or TrueNAS went native and it was retired, then
        # "not loaded" is the correct state and not a fault.
        status = self._hook_status()
        if not status:
            return None
        providers = status.get("patches", {}).get("providers", {})
        if not providers.get("active"):
            return None
        if self._providers_loaded():
            return None
        return Alert(
            TrueCloudPatchNotLoadedAlertClass,
            {"dir": PATCH_DIR},
            key=None,
        )
    def _hook_status(self):
        try:
            with open(os.path.join(PATCH_DIR, "hook_status.json")) as f:
                return json.load(f)
        except (OSError, ValueError):
            return None
    def _providers_loaded(self):
        """True when THIS interpreter holds the patched provider objects.
        Two independent stamps, because the two halves are written separately
        and either can be missing on its own:
        * restic.py -- apply.sh sets `_truecloud_patched` on the wrapper it
          installs over `get_restic_config`.
        * b2.py -- apply.sh binds a B2-specific `get_restic_config` onto
          `B2RcloneRemote`. Comparing it against the base implementation is
          exact and survives renames of the patch's own helper.
        """
        try:
            from middlewared.plugins.cloud_backup.restic import get_restic_config
        except Exception:
            return False
        if not getattr(get_restic_config, "_truecloud_patched", False):
            return False
        try:
            from middlewared.rclone.base import BaseRcloneRemote
            from middlewared.rclone.remote.b2 import B2RcloneRemote
        except Exception:
            return False
        base = getattr(BaseRcloneRemote, "get_restic_config", None)
        b2 = getattr(B2RcloneRemote, "get_restic_config", None)
        return b2 is not None and b2 is not base
 class TrueCloudPatchUpdateAlertSource(ThreadedAlertSource):
    schedule = IntervalSchedule(datetime.timedelta(hours=24))
    run_on_backup_node = False
@@ -32,7 +32,7 @@
 # Derive PATCH_DIR from this script's location (parent of the patch/ directory).
 PATCH_DIR="$(cd "$(dirname "$0")/.." && pwd)"
 LOG="$PATCH_DIR/apply.log"
-VERSION="0.6.1"
+VERSION="0.8.0"
 # Rotate log at 512 KB to avoid unbounded growth on a system volume.
 # Keep two prior generations (.1 and .2) so the last three boots are always available.
@@ -64,17 +64,45 @@ _ensure_writable() {
        rm -f "$dir/.truecloud-probe"
        return 0
    fi
-    # Already our overlay on this exact directory from an earlier run this boot?
+    # Not writable, so any overlay of ours listed on this directory is a
    # SHADOWED leftover rather than a working mount: something remounted the
    # hierarchy above it -- a systemd-sysext merge/refresh over /usr, or
    # middlewared's own docker.configure_nvidia -- and buried it. A live
    # overlay of ours is always writable, so this must never be treated as
    # "already done"; doing so is what let a buried overlay pass for a healthy
    # one and left the backend patch on disk but never loaded.
    if mount | grep -qF "truecloud-${tag} on ${dir} "; then
-        return 0
+        echo "NOTICE: a previous truecloud-${tag} overlay on $dir is shadowed --"
        echo "NOTICE: the hierarchy above it was remounted. Detaching and re-mounting."
        umount -l "$dir" 2>/dev/null
    fi
    # Keep the SAME upperdir across re-mounts: it holds everything patched
    # earlier this boot, so re-mounting restores those files intact instead of
    # re-deriving them. The workdir is scratch and must be empty, so it is
    # recreated -- a stale one left behind by a detached mount fails the mount.
    local upper="/run/truecloud-${tag}-upper" work="/run/truecloud-${tag}-work"
-    mkdir -p "$upper" "$work"
+    mkdir -p "$upper"
    rm -rf "$work" 2>/dev/null
    mkdir -p "$work"
    if mount -t overlay "truecloud-${tag}" \
           -o "lowerdir=$dir,upperdir=$upper,workdir=$work" "$dir" 2>/dev/null; then
        echo "OK: Mounted writable overlay on $dir"
        return 0
    fi
    # A lazily-detached overlay releases its workdir only once its last user is
    # gone, and overlayfs refuses a workdir that is still in use. That would turn
    # the re-mount this function exists to perform into a hard failure, so retry
    # once on a private workdir. It is scratch in /run (tmpfs) and goes away at
    # the next boot; the upperdir, which holds the patched files, is unchanged.
    work="/run/truecloud-${tag}-work.$$"
    rm -rf "$work" 2>/dev/null
    mkdir -p "$work"
    if mount -t overlay "truecloud-${tag}" \
           -o "lowerdir=$dir,upperdir=$upper,workdir=$work" "$dir" 2>/dev/null; then
        echo "OK: Mounted writable overlay on $dir (fresh workdir)"
        return 0
    fi
    rmdir "$work" 2>/dev/null
    echo "WARNING: overlay mount failed on $dir — backend patch will be skipped."
    return 1
 }
@@ -196,6 +224,13 @@ _tc_native_nested=$(printf '%s' "$_tc_info" | sed -n '2p')
 SITE_PKG=$(printf '%s' "$_tc_info"          | sed -n '3p')
 _MW_DIR=$(printf '%s' "$_tc_info"           | sed -n '4p')
 # Record the resolved middlewared directory so patch/wait_restart.sh can check,
 # without re-deriving any of this, whether the patched modules are still on the
 # live filesystem path at the moment it restarts middlewared.
 if [ -n "$_MW_DIR" ]; then
    printf '%s\n' "$_MW_DIR" > "$PATCH_DIR/.mw_dir" 2>/dev/null
 fi
 # Nested support is opt-in; if it was never enabled, it cannot be the reason to
 # keep the patch alive.
 if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]; then
@@ -535,8 +570,8 @@ if _tc_nested is not None:
        # snapshot=true, not just ours. Two consequences, and the second is worse:
        #
        #   * everything below is a NEW failure mode for tasks that worked before we
-        #     were installed. A `zfs.dataset.query` that errors would break a
+        #     were installed. A `zfs list` that errors would break a CloudSync job
-        #     CloudSync job we have no business touching.
+        #     we have no business touching.
        #   * if a CloudSync task ever were staged, nothing would ever tear it down:
        #     the teardown is wired into cloud_backup's restic_backup finally, and
        #     CRUD_BLOCK deliberately leaves CloudSync's nesting guard intact. The
@@ -556,18 +591,28 @@ if _tc_nested is not None:
            # our staging plan would not -- silently omitting it from the backup.
            # Read afterwards, an unsnapshotted dataset instead trips the isdir()
            # check in plan_staging and fails the run loudly. Loud beats silent.
-            datasets = middleware.call_sync(
+            # query_filesystems() reads ZFS directly. It deliberately does NOT use
-                "zfs.dataset.query", [["type", "=", "FILESYSTEM"]]
+            # pool.dataset.query: that applies a visibility policy and hides
-            )
+            # TrueNAS-internal datasets (ix-apps/*, .system/*, .ix-virt/*) -- 84 of
            # 270 on a real pool, including live app data. Staging from the filtered
            # view omits them silently, which is the one thing this must never do.
            datasets = _tc_nested.query_filesystems(middleware)
            # OUR copy of get_dataset_recursive, not the host module's: TrueNAS 26
            # deleted that helper (create_snapshot uses filesystem.statfs now), so
            # calling it out of the module namespace is a NameError there.
            dataset, nested = _tc_nested.get_dataset_recursive(datasets, path)
            if not nested:
-                # No children: stock behaviour, untouched. Stock's `finally` owns
+                # Nothing to STAGE -- but we still own the SWEEP, and that is not a
-                # the snapshot from here (its non-recursive delete is correct,
+                # formality. Stock decides `recursive` by its own rule, and on 26 that
-                # because a non-nested snapshot has no children).
+                # rule is no longer ours: it snapshots recursively whenever the backup
                # path IS the dataset's mountpoint (filesystem.statfs), while
                # get_dataset_recursive() sees nothing to stage when the only
                # descendants are ZVOLs or legacy/none-mountpoint datasets. Stock then
                # deletes the PARENT ONLY. Without this, one snapshot per descendant is
                # orphaned on every run, forever, with no sidecar and no GC to find it --
                # and the backup still reports success.
                _tc_nested.own_snapshot(middleware, name, snapshot, logger=_logger)
                return snapshot, snap_path
            staging_root = _tc_nested.stage_nested(
@@ -581,7 +626,19 @@ if _tc_nested is not None:
            # stays None and its `finally` deletes NOTHING. Sweep the tree ourselves
            # or leak the parent plus one snapshot per descendant dataset (160+ here)
            # on every failed run.
-            _tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger)
+            #
            # The sweep is itself wrapped: a cleanup that raises would REPLACE the
            # original exception with its own, hiding why the backup actually failed.
            # An error handler must not be able to lose the error.
            try:
                _tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger)
            except Exception as _tc_sweep_err:
                if _logger:
                    _logger.error(
                        "truecloud-patch: could not sweep %s after a staging failure "
                        "(%r) -- it is orphaned and must be deleted by hand",
                        snapshot, _tc_sweep_err,
                    )
            raise
        return snapshot, staging_root
@@ -977,8 +1034,13 @@ fi
 # runs (install.sh, recovery) never trigger a restart.
 #
 # The unit runs wait_restart.sh, which blocks until boot has actually
-# settled (systemd job queue drained, docker/apps state terminal) before
+# settled (systemd job queue drained, docker/apps state terminal), then
-# restarting. systemd ordering alone (After=multi-user.target, ≤ v0.0.4)
+# RE-APPLIES this script before restarting. The re-apply is not belt-and-
 # braces: our overlay lives inside /usr, and a systemd-sysext merge or
 # middlewared's docker.configure_nvidia remounts /usr *after* PREINIT and
 # detaches it, so what we patch here can be gone by restart time (seen
 # 2026-08-19). wait_restart.sh re-mounts and re-verifies at the moment it
 # matters. systemd ordering alone (After=multi-user.target, ≤ v0.0.4)
 # fired while ix-reporting and the docker/apps startup were still in flight
 # and killed both — apps and dashboard stats stayed down until the next
 # boot. No Type=oneshot: a oneshot's start job would hold the boot queue
@@ -993,7 +1055,12 @@ echo "--- deferred restart ---"
 #
 # "No module active at all" cannot reach here: that is the kill-switch branch
 # above, which exits.
-if ! grep -aq middlewared "/proc/$PPID/cmdline" 2>/dev/null; then
+if [ "${TRUECLOUD_REAPPLY:-0}" = "1" ]; then
    # Invoked by patch/wait_restart.sh as its pre-restart re-apply pass. That
    # unit already exists to do the restart and verifies the result, so
    # scheduling another one here would be a loop.
    echo "Re-apply pass from wait_restart.sh — that unit owns the restart."
 elif ! grep -aq middlewared "/proc/$PPID/cmdline" 2>/dev/null; then
    echo "Manual run (parent is not middlewared) — no restart scheduled."
 elif [ "$_backend_ok" != "1" ]; then
    echo "Nothing landed on disk — no restart scheduled (nothing new to load)."
@@ -52,7 +52,7 @@ import subprocess
 import sys
 import time
-__version__ = "0.6.1"
+__version__ = "0.8.0"
 _PATCH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 _STATUS_FILE = os.path.join(_PATCH_DIR, "hook_status.json")
@@ -34,7 +34,7 @@ feature exists to prevent, and it would be worse than not having the feature.
 Snapshot lifecycle -- read this before changing anything
 --------------------------------------------------------
-``zfs.snapshot.delete`` defaults to ``recursive=False``, and stock
+The snapshot delete call defaults to ``recursive=False``, and stock
 ``restic_backup()`` calls it with no options. Stock gets away with that because
 its validation means ``recursive`` is never actually True in the field. Enabling
 nested datasets makes recursive snapshots real, so the parent
@@ -62,17 +62,25 @@ import subprocess
 import time
 __all__ = [
    "DELETE_METHODS",
    "SNAPSHOT_SERVICES",
    "STAGING_BASE",
    "StagingError",
    "ZfsError",
    "apply_plan",
    "cleanup_all",
    "cleanup_task",
    "current_mounts_under",
    "delete_snapshot_tree",
    "gc_stale_snapshots",
    "list_snapshot_names",
    "mounted_snapshots",
    "own_snapshot",
    "pick_snapshot_service",
    "plan_staging",
    "query_filesystems",
    "sidecar_for",
    "snapshot_service",
    "snapshot_tree_names",
    "stage_nested",
    "stale_snapshot_names",
@@ -81,9 +89,300 @@ __all__ = [
    "verify_staged",
 ]
 # ── how this module talks to the system ──────────────────────────────────────
 #
 #   READ the truth from ZFS.  MAKE CHANGES through middleware.
 #
 # That split is not stylistic. It was forced by finding, on a real TrueNAS 26 box,
 # that middleware's query APIs apply a VISIBILITY POLICY:
 #
 #   zfs list                 274 datasets      205 from pool.dataset.query
 #   zfs list -t snapshot     274 snapshots     205 from pool.snapshot.query
 #
 # The missing 69 are the datasets TrueNAS considers its own -- `ix-apps/*`,
 # `.system/*`, `.ix-virt/*` -- and on the real pool that is 84 of 270, including
 # `ix-apps`, which holds live application data. Enumerating from that view would
 # have silently omitted every one of them from the staging plan and from the
 # snapshot sweep: a green backup missing data, and one orphaned snapshot per
 # hidden dataset on every run. Both are exactly what this module exists to
 # prevent.
 #
 # This went unnoticed because the patch used to call the PRIVATE `zfs.dataset.query`
 # and `zfs.snapshot.query`, which return everything. TrueNAS 26 deleted them, and
 # the public replacements are NOT like-for-like -- they are filtered. So
 # enumeration now reads ZFS directly, which no policy can filter and which behaves
 # identically on every release.
 #
 # MUTATION still goes through middleware, so TrueNAS's own bookkeeping stays
 # consistent -- and an exact-name delete works fine even on a dataset the query
 # hides. The one wrinkle is that no single snapshot namespace spans every
 # supported release, so it is resolved at runtime rather than pinned:
 #
 #   24.10, 25.04   `zfs.snapshot`  (public back then; `pool.snapshot` does not exist)
 #   25.10          both -- `pool.snapshot` public, `zfs.snapshot` demoted to private
 #   26             `pool.snapshot` only -- `plugins/zfs_/` is gone
 #
 #: Snapshot CRUD namespaces, best first. `tools/compat.py` checks this exact list
 #: (MiddlewareCall.also) with the same predicate the runtime uses -- the namespace
 #: exists AND it defines `delete`/`do_delete` -- and a test binds the two lists
 #: together, so what CI verifies and what runs cannot drift apart.
 SNAPSHOT_SERVICES = ("pool.snapshot", "zfs.snapshot")
 #: The CRUDService method spellings that answer to `<namespace>.delete`. A
 #: CRUDService exposes `delete` from a method NAMED `do_delete`; both are live
 #: across the matrix. `tools/compat.py` accepts exactly this pair.
 DELETE_METHODS = ("delete", "do_delete")
 def pick_snapshot_service(can_delete):
    """First namespace in SNAPSHOT_SERVICES that can actually DELETE for us.
    Pure: `can_delete(namespace) -> bool`. Returns None if no namespace can,
    which is a middleware we have never seen and must not guess about.
    The predicate is "can delete", NOT "the service is registered", and the
    difference is the whole point. `get_service()` only proves the namespace is
    in the registry; it says nothing about whether `delete` still exists on it.
    `tools/compat.py` checks namespace AND method, so if the runtime settled for
    the weaker test the two could disagree — and would, in the one way that
    matters: iX guts a method while keeping its service (they have already done
    exactly that to `pool.snapshot.do_update` on master). compat would try
    `pool.snapshot`, find `delete` gone, fall through to `zfs.snapshot`, and
    report **ok**; the runtime would take `pool.snapshot` because the service is
    still registered, and then fail on every single delete — orphaning the whole
    tree while the backup reports success.
    Same predicate on both sides, so they cannot drift.
    """
    for name in SNAPSHOT_SERVICES:
        if can_delete(name):
            return name
    return None
 #: Where middlewared's own service framework lives. Classes from this package are
 #: PLUMBING, not implementations -- see `_defines_delete`.
 FRAMEWORK_PACKAGE = "middlewared.service"
 def _defines_delete(service):
    """Does this service ITSELF implement a delete -- or merely inherit the framework's?
    The distinction is the whole fix, and getting it wrong is silent.
    `CRUDService` defines `delete` on the BASE class and dispatches to `self.do_delete`
    at call time. So `getattr(service, "delete")` is a bound method on EVERY
    CRUDService subclass, whether or not that subclass still implements one:
        middlewared.plugins.pool_.snapshot.PoolSnapshotService  defines ['do_delete']
        middlewared.service.crud_service.CRUDService            defines ['delete']
    An earlier version of this check asked `callable(getattr(service, "delete"))` and
    was therefore answering "is this a CRUDService?" -- exactly the weaker "is the
    namespace registered?" question that `pick_snapshot_service` exists to avoid. It
    would have picked a gutted `pool.snapshot` and failed every delete.
    So walk the MRO and ignore the framework's generic plumbing: a delete is real only
    where a PLUGIN class defines it. That mirrors `tools/compat.py`, which looks for
    the `def` in the plugin file declaring the namespace.
    """
    for klass in type(service).__mro__:
        module = getattr(klass, "__module__", "") or ""
        if module == FRAMEWORK_PACKAGE or module.startswith(FRAMEWORK_PACKAGE + "."):
            continue                       # the framework's generic CRUD dispatcher
        if any(m in vars(klass) for m in DELETE_METHODS):
            return True
    return False
 def _can_delete(middleware, namespace):
    """Is `<namespace>.delete` actually implemented on this middleware?"""
    try:
        service = middleware.get_service(namespace)
    except Exception:
        # KeyError for an unregistered namespace; AttributeError if `get_service`
        # itself ever goes away. Both mean "cannot use it", and guessing YES on a
        # service that is not really there fails later, mid-backup, holding a
        # snapshot -- the worst possible moment to find out.
        return False
    return _defines_delete(service)
 def snapshot_service(middleware):
    """The snapshot namespace this middleware can actually delete through."""
    name = pick_snapshot_service(lambda n: _can_delete(middleware, n))
    if name is None:
        raise StagingError(
            "middleware exposes no usable snapshot delete ("
            + " / ".join(f"{n}.delete" for n in SNAPSHOT_SERVICES)
            + "). Refusing to stage a nested backup, because the snapshot it "
            "creates could not then be swept."
        )
    return name
 class _Snapshots:
    """This module's entire interface to snapshots, in one object.
    "READ the truth from ZFS, MAKE CHANGES through middleware" is the rule the whole
    module rests on. It used to live in a comment, while `middleware` and the ZFS
    reader were threaded through five functions **as a pair** -- and the namespace was
    re-resolved in each of them. That is one collaborator, not two, so it is one
    object; the rule is now structural rather than remembered.
    Deliberately private and constructed inside the public functions: `apply.sh`
    injects calls to those functions into middlewared itself, so their signatures are
    a boot-time contract with a live NAS and are not worth churning for tidiness.
    """
    def __init__(self, middleware, list_snapshots=None):
        self._mw = middleware
        self._list = list_snapshots or list_snapshot_names
        self._service = None
    @property
    def service(self):
        """The namespace we delete through. Resolved once, on first use.
        Lazy on purpose: resolving it raises when middleware has no usable delete,
        and the read-only paths must not blow up over a mutation they never make.
        """
        if self._service is None:
            self._service = snapshot_service(self._mw)
        return self._service
    def names(self, dataset):
        """Every snapshot at or under `dataset`, from ZFS. Raises if it cannot be read.
        Never from middleware: its snapshot query hides the internal datasets
        (205 of 274 on the test box), and a snapshot the sweep cannot SEE is a
        snapshot nothing will ever collect.
        """
        return self._list(dataset)
    def delete(self, name, recursive=False):
        """Delete one snapshot -- through middleware, so its bookkeeping stays right.
        An exact-name delete works even on a dataset the query hides; it is only
        enumeration that lies.
        """
        options = ({"recursive": True},) if recursive else ()
        return self._mw.call_sync(f"{self.service}.delete", name, *options)
 class ZfsError(Exception):
    """`zfs list` failed. Enumeration is unreliable, so the caller must not guess."""
 def _zfs_lines(args, runner=None, fields=None):
    """`zfs <args>` as a list of tab-split rows. Raises ZfsError if it fails.
    Never returns a partial or empty list on failure: a caller that cannot tell
    "no datasets" from "the command broke" will happily stage nothing, or sweep
    nothing, and report success.
    `fields`, if given, is the exact number of tab-separated columns every row must
    have. A row that does not is an ERROR, not something to skip. `zfs list -H`
    neither quotes nor escapes, so a mountpoint containing a tab or a newline would
    split wrong -- and quietly dropping that row would remove a dataset from the
    staging plan without it appearing in `skipped` either. Silent omission is the
    one thing this module may never do, so it raises instead.
    """
    runner = runner or _run
    try:
        r = runner(["zfs", *args])
    except OSError as e:
        # `zfs` missing from middlewared's PATH raises FileNotFoundError, which is
        # not a ZfsError and would sail past callers that only expect one.
        raise ZfsError(f"could not run zfs: {e}") from e
    if r.returncode != 0:
        raise ZfsError((r.stderr or "").strip() or f"zfs {' '.join(args)} failed")
    rows = [ln.split("\t") for ln in r.stdout.splitlines() if ln.strip()]
    if fields is not None:
        bad = [r for r in rows if len(r) != fields]
        if bad:
            raise ZfsError(
                f"zfs {' '.join(args)} returned {len(bad)} row(s) that do not have "
                f"{fields} tab-separated fields (first: {bad[0]!r}). Refusing to "
                f"guess -- a dropped row is a dataset silently missing from the backup."
            )
    return rows
 def query_filesystems(middleware=None, runner=None):
    """Every FILESYSTEM dataset, in the shape the planner speaks -- read from ZFS.
    NOT from `pool.dataset.query`, and this is the single most important decision
    in this file.
    middleware's dataset query applies a VISIBILITY POLICY: it hides the datasets
    TrueNAS considers its own -- `ix-apps/*`, `.system/*`, `.ix-virt/*`. That is
    **84 of 270 datasets** on the real pool, and `ix-apps` holds live application
    data. Building the staging plan from that view would silently omit every one of
    them. Worse, `plan_staging()` would never even SEE them, so they would not turn
    up in its `skipped` list either -- no warning, no failure, just a green backup
    quietly missing data. That is exactly the failure this whole module exists to
    prevent, and it is the failure the cardinal rule at the top of this file is
    about.
    It worked before only because the patch called the PRIVATE `zfs.dataset.query`,
    which returned everything. TrueNAS 26 deleted it. The public replacement is not
    a like-for-like: it is a filtered view.
    So: **read the truth from ZFS, make changes through middleware.** ZFS cannot
    apply a policy to what it reports, and `zfs list` behaves identically on every
    release -- which also means one code path instead of a version conditional.
    `middleware` is accepted and ignored, so callers need not care where the data
    comes from.
    """
    rows = _zfs_lines(
        ["list", "-H", "-p", "-o", "name,mountpoint,mounted", "-t", "filesystem"],
        runner=runner, fields=3,
    )
    return [
        {
            "name": name,
            "properties": {
                "mountpoint": {"value": mountpoint},
                # `zfs list` prints yes/no; the planner already speaks that.
                "mounted": {"value": mounted},
            },
        }
        for name, mountpoint, mounted in rows
    ]
 def list_snapshot_names(dataset, runner=None):
    """Every snapshot at or under `dataset` -- read from ZFS, for the same reason.
    `pool.snapshot.query` filters exactly like the dataset query does: on this box
    it returned 205 of 274 snapshots, hiding the internal datasets' snapshots. A
    sweep built on that view leaves one orphan per hidden dataset, on every run,
    forever -- which is the bug this module was written to fix in the first place.
    (An EXACT-name delete still works on a hidden dataset, so mutations may keep
    going through middleware. It is only enumeration that lies.)
    """
    rows = _zfs_lines(
        ["list", "-H", "-o", "name", "-t", "snapshot", "-r", dataset],
        runner=runner, fields=1,
    )
    return [r[0] for r in rows]
 #: Where staging trees are assembled. tmpfs; bind mounts consume no space.
 STAGING_BASE = "/run/truecloud-nested"
 #: The kernel's mount table. Late-bound (never a default argument) so a
 #: test can point it somewhere harmless -- a default is frozen at def time,
 #: which is how 19 tests ended up reading the REAL table, one matching name
 #: away from running a real `umount` on the NAS.
 MOUNTS_FILE = "/proc/self/mounts"
 # Which snapshot a staging tree pins is recorded ONLY in the sidecar file, never
 # also in memory. An in-process dict would be a second source of truth that a
 # middlewared restart silently empties -- and it is exactly the restart case that
@@ -118,7 +417,7 @@ def sidecar_for(staging_root: str) -> str:
    return staging_root + ".snapshot"
-def _write_sidecar(staging_root: str, snapshots) -> None:
+def _write_sidecar(staging_root: str, snapshots, logger=None) -> None:
    """Record every snapshot tree this task still owns. One per line.
    A LIST, not a single name -- and that is not over-engineering, it is a bug fix.
@@ -135,22 +434,48 @@ def _write_sidecar(staging_root: str, snapshots) -> None:
    """
    if isinstance(snapshots, str):
        snapshots = [snapshots]
-    with contextlib.suppress(OSError):
+    try:
        os.makedirs(os.path.dirname(staging_root), exist_ok=True)
        with open(sidecar_for(staging_root), "w", encoding="utf-8") as fh:
            fh.write("\n".join(dict.fromkeys(snapshots)))   # de-duped, order kept
    except OSError as e:
        # This used to be suppressed silently, and it is the LAST thing that should be.
        # The sidecar is the only record that these snapshots exist; if the write fails
        # (a full /run, say) cleanup_task finds nothing to sweep, and only the by-name
        # collector -- an hour later -- has any chance of finding them. Failing to
        # write it is not fatal, but it must never be invisible.
        if logger:
            logger.error(
                "truecloud-patch: COULD NOT RECORD the snapshot(s) %s (%r). If this "
                "run does not clean them up itself, only the by-name collector will "
                "ever find them.", ", ".join(snapshots), e,
            )
-def _read_sidecar(staging_root: str):
+def _read_sidecar(staging_root: str, logger=None):
    """Every snapshot tree a previous run recorded here. [] if none.
    Tolerates the old single-line format, which is just a one-element list.
    "There is no sidecar" and "I could not READ the sidecar" are different facts, and
    conflating them is dangerous: `cleanup_task` reads an empty list as "nothing was
    ever staged" and then REMOVES the sidecar -- destroying the only record of a tree
    it could not read. FileNotFoundError is the ordinary case and stays quiet; any
    other OSError is reported, and re-raised so no caller mistakes it for "empty".
    """
    try:
        with open(sidecar_for(staging_root), encoding="utf-8") as fh:
            return [ln.strip() for ln in fh if ln.strip()]
-    except OSError:
+    except FileNotFoundError:
-        return []
+        return []                                   # genuinely nothing recorded
    except OSError as e:
        if logger:
            logger.error(
                "truecloud-patch: could not READ the snapshot record %s (%r). Not "
                "touching it -- it may name snapshots nothing else can find.",
                sidecar_for(staging_root), e,
            )
        raise
 def _remove_sidecar(staging_root: str) -> None:
@@ -271,7 +596,7 @@ def plan_staging(base_dataset, base_mountpoint, path, snapshot_name, datasets,
                 staging_root, probe=_probe_snapdir):
    """Compute the bind-mount plan for staging a nested tree. Pure function.
-    ``datasets`` is a list of dicts shaped like ``zfs.dataset.query`` results:
+    ``datasets`` is what :func:`query_filesystems` returns:
    ``{"name": str, "properties": {"mountpoint": {"value": str},
    "mounted": {"value": "yes"|"no"}}}``.
@@ -345,13 +670,68 @@ def plan_staging(base_dataset, base_mountpoint, path, snapshot_name, datasets,
        mounts.append((src, os.path.join(staging_root, os.path.relpath(mp, path))))
    # ── datasets INSIDE the path but OUTSIDE the snapshot's tree ─────────────
    #
    # The loop above scopes by dataset NAME, which is right: a dataset with no
    # mountpoint cannot be scoped by path at all. But ZFS lets any dataset mount
    # anywhere, so a dataset from a DIFFERENT tree -- even a different pool -- can
    # sit inside the backup path:
    #
    #     Tank/photos   mountpoint=/mnt/Tap/apps/photos
    #
    # It holds data inside the path, so its absence is a hole in the backup. And
    # `zfs snapshot -r Tap@...` does NOT cover it, because recursion follows the
    # DATASET tree, not the directory tree -- so there is no snapshot of it to
    # stage, and no way to capture it consistently with the rest.
    #
    # Before this check it was neither staged, nor reported in `skipped`, nor raised:
    # it simply fell out of the name filter and vanished. The backup reported
    # SUCCESS with that data missing, which is the precise failure this module
    # exists to prevent. Stock has the same blind spot, but stock also REFUSES the
    # nested config outright -- we are the ones relaxing that guard, so the hole is
    # ours to close.
    foreign = []
    for ds in datasets:
        name = ds.get("name", "")
        if name.startswith(ds_prefix) or name == base_dataset:
            continue                                   # in-tree: handled above
        props = ds.get("properties", {})
        mp = props.get("mountpoint", {}).get("value", "")
        # `mp == path` as well as below it. A foreign dataset mounted exactly AT the
        # backup path is the same hole -- and it is worse, because it shadows the base
        # dataset's own directory, so we would stage what is hidden underneath instead
        # of the data actually visible there.
        if not mp or (mp != path and not mp.startswith(path_prefix)):
            continue                                   # not inside the backed-up path
        if props.get("mounted", {}).get("value", "yes") == "no":
            # An unmounted (locked/encrypted) dataset contributes nothing to the live
            # tree, so its absence is not a hole -- exactly as for an in-tree one, 40
            # lines above. Raising here would turn a working nightly backup into a
            # permanent failure the first time somebody locked a dataset.
            skipped.append((name, "dataset is not mounted (locked/encrypted?)"))
            continue
        foreign.append(name)
    if foreign:
        raise StagingError(
            "dataset(s) outside " + repr(base_dataset) + " are mounted inside the "
            "backup path and cannot be captured by its recursive snapshot: "
            + ", ".join(repr(f) for f in sorted(foreign))
            + ". Refusing to back up an incomplete tree -- move them, or back up "
            "their own dataset separately."
        )
    # Parents before children, so each mountpoint exists before we mount onto it.
    mounts.sort(key=lambda m: _depth(m[1]))
    return mounts, skipped
-def current_mounts_under(root, mounts_file="/proc/self/mounts"):
+def current_mounts_under(root, mounts_file=None):
    """Mountpoints at or under ``root``, deepest first. Used for teardown."""
    mounts_file = mounts_file or MOUNTS_FILE
    found = []
    try:
        with open(mounts_file, encoding="utf-8") as fh:
@@ -379,12 +759,13 @@ def _run(cmd):
    )
-def apply_plan(mounts, runner=_run, isdir=os.path.isdir):
+def apply_plan(mounts, runner=None, isdir=os.path.isdir):
    """Execute the bind-mount plan. Blocking; call via ``run_in_thread``.
    Raises StagingError on the first failure, after rolling back what was
    mounted -- a half-built tree must never be handed to the backup tool.
    """
    runner = runner or _run
    if not mounts:
        raise StagingError("empty staging plan")
@@ -437,12 +818,14 @@ def verify_staged(mounts, ismount=os.path.ismount, listdir=os.listdir):
    return True
-def teardown(staging_root, runner=_run, mounts_file="/proc/self/mounts"):
+def teardown(staging_root, runner=None, mounts_file=None):
    """Unmount the staging tree (deepest first) and remove the root.
    Idempotent, and does not depend on an in-memory plan -- so it also cleans up
    leftovers from a crashed run.
    """
    mounts_file = mounts_file or MOUNTS_FILE
    runner = runner or _run
    errors = []
    for mp in current_mounts_under(staging_root, mounts_file=mounts_file):
        res = runner(["umount", mp])
@@ -455,8 +838,9 @@ def teardown(staging_root, runner=_run, mounts_file="/proc/self/mounts"):
    return errors
-def snapdir_automounts(snapshot_name, mounts_file="/proc/self/mounts"):
+def snapdir_automounts(snapshot_name, mounts_file=None):
    """Every ``<dataset>/.zfs/snapshot/<snap>`` ZFS automount for this snapshot."""
    mounts_file = mounts_file or MOUNTS_FILE
    suffix = "/.zfs/snapshot/" + snapshot_name
    found = []
    try:
@@ -472,7 +856,7 @@ def snapdir_automounts(snapshot_name, mounts_file="/proc/self/mounts"):
    return sorted(found, key=_depth, reverse=True)      # deepest first
-def release_snapdirs(snapshot_name, runner=_run, mounts_file="/proc/self/mounts"):
+def release_snapdirs(snapshot_name, runner=None, mounts_file=None):
    """Unmount ZFS's OWN snapshot automounts, so the snapshots can be destroyed.
    Reading anything under ``<dataset>/.zfs/snapshot/<snap>/`` makes ZFS **automount**
@@ -489,6 +873,8 @@ def release_snapdirs(snapshot_name, runner=_run, mounts_file="/proc/self/mounts"
    Deepest first, so a child's automount is released before its parent's.
    """
    mounts_file = mounts_file or MOUNTS_FILE
    runner = runner or _run
    errors = []
    for mp in snapdir_automounts(snapshot_name, mounts_file=mounts_file):
        res = runner(["umount", mp])
@@ -560,7 +946,7 @@ def get_dataset_recursive(datasets, directory):
 def delete_snapshot_tree(middleware, snapshot, logger=None, attempts=4,
-                         sleep=time.sleep):
+                         sleep=None, list_snapshots=None):
    """Delete the parent snapshot AND every child created by ``zfs snapshot -r``.
    Returns the snapshots it could NOT delete -- callers must not throw that away.
@@ -578,6 +964,8 @@ def delete_snapshot_tree(middleware, snapshot, logger=None, attempts=4,
    snapshots hit this, and before the fix they were orphaned permanently.
    """
    dataset, _, snapname = snapshot.partition("@")
    snaps = _Snapshots(middleware, list_snapshots)
    sleep = sleep or time.sleep
    # Release ZFS's own automounts first, or `zfs destroy` refuses with EBUSY on
    # everything restic read in the last few minutes.
@@ -591,15 +979,55 @@ def delete_snapshot_tree(middleware, snapshot, logger=None, attempts=4,
    # through 252 sequential deletes leaves exactly the orphans this function
    # exists to prevent.
    try:
-        middleware.call_sync("zfs.snapshot.delete", snapshot, {"recursive": True})
+        snaps.delete(snapshot, recursive=True)
-        return []
+
-    except Exception as e:  # noqa: BLE001 - fall through to the explicit sweep
+        # CONFIRM it. A delete that returns without raising is not proof that anything
-        # Usually just "parent already gone" (stock's finally won the race once our
+        # was destroyed, and this is the one place where believing it is catastrophic:
-        # mounts were released), which the sweep below handles. Log it rather than
+        # `cleanup_task` reads an empty survivor list as "clean sweep" and REMOVES THE
-        # swallow it: if the real cause is something else, this is the only place
+        # SIDECAR -- the only record the tree ever existed. ~250 snapshots would be
-        # it is visible -- the sweep would report a different, downstream failure.
+        # orphaned per run, with nothing left to find them, and the backup green.
        #
        # Not paranoia about a hypothetical: iX has already gutted
        # `pool.snapshot.do_update` on master into a no-op whose body is commented out
        # and which returns None. An AST check still sees the `def`, and a callable
        # check still sees the method. Only asking ZFS can tell.
        #
        # It costs one `zfs list` (~350ms against 2148 snapshots) on an 18-minute
        # backup, and only on the path that would otherwise skip verification entirely.
        try:
            left = snapshot_tree_names(snapshot, snaps.names(dataset))
        except Exception as e:  # noqa: BLE001 - cannot confirm; do not claim success
            if logger:
                logger.warning(
                    "truecloud-patch: deleted %s but could not confirm it is gone "
                    "(%r); keeping it recorded so the next run re-checks", snapshot, e,
                )
            # Keep OWNING it. Reporting a clean sweep here makes cleanup_task drop the
            # sidecar; if the delete had in fact done nothing, the tree is orphaned
            # with no record. A survivor we later find already gone costs one
            # idempotent retry; a lost record costs the snapshots, permanently.
            return [snapshot]
        if not left:
            return []
        if logger:
-            logger.debug(
+            logger.warning(
                "truecloud-patch: the recursive delete of %s reported success but "
                "%d snapshot(s) are still there; sweeping them by name",
                snapshot, len(left),
            )
        # Fall through to the by-name sweep, which retries and reports survivors.
    except Exception as e:  # noqa: BLE001 - fall through to the explicit sweep
        # "Parent already gone" is the EXPECTED race (stock's finally won, once our
        # mounts were released) and happens on every clean run, so it is debug.
        # Anything else is a real fault -- a namespace that cannot delete, a schema
        # change, a permission error -- and this is the only place it is visible,
        # because the sweep below will report a different, downstream failure. At
        # debug it would never reach disk on stock middlewared, which logs at INFO.
        if logger:
            expected = "does not exist" in str(e).lower()
            (logger.debug if expected else logger.warning)(
                "truecloud-patch: recursive delete of %s failed (%r); sweeping "
                "the tree by name instead", snapshot, e,
            )
@@ -608,13 +1036,14 @@ def delete_snapshot_tree(middleware, snapshot, logger=None, attempts=4,
    # our mounts are released -- which fails the recursive delete while the
    # children survive. Sweep them by name.
    try:
-        snaps = middleware.call_sync(
+        # From ZFS, not middleware: the snapshot query hides internal datasets'
-            "zfs.snapshot.query", [["name", "^", dataset]], {"select": ["name"]}
+        # snapshots (205 of 274 on the test box), and a sweep that cannot see them
-        )
+        # orphans one per hidden dataset on every run. See list_snapshot_names().
        #
        # An empty result means the tree is already gone -- delete nothing, and
        # do not fall back to the parent, which would only log a spurious
        # "does not exist" warning on every clean run.
-        names = snapshot_tree_names(snapshot, [s["name"] for s in snaps])
+        names = snapshot_tree_names(snapshot, snaps.names(dataset))
    except Exception as e:  # noqa: BLE001 - fall back to at least the parent
        if logger:
            logger.warning(
@@ -623,36 +1052,52 @@ def delete_snapshot_tree(middleware, snapshot, logger=None, attempts=4,
            )
        names = [snapshot]
-    def confirm_gone(failed):
+    def still_there(tried, failed):
-        """Drop any name ZFS no longer has, even though its delete raised.
+        """Which of `tried` does ZFS STILL have? The delete's verdict is not evidence.
-        A delete that raised "does not exist" SUCCEEDED as far as we care, and must
+        ZFS is the authority, not the API's return value, and the difference is not
-        not be retried or reported. The query is only a refinement: if it cannot be
+        academic in either direction:
-        answered we keep the delete's own verdict, rather than inventing survivors --
+
-        a false survivor keeps the sidecar forever and is reported as a leak that
+        * A delete that RAISED "does not exist" succeeded as far as we care, and must
-        isn't there.
+          not be retried or reported as a leak.
        * A delete that RETURNED CLEANLY may have done nothing at all. iX has already
          gutted `pool.snapshot.do_update` on master into a no-op whose body is
          commented out and which returns None. Trusting that verdict makes
          `cleanup_task` see "no survivors", drop the sidecar -- the only record -- and
          orphan the whole tree, forever, silently.
        If ZFS cannot be read we cannot check either way -- so keep owning ALL of them.
        The two mistakes are not symmetric:
        * a false survivor SELF-HEALS. The sidecar is kept, the next run reclaims it,
          the delete raises "does not exist", and the record clears.
        * a lost record does NOT. The snapshots are orphaned with nothing pointing at
          them, and only the by-name collector -- an hour later, and only if ZFS is
          readable by then -- has any chance of finding them.
        """
        if not failed:
            return []
        try:
-            live = middleware.call_sync(
+            live = set(snaps.names(dataset))
-                "zfs.snapshot.query", [["name", "^", dataset]], {"select": ["name"]}
+        except Exception:  # noqa: BLE001 - cannot check; keep owning them
-            )
+            return list(tried)
-        except Exception:  # noqa: BLE001 - cannot refine; trust the delete's verdict
+        return [n for n in tried if n in live]
            return list(failed)
        live = {s["name"] for s in live}
        return [n for n in failed if n in live]
    remaining = list(names)
    last_error = {}
    for attempt in range(attempts):
        failed = []
        for name in remaining:
            try:
-                middleware.call_sync("zfs.snapshot.delete", name)
+                snaps.delete(name)
-            except Exception:  # noqa: BLE001 - busy, or already gone; sorted out below
+            except Exception as e:  # noqa: BLE001 - busy, or already gone; sorted below
                # KEEP the reason. This used to discard it and then report every
                # survivor as "(still busy?)" -- which names the one cause that is
                # benign and self-healing, and hides the ones that are permanent (a
                # namespace that cannot delete, a permission error, a schema change).
                # A misleading diagnosis is worse than none: it tells you to wait.
                last_error[name] = e
                failed.append(name)
-        remaining = confirm_gone(failed)
+        remaining = still_there(remaining, failed)
        if not remaining:
            return []
@@ -666,13 +1111,13 @@ def delete_snapshot_tree(middleware, snapshot, logger=None, attempts=4,
        if logger:
            logger.warning(
                "truecloud-patch: could not delete snapshot %s after %d attempts "
-                "(still busy?) -- it will be reclaimed on the next run",
+                "(last error: %r) -- it stays recorded and the next run reclaims it",
-                name, attempts,
+                name, attempts, last_error.get(name),
            )
    return remaining
-def mounted_snapshots(mounts_file="/proc/self/mounts"):
+def mounted_snapshots(mounts_file=None):
    """Every ZFS snapshot something is currently mounted from.
    The device field of a snapshot mount IS the snapshot name (`Tap/apps/x@snap`), for
@@ -681,6 +1126,7 @@ def mounted_snapshots(mounts_file="/proc/self/mounts"):
    protects a concurrently-running backup from the garbage collector, rather than
    trusting an age heuristic to be generous enough.
    """
    mounts_file = mounts_file or MOUNTS_FILE
    live = set()
    try:
        with open(mounts_file, encoding="utf-8") as fh:
@@ -689,12 +1135,17 @@ def mounted_snapshots(mounts_file="/proc/self/mounts"):
                if "@" in dev:
                    live.add(dev.replace("\\040", " "))
    except OSError:
-        return set()
+        # An empty set says "nothing is mounted", which silently switches OFF the GC's
        # protection for snapshots a CONCURRENT run is using -- leaving only the age
        # floor between us and destroying a snapshot out from under a backup that is
        # still uploading. If we cannot read the mount table we do not KNOW what is in
        # use, and must not pretend we do.
        raise
    return live
 def gc_stale_snapshots(middleware, task_name, current_snapshot, logger=None,
-                       now=None, mounts_file="/proc/self/mounts"):
+                       now=None, mounts_file=None, list_snapshots=None):
    """Delete snapshots this task left behind in an earlier run. Returns what remains.
    The backstop for when the RECORD is gone, not just the snapshots: the sidecar lives
@@ -705,13 +1156,15 @@ def gc_stale_snapshots(middleware, task_name, current_snapshot, logger=None,
    Selection is `stale_snapshot_names()`, which is pure and heavily tested, because a
    name match is a weaker claim than a recorded fact and this deletes data on one.
    """
    mounts_file = mounts_file or MOUNTS_FILE
    dataset = current_snapshot.partition("@")[0]
    now = now or datetime.datetime.now(datetime.UTC)
    snaps = _Snapshots(middleware, list_snapshots)
    try:
-        snaps = middleware.call_sync(
+        # From ZFS: middleware's snapshot query hides internal datasets, and an
-            "zfs.snapshot.query", [["name", "^", dataset]], {"select": ["name"]}
+        # orphan it cannot see is an orphan nothing will ever collect.
-        )
+        all_names = snaps.names(dataset)
    except Exception as e:  # noqa: BLE001 - cannot enumerate; collect nothing
        if logger:
            logger.warning(
@@ -720,7 +1173,7 @@ def gc_stale_snapshots(middleware, task_name, current_snapshot, logger=None,
        return []
    stale = stale_snapshot_names(
-        task_name, current_snapshot, [s["name"] for s in snaps], now,
+        task_name, current_snapshot, all_names, now,
        in_use=mounted_snapshots(mounts_file),
    )
    if not stale:
@@ -736,7 +1189,7 @@ def gc_stale_snapshots(middleware, task_name, current_snapshot, logger=None,
    remaining = []
    for name in stale:
        try:
-            middleware.call_sync("zfs.snapshot.delete", name)
+            snaps.delete(name)
        except Exception as e:  # noqa: BLE001 - busy, or gone; either way, next run
            remaining.append(name)
            if logger:
@@ -746,27 +1199,43 @@ def gc_stale_snapshots(middleware, task_name, current_snapshot, logger=None,
    return remaining
-def stage_nested(middleware, path, snapshot, base_dataset, base_mountpoint,
+def own_snapshot(middleware, task_name, snapshot, logger=None, list_snapshots=None):
-                 task_name, datasets, logger=None):
+    """Take ownership of `snapshot`'s whole tree: reclaim, collect, and record it.
    """Build a complete staging tree for `path` from the already-taken `snapshot`.
-    `snapshot` is a full ZFS snapshot name ("Tap@cloud_backup-5-2026...").
+    Call this on EVERY ``snapshot = true`` cloud_backup run — **whether or not the
    tree gets staged**. That unconditionality is the fix for a real leak, so do not
    make it conditional again.
-    `datasets` is the FILESYSTEM dataset list. **It MUST have been enumerated
+    Stock decides whether to take a RECURSIVE snapshot by its own rule, and that
-    AFTER `snapshot` was taken.** A list read beforehand can miss a dataset
+    rule is not ours:
    created in the gap: the recursive snapshot would capture it, but the staging
    plan would not, and its data would be silently omitted from the backup.
    Enumerated afterwards, an unsnapshotted dataset instead trips the isdir()
    check in plan_staging and fails the run loudly.
-    Returns the staging root to hand to the backup tool.
+    ``<= 25.10``
        stock's ``create_snapshot`` calls ``get_dataset_recursive()`` — the very
        function this module vendors. "Stock went recursive" and "we have something
        to stage" were therefore the *same question*, and a non-staged snapshot
        provably had no children. Stock's non-recursive delete was correct.
-    Raises StagingError if the tree cannot be staged completely -- the caller
+    ``26``
-    must let that propagate so the backup fails instead of silently uploading a
+        stock uses ``filesystem.statfs``: ``recursive = (path == the dataset's
-    partial tree. The caller is responsible for deleting `snapshot` in that case
+        mountpoint)``. Now the two rules disagree. A dataset whose only descendants
-    (see SNAPSHOT_BLOCK in apply.sh).
+        are **ZVOLs** or **legacy/none-mountpoint** datasets gets a RECURSIVE
        snapshot — while ``get_dataset_recursive()`` reports nothing to stage,
        because neither kind is a mounted filesystem under ``path``.
    In that gap stock takes one snapshot per descendant and then deletes only the
    parent (its ``finally`` destroys ``path=snapshot``, non-recursively). Nothing
    would ever have found the children: no staging tree, so no sidecar, and the GC
    only ever ran from :func:`stage_nested`. One orphan per zvol/legacy descendant,
    on every run, forever — while the backup reports SUCCESS. That is the exact
    failure this module exists to prevent, reintroduced by a gate.
    So ownership of the sweep is no longer conditional on staging. It is cheap:
    :func:`delete_snapshot_tree` is idempotent, and on a genuinely childless
    snapshot it is one recursive destroy of a snapshot stock has usually already
    removed.
    Returns the staging root; the sidecar sits beside it.
    """
    snapshot_name = snapshot.split("@", 1)[1]
    staging_root = staging_root_for(task_name)
    # A previous run may have crashed mid-flight; never build on top of that.
@@ -790,7 +1259,8 @@ def stage_nested(middleware, path, snapshot, base_dataset, base_mountpoint,
                "truecloud-patch: reclaiming snapshot tree from an earlier "
                "run: %s", stale,
            )
-        pending.extend(delete_snapshot_tree(middleware, stale, logger=logger))
+        pending.extend(delete_snapshot_tree(
            middleware, stale, logger=logger, list_snapshots=list_snapshots))
    if pending and logger:
        logger.warning(
@@ -806,16 +1276,54 @@ def stage_nested(middleware, path, snapshot, base_dataset, base_mountpoint,
    #
    # It runs AFTER the sidecar reclaim on purpose: the recorded path is authoritative
    # and cheap, and the GC should only ever be mopping up what the record lost.
-    pending.extend(
+    pending.extend(gc_stale_snapshots(
-        gc_stale_snapshots(middleware, task_name, snapshot, logger=logger)
+        middleware, task_name, snapshot, logger=logger,
-    )
+        list_snapshots=list_snapshots,
    ))
    # Record the snapshot BEFORE mounting anything, not after. middlewared can
    # die at any point (this patch even schedules a restart at boot), and the
    # sidecar is the only thing that survives it -- an in-process dict would take
    # the sole record of a 160-snapshot tree with it. Writing it after apply_plan
    # would leave exactly the crash window the sidecar exists to close.
-    _write_sidecar(staging_root, [*pending, snapshot])
+    _write_sidecar(staging_root, [*pending, snapshot], logger=logger)
    return staging_root
 def stage_nested(middleware, path, snapshot, base_dataset, base_mountpoint,
                 task_name, datasets, logger=None, list_snapshots=None):
    """Build a complete staging tree for `path` from the already-taken `snapshot`.
    `snapshot` is a full ZFS snapshot name ("Tap@cloud_backup-5-2026...").
    `datasets` is the FILESYSTEM dataset list. **It MUST have been enumerated
    AFTER `snapshot` was taken.** A list read beforehand can miss a dataset
    created in the gap: the recursive snapshot would capture it, but the staging
    plan would not, and its data would be silently omitted from the backup.
    Enumerated afterwards, an unsnapshotted dataset instead trips the isdir()
    check in plan_staging and fails the run loudly.
    Returns the staging root to hand to the backup tool.
    Raises StagingError if the tree cannot be staged completely -- the caller
    must let that propagate so the backup fails instead of silently uploading a
    partial tree. The caller is responsible for deleting `snapshot` in that case
    (see SNAPSHOT_BLOCK in apply.sh).
    """
    snapshot_name = snapshot.split("@", 1)[1]
    # Refuse BEFORE staging, not after restic has run. We are about to pin a recursive
    # snapshot with bind mounts; if this middleware has no usable snapshot delete we
    # could never sweep it, and the honest move is to fail now rather than take a
    # snapshot we cannot clean up. (`_Snapshots` resolves lazily on purpose -- the
    # read-only paths must not raise over a mutation they never make -- so the staging
    # path asks explicitly.)
    snapshot_service(middleware)
    staging_root = own_snapshot(
        middleware, task_name, snapshot, logger=logger,
        list_snapshots=list_snapshots,
    )
    try:
        mounts, skipped = plan_staging(
@@ -853,7 +1361,7 @@ def stage_nested(middleware, path, snapshot, base_dataset, base_mountpoint,
    return staging_root
-def cleanup_task(middleware, task_name, logger=None):
+def cleanup_task(middleware, task_name, logger=None, list_snapshots=None):
    """Tear down a task's staging tree and delete the snapshot it pinned.
    Safe to call unconditionally: a no-op when the task was never staged.
@@ -877,7 +1385,8 @@ def cleanup_task(middleware, task_name, logger=None):
    # finish reclaiming.
    survivors = []
    for snapshot in pinned:
-        survivors.extend(delete_snapshot_tree(middleware, snapshot, logger=logger))
+        survivors.extend(delete_snapshot_tree(
            middleware, snapshot, logger=logger, list_snapshots=list_snapshots))
    # KEEP the sidecar if anything survived. It is the only record that those
    # snapshots exist, and removing it orphans them permanently.
@@ -900,7 +1409,7 @@ def cleanup_task(middleware, task_name, logger=None):
            )
        # The SURVIVORS, not the trees we asked to delete. Writing the original list
        # back would keep re-sweeping trees that are already gone.
-        _write_sidecar(staging_root, survivors)
+        _write_sidecar(staging_root, survivors, logger=logger)
        return
    _remove_sidecar(staging_root)
@@ -909,7 +1418,7 @@ def cleanup_task(middleware, task_name, logger=None):
 # ── offline cleanup (uninstall.sh / recover.sh) ───────────────────────────────
-def cleanup_all(base=None, runner=_run, mounts_file="/proc/self/mounts",
+def cleanup_all(base=None, runner=None, mounts_file=None,
                glob_fn=None, read_sidecar=_read_sidecar):
    """Tear down every staging tree. Used by uninstall.sh and recover.sh.
@@ -920,6 +1429,8 @@ def cleanup_all(base=None, runner=_run, mounts_file="/proc/self/mounts",
    Returns ``(lines, errors)``: report lines to print, and unmount errors.
    """
    mounts_file = mounts_file or MOUNTS_FILE
    runner = runner or _run
    import glob as _glob
    base = base or STAGING_BASE
@@ -929,8 +1440,21 @@ def cleanup_all(base=None, runner=_run, mounts_file="/proc/self/mounts",
    # Report orphaned snapshots BEFORE removing the sidecars that name them --
    # a sidecar is the only record that an interrupted run's snapshot tree (one
    # snapshot per descendant dataset) is still on disk.
    unreadable = set()
    for sc in sorted(glob_fn(os.path.join(base, "*.snapshot"))):
-        for snap in read_sidecar(sc[: -len(".snapshot")]):
+        try:
            recorded = read_sidecar(sc[: -len(".snapshot")])
        except OSError as e:
            unreadable.add(sc)
            # REPORT it and carry on. This function's job is to get the mounts off, and
            # it is called precisely when the box is already in a bad state
            # (recover.sh, uninstall.sh). Letting one unreadable sidecar abort the run
            # would leave the staging tree mounted -- which pins the snapshots, which is
            # the exact situation the caller is trying to escape.
            lines.append(f"  WARNING: could not read the snapshot record {sc} ({e}).")
            lines.append("           It may name snapshots nothing else can find.")
            continue
        for snap in recorded:
            lines.append(f"  NOTE: an interrupted backup left snapshot '{snap}' behind.")
            lines.append(f"        Remove it and its children:  zfs destroy -r '{snap}'")
@@ -946,6 +1470,12 @@ def cleanup_all(base=None, runner=_run, mounts_file="/proc/self/mounts",
    if not errors:
        for sc in glob_fn(os.path.join(base, "*.snapshot")):
            if sc in unreadable:
                # Do NOT delete a record we could not read. We have no idea what it
                # names, and it may be the only thing that knows those snapshots exist.
                # Removing it here would be the very bug the read guard was added for,
                # committed by the cleanup path instead of the backup path.
                continue
            with contextlib.suppress(OSError):
                os.unlink(sc)
        with contextlib.suppress(OSError):
@@ -27,6 +27,50 @@
 # --wait` below waits for that same queue to drain — the unit would deadlock
 # on itself until the timeout. apply.sh schedules this with the default
 # service type, whose start job completes at fork.
 #
 # THE RE-APPLY PASS (added 2026-08-26). Applying the patch at PREINIT and
 # restarting later is only sound if the patched files are still on the live
 # path at the moment middlewared re-imports them. They may not be: our patch
 # lives in an overlay mounted *inside* /usr, and anything that remounts the
 # hierarchy above it detaches or buries that overlay. Two things on a normal
 # TrueNAS box do exactly that, both AFTER our PREINIT hook has run:
 #
 #   - `systemd-sysext merge/refresh` over /usr (an nvidia sysext, for
 #     instance) — `Unmerged '/usr'` then `Merged extensions into '/usr'`;
 #   - middlewared's own `docker.configure_nvidia`, which merges the stock
 #     nvidia sysext over /usr when it brings docker up.
 #
 # PREINIT scripts run sequentially in id order, so a hook registered after
 # ours always wins the race, silently. Observed 2026-08-19: our overlay was
 # mounted at 16:41:56 and a sysext refresh tore /usr down four seconds later;
 # the restart at 16:47:24 then loaded stock modules and every B2 cloud_backup
 # job failed for the next nineteen hours while apply.log said "OK".
 #
 # Ordering the hooks cannot fix this — docker.configure_nvidia re-merges at
 # runtime, long after every PREINIT hook is done. So instead of trusting the
 # PREINIT pass, re-apply immediately before the restart (apply.sh is
 # idempotent and re-mounts a lost overlay, keeping the same upperdir so
 # already-patched files survive), verify the marker is really on the live
 # path, and verify again afterwards.
 PATCH_DIR="$(cd "$(dirname "$0")/.." && pwd)"
 LOG="$PATCH_DIR/apply.log"
 _log() { echo "[wait_restart] $*" >> "$LOG" 2>/dev/null; }
 # Is the providers patch visible on the live filesystem path -- i.e. would a
 # middlewared starting right now import it? Reads the marker apply.sh leaves
 # in restic.py. Returns 0 when patched, 1 when stock, 2 when we cannot tell
 # (no recorded middlewared dir yet, or the file is gone).
 _patch_visible() {
    local mw_dir restic_py
    mw_dir=$(cat "$PATCH_DIR/.mw_dir" 2>/dev/null)
    [ -n "$mw_dir" ] || return 2
    restic_py="$mw_dir/plugins/cloud_backup/restic.py"
    [ -f "$restic_py" ] || return 2
    grep -q "TRUECLOUD_PATCH" "$restic_py" 2>/dev/null && return 0
    return 1
 }
 # 1. systemd layer: wait for the boot job queue to drain. This covers every
 #    ix-* oneshot still activating, including ix-reporting's in-flight midclt
@@ -39,6 +83,9 @@ timeout 900 systemctl is-system-running --wait > /dev/null 2>&1
 #    transitional states (PENDING/INITIALIZING/STOPPING/MIGRATING — see
 #    middlewared/plugins/docker/state_utils.py). An empty answer means
 #    midclt could not respond at all; keep waiting. Cap at 10 minutes.
 #    This also covers docker.configure_nvidia, the runtime /usr re-merge:
 #    waiting for docker to reach a terminal state means the merge that would
 #    bury our overlay has already happened by the time we re-apply below.
 for _ in $(seq 1 120); do
    _status=$(midclt call docker.status 2>/dev/null \
                  | grep -oE '"status": "[A-Z_]+"' | cut -d'"' -f4)
@@ -52,4 +99,70 @@ done
 #    queryable state (smb.configure and friends). Bounded insurance.
 sleep 30
-exec systemctl try-restart middlewared
+# 4. Re-apply pass. Boot has settled, so every sysext merge and docker nvidia
 #    configuration that could bury our overlay is behind us. Re-running
 #    apply.sh is cheap and idempotent: it re-mounts the overlay if it was
 #    detached (same upperdir, so files patched at PREINIT reappear intact)
 #    and re-patches anything that reverted to stock.
 _patch_visible
 case $? in
    0) _log "providers patch still visible on the live path before restart" ;;
    1) _log "PATCH LOST since PREINIT (something remounted /usr) — re-applying" ;;
    *) _log "cannot confirm patch state before restart — re-applying anyway" ;;
 esac
 TRUECLOUD_REAPPLY=1 /bin/bash "$PATCH_DIR/patch/apply.sh"
 if ! _patch_visible; then
    _log "WARNING: patch is STILL not on the live path after the re-apply pass;"
    _log "WARNING: restarting anyway, but middlewared will load stock modules."
 fi
 # 5. The restart itself.
 systemctl try-restart middlewared
 # 6. Record what the restart landed on -- but do NOT restart again on a miss.
 #
 # `try-restart` returns as soon as middlewared is READY; it then brings docker
 # up asynchronously, and `docker.configure_nvidia` merges the stock nvidia
 # sysext over /usr at that point. That detaches our overlay AFTER the new
 # middlewared has already imported the patched modules -- so a disk check here
 # can report "missing" on a perfectly healthy system. Restarting on that signal
 # would restart a correctly-patched middlewared and then hit the same race
 # again, so the disk is deliberately not treated as a verdict after the restart.
 #
 # The authoritative answer is whether the running process holds the patch, and
 # only middlewared can answer that. The alert source installed by apply.sh
 # checks exactly that, in-process and hourly, and is what reports a genuine
 # miss. What is still worth doing here is putting the overlay back, so the next
 # middlewared restart -- whenever and whyever it happens -- finds patched files.
 if _patch_visible; then
    _log "OK: providers patch present on the live path across the restart"
 else
    _log "overlay detached again after the restart (expected when docker's"
    _log "nvidia sysext merge follows it) -- re-mounting for the next restart."
    _log "Whether THIS middlewared loaded the patch is answered in-process by"
    _log "the 'installed but NOT loaded' alert, not by this check."
    # Preserve hook_status.json's patched_at across this re-mount.
    #
    # create_task.py verify decides "loaded" by comparing middlewared's start
    # time against patched_at. This re-apply restores the SAME patch the boot
    # pass already applied, but it runs *after* the restart -- so letting it
    # re-stamp would make patched_at newer than the process that correctly
    # imported the patch, and verify would report FAIL forever, on every boot
    # where docker's sysext merge detaches the overlay. That is precisely the
    # lying-status failure this release exists to remove, so do not introduce a
    # new one. The snapshot lives in /run, never in the repo: a leftover file
    # there would leave the tree dirty and update.sh refuses to run over that.
    _saved_status=/run/truecloud-hook_status.pre
    cp -p "$PATCH_DIR/hook_status.json" "$_saved_status" 2>/dev/null
    TRUECLOUD_REAPPLY=1 /bin/bash "$PATCH_DIR/patch/apply.sh"
    if [ -f "$_saved_status" ]; then
        mv -f "$_saved_status" "$PATCH_DIR/hook_status.json" 2>/dev/null
    fi
 fi
 _log "=== deferred restart complete ==="
@@ -17,7 +17,7 @@
 #   bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh
 #   systemctl restart middlewared
-VERSION="0.6.1"
+VERSION="0.8.0"
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -140,19 +140,60 @@ class TestSnapshotLeak:
    every path that creates one must also sweep the whole tree.
    """
-    def test_staging_failure_deletes_the_snapshot_tree(self):
+    # The behaviour these once asserted as substrings -- the sweep, the re-raise, the
-        # On a staging failure, sync.py's `snapshot, local_path = await
+    # teardown in the finally -- is now asserted STRUCTURALLY, against the parsed
-        # create_snapshot(...)` never completes, so its local `snapshot` stays
+    # block: see TestTheStagingFailurePathReallyReRaises and
-        # None and its finally deletes nothing. We must sweep it ourselves.
+    # TestTheSyncBlockAlwaysTearsDown. As substring checks they were satisfied by
-        block = extract_blocks()["SNAPSHOT_ASYNC"]
+    # COMMENTS ("a cleanup that raises...", "cleanup_task gets logger=None"), so
-        assert "except Exception:" in block
+    # deleting the actual `raise` and the actual cleanup call both left the suite
-        assert "delete_snapshot_tree" in block
+    # green -- reinstating a silently-empty backup and ~250 orphans per run.
        assert "raise" in block
-    def test_sync_block_cleans_up_on_every_path(self):
+    def test_the_snapshot_block_still_owns_the_snapshot_when_not_staging(self):
-        block = extract_blocks()["SYNC_ASYNC"]
+        # The TrueNAS 26 zvol/legacy orphan: stock decides `recursive` by its own rule
-        assert "finally:" in block
+        # (path == mountpoint) and deletes only the parent, so we must record the
-        assert "cleanup_task" in block
+        # snapshot even on the path where we stage nothing.
        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
            stage = functions(tree_of(name), "_tc_stage")[0]
            assert calls_to(stage, "_tc_nested.own_snapshot"), (
                f"{name} hands an unstaged snapshot back to stock, whose delete is "
                f"non-recursive -- every zvol/legacy child is orphaned, every run"
            )
    def test_the_staging_plan_is_enumerated_from_ZFS(self):
        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
            stage = functions(tree_of(name), "_tc_stage")[0]
            assert calls_to(stage, "_tc_nested.query_filesystems"), (
                "the staging plan must come from query_filesystems() (which reads ZFS "
                "unfiltered); middleware's query hides ix-apps/*, .system/*, .ix-virt/*"
            )
            assert not calls_to(stage, "middleware.call_sync"), (
                "the block calls middleware directly again -- its dataset/snapshot "
                "queries are FILTERED and silently omit 84 of 270 datasets"
            )
    def test_the_vendored_helper_is_used_not_the_host_module(self):
        # TrueNAS 26 DELETED get_dataset_recursive from plugins/cloud/snapshot.py, so
        # calling it out of the host module's namespace is a NameError there.
        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
            stage = functions(tree_of(name), "_tc_stage")[0]
            assert calls_to(stage, "_tc_nested.get_dataset_recursive"), (
                "must call OUR vendored copy: TrueNAS 26 deleted the host's"
            )
    def test_datasets_are_enumerated_AFTER_the_snapshot(self):
        # A dataset created between the listing and the snapshot would be captured by
        # the recursive snapshot but missing from the staging plan -- silently omitted.
        # Read afterwards, it instead trips plan_staging's probe and fails loudly.
        for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
            src = extract_blocks()[name]
            code = "\n".join(
                ln for ln in src.splitlines() if not ln.lstrip().startswith("#")
            )
            # _tc_stage receives `snapshot` as a parameter -- i.e. it is taken by the
            # caller, before any of this runs. If the enumeration ever moves ahead of
            # create_snapshot it can only do so by leaving _tc_stage.
            assert "def _tc_stage(middleware, path, name, snapshot, snap_path)" in code
            assert "query_filesystems" in code
 def test_crud_block_is_scoped_to_cloud_backup():
@@ -438,8 +479,275 @@ class TestOnlyOurOwnTasksAreTouched:
        # The point is to add NO new failure mode to a CloudSync task. If any
        # middleware call happened before the bail-out, we would already have broken
        # the thing we are trying not to touch.
        #
        # Checked against whichever interactions the block ACTUALLY contains, not a
        # fixed list: the dataset query moved behind `_tc_nested.query_filesystems()`
        # when it switched to the public pool.* API, and a hardcoded
        # `middleware.call_sync(` simply stopped being found -- a test that silently
        # stops testing is worse than no test.
        block = extract_blocks()[name]
        gate = block.index('if not name.startswith("cloud_backup"):')
-        for call in ("middleware.call_sync(", "_tc_nested.stage_nested(",
+
-                     "_tc_nested.delete_snapshot_tree("):
+        interactions = [
            "middleware.call_sync(",
            "_tc_nested.query_filesystems(",
            "_tc_nested.stage_nested(",
            "_tc_nested.delete_snapshot_tree(",
        ]
        present = [c for c in interactions if c in block]
        assert present, "found no middleware interaction at all -- the test is vacuous"
        for call in present:
            assert gate < block.index(call), f"{call} runs before the cloud_backup gate"
 # ── structural assertions ────────────────────────────────────────────────────
 #
 # `assert "raise" in block` was TRUE because a COMMENT in the block says "a cleanup
 # that raises would replace the original exception". `assert "cleanup_task" in block`
 # was TRUE because a comment says "cleanup_task gets logger=None". Deleting the actual
 # `raise`, and deleting the actual cleanup call from the `finally`, both left the suite
 # green -- while reinstating, respectively, a silently-empty backup and ~250 orphaned
 # snapshots per run.
 #
 # A test that a comment can satisfy is not a test. These parse the block and assert on
 # the CODE.
 def tree_of(name):
    return ast.parse(textwrap.dedent(extract_blocks()[name]))
 def functions(tree, name):
    return [
        n for n in ast.walk(tree)
        if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef) and n.name == name
    ]
 def calls_to(node, dotted):
    """Every Call in `node` whose callee renders as `dotted` (e.g. a.b.c)."""
    out = []
    for n in ast.walk(node):
        if isinstance(n, ast.Call):
            try:
                if ast.unparse(n.func) == dotted:
                    out.append(n)
            except Exception:                                    # noqa: BLE001
                pass
    return out
 class TestTheStagingFailurePathReallyReRaises:
    """If staging fails and we swallow it, restic backs up the UN-STAGED path.
    That is the silently-empty backup this entire module exists to prevent: stock
    points the tool at the parent's `.zfs/snapshot/`, where child datasets are
    invisible. The exception MUST propagate.
    """
    @pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
    def test_the_handler_sweeps_the_snapshot_and_re_raises(self, name):
        stage = functions(tree_of(name), "_tc_stage")
        assert stage, "_tc_stage is gone"
        handlers = [
            h for t in ast.walk(stage[0]) if isinstance(t, ast.Try)
            for h in t.handlers
        ]
        assert handlers, "the staging failure handler is gone"
        sweeps = any(calls_to(h, "_tc_nested.delete_snapshot_tree") for h in handlers)
        assert sweeps, (
            "a staging failure no longer sweeps the snapshot. sync.py's `snapshot` "
            "local stays None, so ITS finally deletes nothing -- the whole tree leaks "
            "on every failed run."
        )
        # A bare `raise` directly in the handler body -- not one nested inside the
        # defensive try/except that wraps the sweep.
        reraises = any(
            any(isinstance(s, ast.Raise) and s.exc is None for s in h.body)
            for h in handlers
        )
        assert reraises, (
            "the staging failure is SWALLOWED. restic then runs against the un-staged "
            "path and uploads a near-empty tree, reporting SUCCESS."
        )
 class TestTheSyncBlockAlwaysTearsDown:
    """The teardown is what unmounts the staging tree and sweeps the snapshot.
    It must run on EVERY exit from restic_backup -- success, failure, or exception --
    or the bind mounts pin the snapshot and the tree is orphaned.
    """
    @pytest.mark.parametrize("name", ["SYNC_ASYNC", "SYNC_SYNC"])
    def test_cleanup_runs_in_a_finally(self, name):
        fns = functions(tree_of(name), "restic_backup")
        assert fns, "the restic_backup wrapper is gone"
        tries = [t for t in ast.walk(fns[0]) if isinstance(t, ast.Try) and t.finalbody]
        assert tries, "restic_backup no longer has a try/finally"
        cleans = any(
            "cleanup_task" in ast.unparse(stmt)
            for t in tries for stmt in t.finalbody
        )
        assert cleans, (
            "cleanup_task is not called in the finally. The staging tree is never torn "
            "down, its bind mounts pin the snapshot, and ~250 snapshots leak per run."
        )
 class TestTheBlockingWorkNeverRunsOnTheEventLoop:
    """`zfs list` and `call_sync` are BLOCKING. On <=25.10 these blocks are async.
    Running them directly on middlewared's event loop stalls the whole daemon.
    """
    @pytest.mark.parametrize("name,fn", [
        ("SNAPSHOT_ASYNC", "create_snapshot"),
        ("SYNC_ASYNC", "restic_backup"),
    ])
    def test_the_async_flavour_hops_to_a_thread(self, name, fn):
        fns = functions(tree_of(name), fn)
        assert fns and isinstance(fns[0], ast.AsyncFunctionDef)
        assert calls_to(fns[0], "middleware.run_in_thread"), (
            f"{name}.{fn} does the blocking work on the asyncio event loop"
        )
    @pytest.mark.parametrize("name,fn", [
        ("SNAPSHOT_SYNC", "create_snapshot"),
        ("SYNC_SYNC", "restic_backup"),
    ])
    def test_the_sync_flavour_does_not(self, name, fn):
        # On 26 stock already runs this in the thread pool; hopping again would be
        # wrong (and there is no event loop to protect).
        fns = functions(tree_of(name), fn)
        assert fns and isinstance(fns[0], ast.FunctionDef)
        assert not calls_to(fns[0], "middleware.run_in_thread")
 def test_the_flavour_mapping_is_not_inverted():
    # `_snapshot_block = SNAPSHOT_ASYNC if _flavour else SNAPSHOT_SYNC` -- inverting it
    # injects an async wrapper on 26 (a coroutine gets unpacked as a tuple) or a sync
    # one on 25.10 (the event loop blocks). Every nested backup breaks, both ways.
    with open(APPLY_SH, encoding="utf-8") as fh:
        code = " ".join(
            ln for ln in fh.read().splitlines() if not ln.lstrip().startswith("#")
        )
    code = re.sub(r"\s+", " ", code)          # the assignments are space-aligned
    for block in ("SNAPSHOT", "CRUD", "SYNC"):
        assert f"{block}_ASYNC if _flavour else {block}_SYNC" in code, (
            f"the {block} flavour mapping is missing or inverted: _flavour is True for "
            f"an ASYNC middleware, so it must select {block}_ASYNC"
        )
 # ── the compat preflight ─────────────────────────────────────────────────────
 #
 # This is the guard that stands between a broken middleware and a live NAS: at every
 # boot, apply.sh checks the patch's assumptions against the middlewared actually
 # installed, and REFUSES to apply a module whose assumptions no longer hold.
 #
 # It had no test. An audit turned it into a no-op eight different ways -- `verdict()`
 # always returning 'ok', the broken branch never firing, the kill switch never honoured
 # -- and the suite stayed green every time. The most consequential safety net in the
 # project was unguarded.
 def preflight_heredoc():
    """The preflight's Python, lifted out of apply.sh and made runnable.
    Extracted, not reimplemented: a reimplementation would happily pass while the
    SHIPPED preflight stayed broken, which is exactly the failure being guarded.
    """
    with open(APPLY_SH, encoding="utf-8") as fh:
        sh = fh.read()
    # Line-based: the compat heredoc opens with `<<'PYEOF'` on the _tc_compat line and
    # closes at the next bare PYEOF. (A regex that matched `<< 'PYEOF'` silently found
    # the OTHER heredoc and ran a different script entirely.)
    lines = sh.splitlines()
    start = next(
        i for i, ln in enumerate(lines)
        if ln.startswith("_tc_compat=$(") and "<<'PYEOF'" in ln
    )
    end = next(i for i in range(start + 1, len(lines)) if lines[i].strip() == "PYEOF")
    m = "\n".join(lines[start + 1:end])
    assert m, "could not find the compat preflight heredoc in apply.sh"
    return m
 def run_preflight(result, tmp_path):
    """Run the SHIPPED preflight against a fake compat.check_tree result.
    The heredoc does `import sys`, so a fake `sys` in the namespace is immediately
    rebound to the real module -- drive the real one instead.
    """
    import contextlib
    import io
    import sys
    import types
    src = preflight_heredoc()
    fake = types.ModuleType("compat")
    fake.check_tree = lambda _mw: result
    saved_mod = sys.modules.get("compat")
    saved_argv = sys.argv
    sys.modules["compat"] = fake
    sys.argv = ["x", "/patch", "/mw", str(tmp_path / "compat.json")]
    buf = io.StringIO()
    try:
        with contextlib.redirect_stdout(buf):
            exec(compile(src, "apply.sh:preflight", "exec"), {"__name__": "__main__"})  # noqa: S102
    except SystemExit:
        pass
    finally:
        sys.argv = saved_argv
        if saved_mod is not None:
            sys.modules["compat"] = saved_mod
        else:
            sys.modules.pop("compat", None)
    return buf.getvalue().splitlines()
 def _mod(ok=True, native=False, unknown=False, problems=()):
    return {"ok": ok, "native": native, "unknown": unknown, "problems": list(problems)}
 class TestTheBootPreflightRefusesABrokenMiddleware:
    def test_a_healthy_tree_is_ok(self, tmp_path):
        out = run_preflight({"providers": _mod(), "nested": _mod()}, tmp_path)
        assert out[:2] == ["ok", "ok"]
    def test_a_broken_module_is_reported_broken(self, tmp_path):
        out = run_preflight({
            "providers": _mod(),
            "nested": _mod(ok=False, problems=[
                {"id": "x", "detail": "gone", "why": "orphans every run"},
            ]),
        }, tmp_path)
        assert "broken" in out, (
            "the preflight did not report a module whose assumptions FAILED. It would "
            "be injected into a middleware it does not fit -- broken backups, "
            "discovered at restore time."
        )
    def test_a_module_that_went_NATIVE_is_also_not_applied(self, tmp_path):
        # 'native' answers "do we still need it?", 'ok' answers "is it safe to inject?".
        # Applying a module TrueNAS now implements itself is not safe either.
        out = run_preflight({
            "providers": _mod(),
            "nested": _mod(ok=False, native=True),
        }, tmp_path)
        assert "broken" in out
    def test_an_UNKNOWN_verdict_is_not_reported_as_broken(self, tmp_path):
        # A network error or an unreadable file is not iX deleting our symbols. Calling
        # it broken would switch a working module off on a healthy box.
        out = run_preflight({
            "providers": _mod(unknown=True),
            "nested": _mod(unknown=True),
        }, tmp_path)
        assert "broken" not in out
@@ -15,6 +15,7 @@ dangerous thing this file can say -- it means "TrueNAS does this now, retire the
 module" -- and it rests on nothing more than a substring match.
 """
 import json
 import os
 import sys
@@ -23,6 +24,7 @@ import pytest
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
 import compat  # noqa: E402
 import compat_publish  # noqa: E402
 from compat import (  # noqa: E402
    NESTED,
    PROVIDERS,
@@ -51,24 +53,44 @@ GOOD = {
        "async def restic_backup(middleware, job, cloud_backup, dry_run=False, "
        "rate_limit=None):\n    pass\n"
    ),
-    # The middlewared METHODS the injected code calls. TrueNAS 26 deleted both of
+    # The middlewared METHODS the injected code calls. These now go through the
-    # these files, taking zfs.dataset.query / zfs.snapshot.query / zfs.snapshot.delete
+    # PUBLIC pool.* API: TrueNAS 26 deleted plugins/zfs_/ outright, taking the whole
-    # with them -- see TestMiddlewareMethodsWeCall.
+    # private zfs.* service with it -- see TestMiddlewareMethodsWeCall.
-    "plugins/zfs_/dataset.py": (
+    #
-        "class ZFSDataset(CRUDService):\n"
+    # This default tree is a MODERN box (25.10/26): it has pool.snapshot and no
    # zfs.snapshot. The older shape is built explicitly where it is tested.
    # Not a plugin: a method on the middleware OBJECT. `snapshot_service()` resolves
    # the snapshot namespace through it, so if it vanishes the module cannot sweep the
    # snapshot it just took.
    "utils/plugins.py": (
        "class LoadPluginsMixin:\n"
        "    def get_service(self, name):\n        pass\n"
    ),
    "plugins/pool_/dataset.py": (
        "class PoolDatasetService(CRUDService):\n"
        "    class Config:\n"
-        "        namespace = 'zfs.dataset'\n"
+        "        namespace = 'pool.dataset'\n"
        "    def query(self, filters, options):\n        pass\n"
    ),
-    "plugins/zfs_/snapshot.py": (
+    "plugins/pool_/snapshot.py": (
-        "class ZFSSnapshot(CRUDService):\n"
+        "class PoolSnapshotService(CRUDService):\n"
        "    class Config:\n"
-        "        namespace = 'zfs.snapshot'\n"
+        "        namespace = 'pool.snapshot'\n"
        "    def query(self, filters, options):\n        pass\n"
        "    def delete(self, id_, options={}):\n        pass\n"
    ),
 }
 #: A 24.10/25.04 box: `pool.snapshot` does not exist yet and the snapshot CRUD
 #: service still answers to the (then-public) `zfs.snapshot`.
 ZFS_ERA_SNAPSHOT = (
    "class ZFSSnapshot(CRUDService):\n"
    "    class Config:\n"
    "        namespace = 'zfs.snapshot'\n"
    "    def query(self, filters, options):\n        pass\n"
    "    def do_delete(self, id_, options={}):\n        pass\n"
 )
 def loader(files):
    def load(path):
@@ -284,9 +306,23 @@ class TestAsyncFlavour:
        broken["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429")
        assert compat.async_flavour(loader(broken)) is None
-    def test_the_real_truenas_versions(self):
+    def test_it_reads_STOCK_source_not_our_own_injected_block(self):
-        # Pinning the actual fact this whole port exists for.
+        # This was a byte-identical copy of test_async_middleware_is_detected under a
-        assert compat.async_flavour(loader(GOOD)) is True
+        # name that promised more. The fact worth pinning: apply.sh re-runs on an
        # ALREADY-PATCHED overlay, so the probe must cut our block off first -- our own
        # SNAPSHOT_SYNC wrapper is a plain `def create_snapshot`, and reading it would
        # report a 25.10 box as synchronous and inject the wrong flavour.
        patched = dict(GOOD)
        patched["plugins/cloud/snapshot.py"] = (
            GOOD["plugins/cloud/snapshot.py"]
            + "\n# TRUECLOUD_PATCH\n"
            + 'def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
        )
        assert compat.async_flavour(loader(patched)) is True, (
            "the flavour probe read our own injected block and concluded the box is "
            "synchronous -- it would then inject a sync wrapper into an async "
            "middleware, and every nested backup would break"
        )
 class TestMiddlewareMethodsWeCall:
@@ -303,52 +339,86 @@ class TestMiddlewareMethodsWeCall:
    per descendant dataset (250 on a real pool) on every run, forever.
    """
-    ZFS_SNAPSHOT = (
+    POOL_SNAPSHOT = GOOD["plugins/pool_/snapshot.py"]
        "class ZFSSnapshot(CRUDService):\n"
        "    class Config:\n"
        "        namespace = 'zfs.snapshot'\n"
        "    def query(self, filters, options):\n        pass\n"
        "    def delete(self, id_, options={}):\n        pass\n"
    )
    ZFS_DATASET = (
        "class ZFSDataset(CRUDService):\n"
        "    class Config:\n"
        "        namespace = 'zfs.dataset'\n"
        "    def query(self, filters, options):\n        pass\n"
    )
    def _tree(self, **over):
        files = dict(GOOD)
        files["plugins/zfs_/snapshot.py"] = self.ZFS_SNAPSHOT
        files["plugins/zfs_/dataset.py"] = self.ZFS_DATASET
        files.update(over)
        return files
    #: A 26 box: pool.snapshot only.
    def _modern(self, **over):
        return self._tree(**over)
    #: A 24.10/25.04 box: zfs.snapshot only -- plugins/pool_/snapshot.py does not
    #: exist yet.
    def _zfs_era(self, **over):
        return self._tree(**{
            "plugins/pool_/snapshot.py": None,
            "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
            **over,
        })
    def test_present_methods_are_ok(self):
-        r = check_files(self._tree())
+        r = check_files(self._modern())
        assert r[NESTED]["ok"], r[NESTED]["problems"]
-    def test_a_deleted_plugin_file_is_broken(self):
+    def test_the_OLD_zfs_era_snapshot_service_also_satisfies_the_call(self):
-        # Literally TrueNAS 26: plugins/zfs_/snapshot.py does not exist.
+        # 24.10 and 25.04 have no `pool.snapshot` at all -- the CRUD service is the
-        r = check_files(self._tree(**{"plugins/zfs_/snapshot.py": None}))
+        # then-public `zfs.snapshot`. Pinning only the modern spelling marked both of
        # those releases BROKEN and would have switched nested snapshots OFF on boxes
        # where they work perfectly. The runtime picks the same way; see
        # pick_snapshot_service().
        r = check_files(self._zfs_era())
        assert r[NESTED]["ok"], r[NESTED]["problems"]
    def test_it_is_broken_only_when_NEITHER_namespace_exists(self):
        # The real failure: middleware drops the last spelling we know how to call.
        r = check_files(self._tree(**{
            "plugins/pool_/snapshot.py": None,
            "plugins/zfs_/snapshot.py": None,
        }))
        assert is_broken(r[NESTED])
        details = " ".join(p["detail"] for p in r[NESTED]["problems"])
-        assert "zfs.snapshot.delete" in details
+        assert "pool.snapshot.delete" in details
        assert "zfs.snapshot.delete" in details, (
            "the report must say BOTH spellings were tried, or whoever reads it will "
            "think we simply never looked for the one their box has"
        )
    def test_we_do_NOT_depend_on_a_middleware_dataset_query_at_all(self):
        # iX could delete plugins/pool_/dataset.py tomorrow and the patch would not
        # care, because the staging plan is enumerated from ZFS, not from middleware.
        #
        # That is deliberate, and it was expensive to learn. `pool.dataset.query`
        # exists and is correctly shaped -- and it LIES: it applies a visibility
        # policy that hides ix-apps/*, .system/* and .ix-virt/* (84 of 270 datasets
        # on the real pool, including live app data). No source check could ever
        # have caught that; only running it could. So there is no assumption here
        # left to break.
        r = check_files(self._modern(**{"plugins/pool_/dataset.py": None}))
        assert r[NESTED]["ok"], r[NESTED]["problems"]
        ids = {c.id for c in compat.MIDDLEWARE_CALLS}
        assert not any("dataset" in i or "query" in i for i in ids), (
            "a dataset/snapshot QUERY assumption crept back into the manifest -- "
            "middleware's queries are filtered; enumerate from ZFS"
        )
    def test_a_renamed_namespace_is_broken(self):
        r = check_files(self._tree(**{
-            "plugins/zfs_/snapshot.py": self.ZFS_SNAPSHOT.replace(
+            "plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace(
-                "'zfs.snapshot'", "'zfs.resource.snapshot'"),
+                "'pool.snapshot'", "'zfs.resource.snapshot'"),
            "plugins/zfs_/snapshot.py": None,
        }))
        assert is_broken(r[NESTED])
    def test_the_CRUDService_do_prefix_is_accepted(self):
-        # 24.10 and 25.04 declare `do_delete`; 25.10 renamed it to `delete`. BOTH
+        # A CRUDService exposes `delete` from a method NAMED `do_delete`. Both
-        # answer to zfs.snapshot.delete. Accepting only the literal name reported the
+        # spellings are live across the matrix. Accepting only the literal name
-        # two older releases as broken -- a false BROKEN that would have switched off
+        # reported working releases as broken.
-        # nested snapshots on boxes where they work perfectly.
+        r = check_files(self._modern(**{
-        r = check_files(self._tree(**{
+            "plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace(
            "plugins/zfs_/snapshot.py": self.ZFS_SNAPSHOT.replace(
                "def delete(", "def do_delete("),
        }))
        assert r[NESTED]["ok"], r[NESTED]["problems"]
@@ -356,6 +426,383 @@ class TestMiddlewareMethodsWeCall:
    def test_the_snapshot_delete_reason_names_the_orphan_risk(self):
        # If this ever regresses, whoever reads the bug report must understand that
        # it is not a cosmetic failure.
-        r = check_files(self._tree(**{"plugins/zfs_/snapshot.py": None}))
+        r = check_files(self._tree(**{
            "plugins/pool_/snapshot.py": None,
            "plugins/zfs_/snapshot.py": None,
        }))
        whys = " ".join(p["why"] for p in r[NESTED]["problems"])
        assert "orphan" in whys
 class TestTheMethodCheckIsNotJustANamespaceCheck:
    """compat must verify the METHOD, not merely that the namespace still exists.
    Deleting the method check entirely used to leave all 304 tests green -- so the
    "namespace AND method" claim was unenforced and silently revertible. It is the
    half of the predicate that catches iX gutting a method while keeping its service,
    which they have already done to `pool.snapshot.do_update` on master.
    """
    def test_a_namespace_that_no_longer_defines_delete_is_broken(self):
        gutted = (
            "class PoolSnapshotService(CRUDService):\n"
            "    class Config:\n"
            "        namespace = 'pool.snapshot'\n"
            "    def query(self, filters, options):\n        pass\n"
            # do_delete is GONE -- the service is still registered and still a
            # CRUDService, so it still INHERITS a callable `delete`.
        )
        r = check_files(with_(**{
            "plugins/pool_/snapshot.py": gutted,
            "plugins/zfs_/snapshot.py": None,      # no fallback either
        }))
        assert is_broken(r[NESTED]), (
            "a namespace with no delete must be BROKEN. Checking only that the "
            "namespace exists would apply the patch to a box that cannot sweep its "
            "own snapshots."
        )
    def test_the_alternative_still_saves_it_when_only_the_primary_is_gutted(self):
        gutted = (
            "class PoolSnapshotService(CRUDService):\n"
            "    class Config:\n"
            "        namespace = 'pool.snapshot'\n"
            "    def query(self, filters, options):\n        pass\n"
        )
        r = check_files(with_(**{
            "plugins/pool_/snapshot.py": gutted,
            "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
        }))
        assert r[NESTED]["ok"], r[NESTED]["problems"]
 class TestUnreadableIsNeverOkAndNeverBroken:
    """A rate limit is not a regression, and it is not a clean bill of health either.
    compat runs ~30 unauthenticated GitHub requests per matrix; 429 is a real outcome.
    It also runs at BOOT against the installed tree, where a read can fail with EACCES.
      * treating unreadable as BROKEN repaints the README, files a bug report, and
        makes apply.sh refuse the module on a box where it works.
      * treating it as OK injects a module whose delete may be gone.
    Both mutations used to pass the whole suite.
    """
    def test_both_spellings_unreadable_is_unknown_not_broken(self):
        r = check_files(with_(**{
            "plugins/pool_/snapshot.py": Unreadable("HTTP 429"),
            "plugins/zfs_/snapshot.py": Unreadable("HTTP 429"),
        }))
        assert not is_broken(r[NESTED]), "a 429 is not iX deleting the snapshot service"
        assert r[NESTED]["unknown"]
    def test_an_unreadable_primary_with_a_healthy_alternative_is_ok(self):
        r = check_files(with_(**{
            "plugins/pool_/snapshot.py": Unreadable("HTTP 429"),
            "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
        }))
        assert r[NESTED]["ok"], r[NESTED]["problems"]
        assert not r[NESTED]["unknown"], (
            "one spelling answered the question; the other's 429 is irrelevant"
        )
    def test_a_missing_primary_with_an_unreadable_alternative_is_unknown(self):
        # We cannot tell whether the box is broken. Saying either would be a guess.
        r = check_files(with_(**{
            "plugins/pool_/snapshot.py": None,
            "plugins/zfs_/snapshot.py": Unreadable("HTTP 429"),
        }))
        assert not is_broken(r[NESTED])
        assert r[NESTED]["unknown"]
 class TestGetServiceIsChecked:
    """The runtime resolves the snapshot namespace through `middleware.get_service`.
    It is not a plugin method, so the manifest had no way to express it and never
    checked it. If it vanishes, `_can_delete` reports BOTH namespaces unusable and
    every nested backup fails -- on a box the preflight had declared healthy.
    """
    def test_a_middleware_without_get_service_is_broken(self):
        r = check_files(with_(**{"utils/plugins.py": None}))
        assert is_broken(r[NESTED])
        details = " ".join(p["detail"] for p in r[NESTED]["problems"])
        assert "get_service" in details
 class TestATransientNetworkBlipDoesNotWakeAnybody:
    """The fingerprint must digest what iX BROKE, not what GitHub failed to serve.
    `unknown` problems (a 429 on one of ~30 unauthenticated fetches, an EACCES at boot)
    used to be folded into an already-broken module's problem list, so one blip flipped
    the fingerprint, `compat_publish` rewrote the issue body, and the next clean run
    rewrote it back. Daily churn is what teaches people to ignore the bot -- which is
    the whole thing this fingerprint exists to prevent.
    """
    def _rows(self, files):
        return [{"ref": "master", "modules": check_files(files)}]
    def test_an_unreadable_file_does_not_change_the_fingerprint_of_a_broken_ref(self):
        # The blip must land in the SAME module that is broken. Put it in `providers`
        # (which is healthy) and `fingerprint()` skips the whole module via
        # `is_broken(m)` -- so the `state` filter under test never runs and the test
        # passes no matter what the code does. `nested` is the broken one here, so the
        # unreadable file goes in `nested` too.
        broken = with_(**{
            "plugins/cloud/snapshot.py":
                "async def create_snapshot(name, path, middleware):\n    return 1, 2\n",
        })
        clean = compat.fingerprint(self._rows(broken))
        blipped = dict(broken)
        blipped["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429")   # nested
        assert compat.fingerprint(self._rows(blipped)) == clean, (
            "a rate-limited fetch changed the fingerprint, so the bot rewrites the "
            "issue body and then rewrites it back tomorrow"
        )
    def test_a_REAL_new_finding_still_changes_it(self):
        # ...and the anti-noise measure must not have made it deaf.
        broken = with_(**{
            "plugins/cloud/snapshot.py":
                "async def create_snapshot(name, path, middleware):\n    return 1, 2\n",
        })
        worse = dict(broken)
        worse["plugins/cloud_backup/restic.py"] = (
            "class ResticConfig:\n    cmd: list\n\n"
            "def get_restic_config(entry, credentials):\n    pass\n"
        )
        assert compat.fingerprint(self._rows(worse)) != compat.fingerprint(self._rows(broken))
 class TestTheBotFindsItsOwnIssueOnBOTHForges:
    """`find_issue` decides "have I already filed this?" -- and it ran on two forges.
    It used to skip pull requests with `"pull_request" not in i`. GitHub omits that key
    on a plain issue; **Gitea sends it as `null`**. So on Gitea every issue looked like
    a PR, the match list was always empty, and the bot took the "nothing filed yet"
    branch on EVERY run: nine duplicate copies of the same report on the canonical
    forge, four of them filed after the commit that was supposed to stop exactly this.
    It is the same failure the spam fix was written to prevent, moved from comments to
    issues -- and it survived because `find_issue` was the one function here with no
    test. So the payload shapes are pinned, per forge, by hand.
    """
    TITLE = compat_publish.TITLE
    def _find(self, monkeypatch, payload):
        monkeypatch.setattr(compat_publish, "_call", lambda *a, **k: payload)
        return compat_publish.find_issue("https://forge/api", "tok", self.TITLE)
    def test_gitea_sends_pull_request_as_null_and_the_issue_is_still_found(self, monkeypatch):
        found = self._find(monkeypatch, [
            {"number": 7, "title": self.TITLE, "state": "open", "pull_request": None},
            {"number": 1, "title": self.TITLE, "state": "open", "pull_request": None},
        ])
        assert found is not None, (
            "find_issue missed a Gitea issue, so the bot files a NEW duplicate report "
            "every run -- which is how nine of them piled up"
        )
        assert found["number"] == 1, "lowest-numbered wins"
    def test_github_omits_the_key_entirely_and_the_issue_is_still_found(self, monkeypatch):
        found = self._find(monkeypatch, [
            {"number": 2, "title": self.TITLE, "state": "open"},
        ])
        assert found is not None and found["number"] == 2
    def test_a_real_PR_with_the_same_title_is_still_skipped_on_both(self, monkeypatch):
        # The reason the filter exists at all: both forges list PRs on /issues, and
        # commenting on a PR instead of the bug report would be worse than useless.
        assert self._find(monkeypatch, [
            {"number": 3, "title": self.TITLE, "state": "open",           # Gitea PR
             "pull_request": {"merged": False}},
            {"number": 4, "title": self.TITLE, "state": "open",           # GitHub PR
             "pull_request": {"url": "https://api.github.com/..."}},
        ]) is None
    def test_an_unrelated_issue_is_not_mistaken_for_the_report(self, monkeypatch):
        assert self._find(monkeypatch, [
            {"number": 1, "title": "TypeError when create B2 backup on Electric Eel",
             "state": "closed", "pull_request": None},
        ]) is None
 class TestTheNextMaintenanceReleaseIsChecked:
    """`release/25.10.5` fell through every sieve, and it is the one that reaches users.
    Shipped versions come from `TS-*` TAGS; unreleased ones come from `release/*`
    BRANCHES that carry `-BETA`/`-RC`. A branched-but-untagged MAINTENANCE release is
    neither: no tag, and its line (25.10) has already shipped, so the "prereleases of
    a shipped line are history" filter threw it out. It was invisible.
    That is backwards. `release/24.10-RC.2` is history -- nobody can install it. But
    `release/25.10.5` is the FUTURE of a shipped line: it is what a 25.10.4 box gets
    on its next update. A break there ships to real users before the daily check has
    ever looked at it.
    """
    TAGS = ["TS-24.10.2.4", "TS-25.04.2.6", "TS-25.10.4"]
    HEADS = [
        "release/25.10.4.1",
        "release/25.10.5",          # branched, untagged -- the next maintenance release
        "release/24.10-RC.2",       # history: its line shipped long ago
        "release/25.20.2.2",        # iX's typo branch: 25.20 is not a TrueNAS version
        "release/26.0.0-BETA.3",
        "master",
    ]
    def _refs(self, monkeypatch):
        monkeypatch.setattr(
            compat, "_ls_remote",
            lambda remote, what: self.TAGS if what == "--tags" else self.HEADS)
        return compat.discover_refs("origin")
    def test_the_next_maintenance_release_is_checked(self, monkeypatch):
        assert "release/25.10.5" in self._refs(monkeypatch), (
            "the next thing a 25.10.4 box updates to is not checked, so a break in it "
            "reaches users before the bot ever sees it"
        )
    def test_a_superseded_maintenance_branch_is_not(self, monkeypatch):
        # 25.10.4.1 sorts OLDER than the newest tag TS-25.10.4? No -- it is NEWER, and
        # both are on the 25.10 line, so only the newest branch on the line is taken.
        refs = self._refs(monkeypatch)
        assert "release/25.10.4.1" not in refs, "only the newest branch per line"
    def test_the_typo_branch_stays_out(self, monkeypatch):
        # 25.20 has no TS tag, so it is not a release line at all. A typo branch in the
        # matrix reads as a real supported release we are silently broken on.
        assert "release/25.20.2.2" not in self._refs(monkeypatch)
    def test_a_prerelease_of_an_already_shipped_line_stays_out(self, monkeypatch):
        assert "release/24.10-RC.2" not in self._refs(monkeypatch)
    def test_an_untagged_branch_counts_as_UNRELEASED(self, monkeypatch):
        # The exit code keys off this. Calling 25.10.5 "shipped" would fail the build
        # as a live outage on a version nobody is running yet.
        assert compat.is_unreleased("release/25.10.5")
        assert compat.is_unreleased("master")
        assert not compat.is_unreleased("TS-25.10.4")
 class TestMasterIsNotTheNextRelease:
    """A red `master` row used to read as "the version you are about to install".
    On 2026-07-14 master was 27-dev -- every recent commit targeted 27.0.0-BETA.1 --
    while 26 was still in beta on its own branches. So `master BROKEN` meant "iX will
    break us a major release from now", but the matrix said "master _(unreleased)_",
    which any reader takes as the next thing out the door. For a table whose whole job
    is helping somebody decide whether to trust this with their backups, that is a
    false alarm in the worst possible place.
    """
    def _rows(self, refs):
        return [{"ref": r, "unreleased": compat.is_unreleased(r), "modules": {}}
                for r in refs]
    def test_master_is_labelled_with_the_major_AFTER_the_newest_known_one(self):
        rows = self._rows(["TS-25.10.4", "release/26.0.0-BETA.3", "master"])
        assert compat.dev_label(rows) == "27-dev"
    def test_it_rolls_over_on_its_own_when_the_next_beta_branches(self):
        # Derived, not hardcoded: when release/27.0.0-BETA.1 appears, master is 28-dev.
        rows = self._rows(["TS-26.0.0", "release/27.0.0-BETA.1", "master"])
        assert compat.dev_label(rows) == "28-dev"
    def test_the_rendered_matrix_says_dev_not_unreleased(self):
        healthy = check_files(with_())
        rows = [
            {"ref": r, "unreleased": compat.is_unreleased(r), "modules": healthy}
            for r in ("TS-25.10.4", "release/26.0.0-BETA.3", "master")
        ]
        md = compat.render_markdown(rows)
        assert "master _(27-dev)_" in md
        assert "master _(unreleased)_" not in md
        # ...and the ordinary rows are untouched.
        assert "| 25.10.4 |" in md
        assert "| 26.0.0-BETA.3 _(unreleased)_ |" in md
 class TestTheBodyIsTruthAndCommentsAreTheChangelog:
    """An unchanged fingerprint used to freeze the BODY, not just silence the comments.
    Two different questions were sharing one answer. "Have the findings changed?" gates
    COMMENTS -- they notify, and a daily "still broken, same as yesterday" is what
    teaches people to ignore the one that finally matters. But "is the body still
    true?" gates the BODY, and editing a body notifies nobody, so keeping it honest
    costs nothing.
    Conflated, an unchanged fingerprint meant the report could never be corrected --
    and the fingerprint deliberately ignores everything that moves on its own, which
    includes how a row is LABELLED. Relabelling master `27-dev` would have reached the
    README and never the issue anybody opens.
    """
    ROWS = [{"ref": "master", "unreleased": True,
             "modules": check_files(with_(**{
                 "plugins/cloud_backup/restic.py":
                     "class ResticConfig:\n    cmd: list\n\n"
                     "def get_restic_config(entry, credentials):\n    pass\n",
             }))}]
    def _run(self, monkeypatch, tmp_path, existing_body):
        calls = []
        def fake(url, token, method="GET", data=None):
            calls.append((method, url, data))
            if url.endswith("/issues?state=all&per_page=100&limit=100"):
                return [{"number": 1, "title": compat_publish.TITLE,
                         "state": "open", "body": existing_body,
                         "pull_request": None}]
            return {"number": 1}
        monkeypatch.setattr(compat_publish, "_call", fake)
        matrix = tmp_path / "m.json"
        matrix.write_text(json.dumps(self.ROWS))
        compat_publish.main([
            "prog", "--api", "https://forge/api", "--token", "t",
            "--matrix", str(matrix)])
        return calls
    def _writes(self, calls):
        patched = [c for c in calls if c[0] == "PATCH"]
        commented = [c for c in calls if c[0] == "POST" and c[1].endswith("/comments")]
        return patched, commented
    def test_identical_body_and_findings_touches_nothing(self, monkeypatch, tmp_path):
        body = compat.render_issue(self.ROWS)
        patched, commented = self._writes(self._run(monkeypatch, tmp_path, body))
        assert not patched and not commented, "a quiet run must be completely silent"
    def test_a_relabel_refreshes_the_body_but_says_NOTHING(self, monkeypatch, tmp_path):
        # Same findings (same fingerprint), different rendering -- the exact shape of
        # the master -> 27-dev relabel.
        stale = compat.render_issue(self.ROWS).replace("27-dev", "unreleased")
        assert compat.extract_fingerprint(stale) == compat.fingerprint(self.ROWS)
        patched, commented = self._writes(self._run(monkeypatch, tmp_path, stale))
        assert patched, "the body was left stale, so the issue keeps telling lies"
        assert "27-dev" in patched[0][2]["body"]
        assert not commented, (
            "a rendering change is not news -- commenting on it is how the bot gets "
            "muted before the next real finding"
        )
    def test_a_REAL_findings_change_still_comments(self, monkeypatch, tmp_path):
        # ...and the fix must not have made it mute.
        stale = compat.render_issue(self.ROWS).replace(
            compat.fingerprint(self.ROWS), "0" * 16)
        patched, commented = self._writes(self._run(monkeypatch, tmp_path, stale))
        assert patched and commented, "a genuine change must still notify"
    def test_a_body_differing_only_by_CRLF_is_not_rewritten(self, monkeypatch, tmp_path):
        # Forges round-trip line endings. Without normalising, every run would rewrite
        # the body -- silent, but it churns updated_at and looks freshly touched daily.
        body = compat.render_issue(self.ROWS).replace("\n", "\r\n")
        patched, _ = self._writes(self._run(monkeypatch, tmp_path, body))
        assert not patched
@@ -0,0 +1,210 @@
 """Behavioural tests for the "installed but NOT loaded" alert.
 apply.log can only report what was written to disk. Whether the middlewared that
 restarted afterwards actually imported those files is a different fact, and when
 the two disagree nothing else notices: on 2026-08-19 every B2 backup failed for
 nineteen hours while the log said OK. This alert is the only thing that closes
 that gap, so it is tested against real objects rather than by reading source.
 The middlewared package does not exist off-box, so the modules the alert source
 imports are stubbed here.
 """
 import importlib.util
 import json
 import os
 import sys
 import types
 import pytest
 ALERT_SRC = os.path.join(os.path.dirname(__file__), "..", "patch", "alert_source.py")
 class _StubAlertClass:
    pass
 class _StubThreadedAlertSource:
    pass
 class _StubAlert:
    def __init__(self, klass, args=None, key=None):
        self.klass = klass
        self.args = args
        self.key = key
 def _module(name):
    mod = types.ModuleType(name)
    sys.modules[name] = mod
    return mod
@pytest.fixture
 def alert_source(monkeypatch, tmp_path):
    """Load patch/alert_source.py against stubbed middlewared modules."""
    for name in list(sys.modules):
        if name == "middlewared" or name.startswith("middlewared."):
            monkeypatch.delitem(sys.modules, name, raising=False)
    _module("middlewared")
    _module("middlewared.alert")
    base = _module("middlewared.alert.base")
    base.Alert = _StubAlert
    base.AlertClass = _StubAlertClass
    base.ThreadedAlertSource = _StubThreadedAlertSource
    base.AlertCategory = types.SimpleNamespace(SYSTEM="SYSTEM")
    base.AlertLevel = types.SimpleNamespace(
        INFO="INFO", WARNING="WARNING", CRITICAL="CRITICAL"
    )
    schedule = _module("middlewared.alert.schedule")
    schedule.IntervalSchedule = lambda delta: ("interval", delta)
    spec = importlib.util.spec_from_file_location("_tc_alert_source", ALERT_SRC)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    mod.PATCH_DIR = str(tmp_path)
    return mod
 def _write_status(tmp_path, providers_active=True):
    payload = {
        "patched_at": "2026-08-26T00:00:00Z",
        "patches": {
            "providers": {"ok": True, "active": providers_active, "detail": "x"},
            "nested_snapshots": {"ok": True, "active": True, "detail": "x"},
        },
    }
    (tmp_path / "hook_status.json").write_text(json.dumps(payload))
 def _install_provider_modules(monkeypatch, *, restic_patched, b2_patched):
    """Stub the two modules the alert inspects, in the requested state."""
    plugins = _module("middlewared.plugins")
    _module("middlewared.plugins.cloud_backup")
    restic = _module("middlewared.plugins.cloud_backup.restic")
    def get_restic_config(task):
        return None
    if restic_patched:
        get_restic_config._truecloud_patched = True
    restic.get_restic_config = get_restic_config
    rclone_base = _module("middlewared.rclone.base")
    _module("middlewared.rclone")
    _module("middlewared.rclone.remote")
    b2_mod = _module("middlewared.rclone.remote.b2")
    class BaseRcloneRemote:
        def get_restic_config(self, task):
            raise NotImplementedError
    class B2RcloneRemote(BaseRcloneRemote):
        pass
    if b2_patched:
        B2RcloneRemote.get_restic_config = staticmethod(lambda task: ("url", {}))
    rclone_base.BaseRcloneRemote = BaseRcloneRemote
    b2_mod.B2RcloneRemote = B2RcloneRemote
    b2_mod.BaseRcloneRemote = BaseRcloneRemote
    plugins.__path__ = []
    for name in (
        "middlewared.plugins",
        "middlewared.plugins.cloud_backup",
        "middlewared.plugins.cloud_backup.restic",
        "middlewared.rclone",
        "middlewared.rclone.base",
        "middlewared.rclone.remote",
        "middlewared.rclone.remote.b2",
    ):
        monkeypatch.setitem(sys.modules, name, sys.modules[name])
 def _source(alert_source):
    cls = alert_source.TrueCloudPatchNotLoadedAlertSource
    return cls.__new__(cls)
 def test_no_alert_when_patch_is_loaded(alert_source, monkeypatch, tmp_path):
    _write_status(tmp_path)
    _install_provider_modules(monkeypatch, restic_patched=True, b2_patched=True)
    assert _source(alert_source)._check() is None
 def test_alert_when_middlewared_loaded_stock_modules(alert_source, monkeypatch, tmp_path):
    """The exact 2026-08-19 state: patched on disk, stock in the process."""
    _write_status(tmp_path)
    _install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
    alert = _source(alert_source)._check()
    assert alert is not None
    assert alert.klass is alert_source.TrueCloudPatchNotLoadedAlertClass
 def test_alert_when_only_b2_half_is_missing(alert_source, monkeypatch, tmp_path):
    # b2.py is the half that supplies B2's get_restic_config. restic.py alone
    # being patched still means every B2 task raises NotImplementedError.
    _write_status(tmp_path)
    _install_provider_modules(monkeypatch, restic_patched=True, b2_patched=False)
    assert _source(alert_source)._check() is not None
 def test_alert_when_only_restic_half_is_missing(alert_source, monkeypatch, tmp_path):
    _write_status(tmp_path)
    _install_provider_modules(monkeypatch, restic_patched=False, b2_patched=True)
    assert _source(alert_source)._check() is not None
 def test_silent_when_the_kill_switch_is_set(alert_source, monkeypatch, tmp_path):
    # The operator turned the patch off on purpose; stock is the intended state.
    _write_status(tmp_path)
    (tmp_path / "disabled").write_text("")
    _install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
    assert _source(alert_source)._check() is None
 def test_silent_when_providers_module_is_retired(alert_source, monkeypatch, tmp_path):
    # TrueNAS went native for B2: not loading our providers patch is correct.
    _write_status(tmp_path, providers_active=False)
    _install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
    assert _source(alert_source)._check() is None
 def test_silent_when_the_patch_was_never_applied_here(alert_source, monkeypatch, tmp_path):
    # No hook_status.json at all -- nothing claims a patch, so nothing is broken.
    _install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
    assert _source(alert_source)._check() is None
 def test_update_alert_silencer_does_not_mute_a_broken_backup_path(
    alert_source, monkeypatch, tmp_path
 ):
    # update_alerts_disabled mutes release notifications. It must not hide the
    # fact that TrueCloud backups are silently running stock.
    _write_status(tmp_path)
    (tmp_path / "update_alerts_disabled").write_text("")
    _install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
    assert _source(alert_source)._check() is not None
 def test_check_sync_never_raises(alert_source, monkeypatch, tmp_path):
    """An alert source that raises is polled forever inside middlewared."""
    _write_status(tmp_path)
    def boom(self):
        raise RuntimeError("provider import exploded")
    monkeypatch.setattr(
        alert_source.TrueCloudPatchNotLoadedAlertSource, "_check", boom, raising=True
    )
    assert _source(alert_source).check_sync() is None
 def test_alert_is_critical_and_names_the_recovery_command(alert_source):
    klass = alert_source.TrueCloudPatchNotLoadedAlertClass
    assert klass.level == "CRITICAL"
    assert "install.sh" in klass.text
@@ -0,0 +1,199 @@
 """The deferred restart must re-apply the patch before it restarts middlewared.
 Patching at PREINIT and restarting minutes later is only sound while the patched
 files are still on the live path when middlewared re-imports them. They may not
 be: the patch lives in an overlay mounted inside /usr, and anything that
 remounts that hierarchy detaches it. On 2026-08-19 a systemd-sysext refresh over
 /usr ran four seconds after apply.sh mounted its overlay; the deferred restart
 then loaded stock modules and every B2 cloud_backup job failed for nineteen
 hours while apply.log reported "OK".
 These tests pin the ordering that makes that non-recoverable failure impossible:
 re-apply, verify, restart, verify again.
 """
 import os
 import re
 import subprocess
 import pytest
 HERE = os.path.dirname(__file__)
 WAIT_RESTART = os.path.join(HERE, "..", "patch", "wait_restart.sh")
 APPLY_SH = os.path.join(HERE, "..", "patch", "apply.sh")
 def wait_restart_source():
    with open(WAIT_RESTART, encoding="utf-8") as fh:
        return fh.read()
 def apply_source():
    with open(APPLY_SH, encoding="utf-8") as fh:
        return fh.read()
 def test_wait_restart_is_executable():
    # apply.sh schedules it as `/bin/bash <script>`, but install.sh ships exec
    # bits and a mode-only diff once blocked update.sh outright (v0.6.0).
    assert os.access(WAIT_RESTART, os.X_OK)
 def test_wait_restart_is_syntactically_valid():
    subprocess.run(["bash", "-n", WAIT_RESTART], check=True)
 def test_reapply_runs_before_the_restart():
    src = wait_restart_source()
    reapply = src.index("TRUECLOUD_REAPPLY=1")
    restart = src.index("systemctl try-restart middlewared")
    assert reapply < restart, "the re-apply pass must precede the restart"
 def test_restart_is_not_exec_so_verification_can_follow():
    # Up to v0.7.0 the script ended in `exec systemctl try-restart middlewared`,
    # which replaces the shell -- nothing could run afterwards. The post-restart
    # verification only exists if the restart is a plain call.
    src = wait_restart_source()
    assert not re.search(r"^\s*exec\s+systemctl", src, re.M)
 def test_middlewared_is_restarted_exactly_once():
    """No restart loop.
    `try-restart` returns at READY; middlewared then brings docker up, and
    docker.configure_nvidia merges the nvidia sysext over /usr right about then
    -- detaching the overlay AFTER the patched modules are already imported. A
    disk check after the restart therefore false-negatives on a healthy system,
    and restarting on that signal would restart a correctly-patched middlewared
    straight back into the same race.
    """
    src = wait_restart_source()
    assert src.count("systemctl try-restart middlewared") == 1
 def test_patch_is_verified_after_the_restart():
    src = wait_restart_source()
    restart = src.index("systemctl try-restart middlewared")
    assert "_patch_visible" in src[restart:], (
        "the script must check what the restart actually loaded"
    )
 def test_verification_reads_the_marker_apply_sh_writes():
    # _patch_visible greps restic.py for TRUECLOUD_PATCH; apply.sh must still be
    # the thing that puts it there, or the check silently always fails.
    assert "TRUECLOUD_PATCH" in wait_restart_source()
    assert "TRUECLOUD_PATCH" in apply_source()
 def test_verification_uses_the_recorded_middlewared_dir():
    # wait_restart.sh must not re-derive site-packages; apply.sh records it.
    assert ".mw_dir" in wait_restart_source()
    assert ".mw_dir" in apply_source()
 def test_apply_sh_records_the_middlewared_dir():
    src = apply_source()
    assert re.search(r'>\s*"\$PATCH_DIR/\.mw_dir"', src), (
        "apply.sh must write the resolved middlewared dir for wait_restart.sh"
    )
 def test_reapply_pass_does_not_schedule_another_restart():
    # wait_restart.sh owns the restart. If the re-apply pass scheduled its own
    # transient unit, each boot would spawn restarts recursively.
    src = apply_source()
    guard = src.index('if [ "${TRUECLOUD_REAPPLY:-0}" = "1" ]; then')
    systemd_run = src.index("systemd-run --no-block")
    assert guard < systemd_run, (
        "the TRUECLOUD_REAPPLY branch must short-circuit before systemd-run"
    )
 def test_shadowed_overlay_is_remounted_not_accepted():
    """A buried overlay must never pass for a healthy one.
    _ensure_writable reaches its mount-table check only when the directory is
    NOT writable -- and a live overlay of ours is always writable. So a
    truecloud mount listed at that point is shadowed, and returning 0 there is
    exactly how a detached overlay used to masquerade as applied.
    """
    src = apply_source()
    start = src.index("_ensure_writable()")
    end = src.index("\n}", start)
    body = src[start:end]
    check = body.index('mount | grep -qF "truecloud-${tag} on ${dir} "')
    following = body[check:]
    # The old code did `return 0` immediately inside this branch.
    branch_end = following.index("fi")
    assert "return 0" not in following[:branch_end]
    assert "umount -l" in following[:branch_end]
 def test_workdir_is_recreated_before_mounting():
    # overlayfs refuses a workdir left behind by a detached mount, so a stale
    # one would turn every re-mount attempt into "overlay mount failed".
    src = apply_source()
    start = src.index("_ensure_writable()")
    end = src.index("\n}", start)
    body = src[start:end]
    assert re.search(r'rm -rf "\$work"', body)
 def test_upperdir_is_preserved_across_remounts():
    # The upperdir holds everything patched earlier this boot; reusing it is
    # what lets a re-mount restore those files instead of re-deriving them.
    src = apply_source()
    start = src.index("_ensure_writable()")
    end = src.index("\n}", start)
    body = src[start:end]
    assert 'rm -rf "$upper"' not in body
@pytest.mark.parametrize("state", ["0", "1", "2"])
 def test_patch_visible_returns_three_distinct_states(state):
    # patched / stock / cannot-tell must stay distinguishable: "cannot tell"
    # has to re-apply rather than assume the patch is fine.
    src = wait_restart_source()
    assert f"return {state}" in src or f") return {state}" in src
 def test_mount_retries_on_a_private_workdir():
    """A lazily-detached overlay can still pin the shared workdir.
    overlayfs refuses a workdir that is in use, so without a retry the re-mount
    this whole fix depends on would fail exactly when it is most needed.
    """
    src = apply_source()
    start = src.index("_ensure_writable()")
    end = src.index("\n}", start)
    body = src[start:end]
    assert body.count("mount -t overlay") == 2, "expected a retry mount"
    assert 'work="/run/truecloud-${tag}-work.$$"' in body
 def test_post_restart_remount_preserves_the_patched_at_stamp():
    """create_task.py verify compares middlewared's start time to patched_at.
    The post-restart re-mount restores the same patch the boot pass applied, so
    letting apply.sh re-stamp would make patched_at newer than the process that
    correctly imported it -- verify would then report FAIL forever on every boot
    where docker's sysext merge detaches the overlay.
    """
    src = wait_restart_source()
    tail = src[src.index("systemctl try-restart middlewared"):]
    assert "hook_status.json" in tail
    save = tail.index("/run/truecloud-hook_status.pre")
    reapply = tail.index("TRUECLOUD_REAPPLY=1")
    restore = tail.rindex("hook_status.json")
    assert save < reapply < restore, "snapshot must bracket the re-apply"
 def test_status_snapshot_is_not_written_into_the_repo():
    """A leftover file in the repo dir leaves the tree dirty, and update.sh
    refuses to run over a dirty tree -- that once made the patch un-updatable."""
    src = wait_restart_source()
    assert "/run/truecloud-hook_status.pre" in src
    assert '"$PATCH_DIR/hook_status.json.pre' not in src
@@ -10,7 +10,8 @@ import re
 import pytest
-WORKFLOWS = os.path.join(os.path.dirname(__file__), "..", ".github", "workflows")
+ROOT = os.path.join(os.path.dirname(__file__), "..")
 WORKFLOWS = os.path.join(ROOT, ".github", "workflows")
 def workflow_files():
@@ -84,11 +85,77 @@ class TestBothForges:
        assert "if: ${{ contains(github.server_url, 'github.com') }}" in src
        assert "if: ${{ !contains(github.server_url, 'github.com') }}" in src
-    def test_compat_files_an_issue_on_each_forge(self):
+    def test_compat_files_its_report_through_ONE_implementation(self):
        # It used to be two near-identical shell steps, one per forge. Two copies of
        # "find the issue, decide whether to comment, post it" is two chances to drift,
        # and the Gitea one duplicated an issue for real.
        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
            src = fh.read()
-        assert "file a bug report (GitHub)" in src
+        assert "tools/compat_publish.py" in src
-        assert "file a bug report (Gitea)" in src
+        assert "file a bug report (GitHub)" not in src
        assert "file a bug report (Gitea)" not in src
 class TestTheBotDoesNotSpam:
    """It left 11 identical 3,000-character comments on one issue in a single day.
    A bot that repeats itself daily gets muted — and then the next REAL finding is
    scrolled past, which defeats the entire reason for building it.
    """
    def publisher(self):
        with open(os.path.join(ROOT, "tools", "compat_publish.py"), encoding="utf-8") as fh:
            return fh.read()
    def test_it_compares_a_fingerprint_before_saying_anything(self):
        src = self.publisher()
        assert "extract_fingerprint" in src
        assert "staying quiet" in src
    def test_the_body_is_edited_in_place_not_appended_to(self):
        src = self.publisher()
        assert '"PATCH"' in src, "the issue body must be updated, not commented onto"
    def test_it_closes_the_issue_when_everything_is_fixed(self):
        src = self.publisher()
        assert '"state": "closed"' in src
    def test_the_matrix_refresh_opens_a_PR_rather_than_pushing_to_main(self):
        # An unattended push to main from CI is exactly what the release barrier exists
        # to prevent: a bot that can move main can move it somewhere nobody looked.
        #
        # Checked against CODE, not comments — the step's own commentary explains what
        # it replaced, and that mention must not read as the thing itself.
        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
            src = fh.read()
        code = "\n".join(
            ln for ln in src.splitlines() if not ln.lstrip().startswith("#")
        )
        assert "/pulls" in code, "the matrix refresh must open a PR"
        assert "HEAD:main" not in code, "CI still pushes straight to main"
    def test_the_matrix_PR_targets_the_CANONICAL_forge_not_the_mirror(self):
        # GitHub is a one-way mirror: a PR merged there would be silently clobbered by
        # the next `fleet-repos mirror` push from Gitea. A bot opening PRs against a
        # mirror is a bot doing nothing, slowly.
        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
            src = fh.read()
        i = src.index("refresh the README matrix")
        step = src[i:i + 400]
        assert "!contains(github.server_url, 'github.com')" in step, (
            "the matrix PR must be opened on Gitea (canonical), not GitHub (mirror)"
        )
    def test_the_workflow_has_the_permissions_its_steps_actually_need(self):
        # It shipped with `contents: read` while the step pushed a branch and opened a
        # PR — it would have died with a 403 on the first scheduled run, and I would
        # have had a bot that silently never worked.
        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
            src = fh.read()
        perms = src[src.index("permissions:"):src.index("jobs:")]
        assert "contents: write" in perms, "pushing a branch needs contents: write"
        assert "pull-requests: write" in perms, "opening a PR needs pull-requests: write"
        assert "issues: write" in perms
 class TestCompatCannotSilentlyPass:
@@ -105,3 +172,81 @@ class TestCompatCannotSilentlyPass:
        with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
            src = fh.read()
        assert "steps.check.outputs.shipped_broken != '0'" in src
 class TestActionCacheRace:
    """CI must not run concurrent jobs on the self-hosted runner.
    `act` caches each ACTION as one shared clone under /root/.cache/act/<hash>
    and re-pulls it per job, so jobs starting together fight over that directory
    and the loser dies with `lstat .../<file>: no such file or directory` before
    any test runs -- a red `main` with zero suite output and a different victim
    each push. The runner also force-pulls its base image per job, so job count
    is also Docker Hub pull count, and four-per-push exhausted the anonymous
    limit in an afternoon. Both problems have the same cure: one job.
    """
    def _ci(self):
        with open(os.path.join(WORKFLOWS, "ci.yml"), encoding="utf-8") as fh:
            return fh.read()
    def test_ci_runs_as_exactly_one_job(self):
        """The fix is the absence of concurrency, not the absence of one action.
        Dropping astral-sh/setup-uv only shrank the surface -- every job still
        used actions/checkout. A single job cannot race itself whatever actions
        it uses, which is why this, and not the action count, is the invariant.
        """
        ci = self._ci()
        # Scope to the jobs: block -- `on:` has two-space keys of its own
        # (push/pull_request/workflow_dispatch) that look identical otherwise.
        body = ci[ci.index("\njobs:"):]
        jobs = re.findall(r"^  (\w[\w-]*):$", body, re.M)
        assert len(jobs) == 1, (
            f"ci.yml defines {len(jobs)} jobs ({jobs}); concurrent jobs on the "
            "self-hosted runner race on act's shared action cache and multiply "
            "Docker Hub pulls. Keep CI to one job."
        )
    def test_no_matrix_reintroduces_parallel_jobs(self):
        ci = self._ci()
        assert "strategy:" not in ci and "matrix:" not in ci, (
            "a matrix fans out into concurrent jobs again -- sweep versions "
            "inside one job instead"
        )
    def test_every_python_version_still_runs_after_one_fails(self):
        """`fail-fast: false` is what the loop has to preserve.
        A 3.11 break must not hide whether 3.12 and 3.13 are fine; that is
        precisely the information you want at that moment.
        """
        ci = self._ci()
        assert 'PYTHONS: "3.11 3.12 3.13"' in ci
        assert ci.count("fail=1") >= 2, "the sweeps must collect failures, not exit early"
    def test_uv_is_installed_without_an_action(self):
        """Checks `uses:` directives, not prose.
        The comment in ci.yml names the action it deliberately avoids, and that
        explanation is the most useful thing in the file -- a test that greps the
        raw text would forbid documenting the very lesson it enforces. Parsed
        with a regex rather than PyYAML on purpose: CI runs `uvx pytest`, whose
        environment holds pytest and nothing else, so a third-party import here
        fails on the runner while passing locally.
        """
        ci = self._ci()
        used = re.findall(r"^\s*-?\s*uses:\s*(\S+)", ci, re.M)
        assert not [u for u in used if "setup-uv" in u], (
            "the action was only fetching a binary; a run: step does the same "
            "with one less moving part"
        )
        assert "astral.sh/uv/" in ci
    def test_the_uv_version_is_pinned(self):
        ci = self._ci()
        assert re.search(r'UV_VERSION:\s*"\d+\.\d+\.\d+"', ci), (
            "an unpinned uv lets any upstream release turn main red with no "
            "code change here -- the same rule ruff is pinned under"
        )
        assert "https://astral.sh/uv/${UV_VERSION}/install.sh" in ci
@@ -43,6 +43,7 @@ from __future__ import annotations
 import argparse
 import ast
 import hashlib
 import json
 import os
 import sys
@@ -132,9 +133,43 @@ ASSUMPTIONS = [
        params=["middleware", "job", "cloud_backup"],
        why="SYNC_BLOCK wraps it to tear down bind mounts in a finally",
    ),
    Assumption(
        # Not a plugin method -- a method on the middleware OBJECT itself, which the
        # manifest had no way to express and therefore never checked.
        #
        # The nested module calls `middleware.get_service(<ns>)` to decide whether to
        # sweep snapshots through `pool.snapshot` or `zfs.snapshot` (see
        # SNAPSHOT_SERVICES). If it ever disappears, `_can_delete()` catches the
        # AttributeError, reports BOTH namespaces unusable, and every nested backup
        # fails -- loudly, but only at RUN time, on a box the preflight had already
        # declared healthy. Checking it costs one file read.
        "get-service", NESTED, "utils/plugins.py",
        "LoadPluginsMixin.get_service", kind="method",
        params=["self", "name"],
        why="snapshot_service() resolves the snapshot namespace through it; without "
            "it the module cannot sweep the snapshot it just took",
    ),
 ]
 def accepted_spellings(name):
    """The method names that satisfy a call to `<namespace>.<name>`.
    A CRUDService exposes `create`/`update`/`delete` from methods NAMED
    `do_create`/`do_update`/`do_delete`. Both are live across the matrix: 24.10 and
    25.04 declare `do_delete`, 25.10 renamed it to `delete`, and all of them answer
    to `<ns>.delete`. Accepting only the literal name reported working releases as
    BROKEN and would have switched nested snapshots off on boxes where they work.
    """
    return (name, f"do_{name}")
 #: The spellings that satisfy `<ns>.delete`. A test binds this to the runtime's
 #: `truecloud_nested.DELETE_METHODS`, so the checker and the patch cannot come to
 #: disagree about what "can delete" means on the same box.
 DELETE_NAMES = accepted_spellings("delete")
 class MiddlewareCall:
    """A middlewared METHOD the injected code calls at runtime.
@@ -161,86 +196,166 @@ class MiddlewareCall:
    the whole design: declining is always the cheaper mistake.
    """
-    def __init__(self, ident, module, method, path, why=""):
+    def __init__(self, ident, module, method, path, why="", also=()):
        self.id = ident
        self.module = module
-        self.method = method              # "zfs.snapshot.delete"
+        self.method = method              # "pool.snapshot.delete"
        self.path = path                  # plugin file that declares it
        self.why = why
        #: Equally acceptable spellings of the SAME call, as (method, path) pairs.
        #:
        #: No single snapshot namespace spans every supported release. 24.10 and
        #: 25.04 expose the CRUD service as the public `zfs.snapshot`; 25.10
        #: promoted it to `pool.snapshot` and demoted `zfs.snapshot` to private;
        #: 26 deleted `plugins/zfs_/` entirely. Pinning either one alone marks
        #: half the matrix BROKEN and declines to apply on versions that work
        #: perfectly well.
        #:
        #: The call is satisfied if ANY option is present. The runtime picks the
        #: same way -- see `pick_snapshot_service()` in the nested module -- so
        #: what this checks and what the patch does cannot drift apart.
        self.also = tuple(also)
    @property
-    def namespace(self):
+    def options(self):
-        return self.method.rsplit(".", 1)[0]
+        """Every (method, path) that would satisfy this call, best first."""
        return ((self.method, self.path), *self.also)
-    @property
+    @staticmethod
-    def name(self):
+    def namespace_of(method):
-        return self.method.rsplit(".", 1)[1]
+        return method.rsplit(".", 1)[0]
    @staticmethod
    def name_of(method):
        return method.rsplit(".", 1)[1]
 #: Every middlewared method the nested module calls at runtime.
 #: The middleware methods the nested module CALLS.
 #:
 #: These used to be the PRIVATE `zfs.*` service (`zfs.dataset.query`,
 #: `zfs.snapshot.delete`, `zfs.snapshot.query`). TrueNAS 26 deleted
 #: `plugins/zfs_/` outright and every one of them vanished -- silently, because a
 #: private service carries no stability contract and nothing warned us. The patch
 #: would have applied cleanly and then failed on the first backup.
 #:
 #: The replacements are the PUBLIC `pool.*` API, and switching to it is not merely
 #: a TrueNAS 26 fix -- it is the correct call on every version:
 #:
 #:   * It is public, documented, and covered by iX's deprecation policy, so it
 #:     cannot be deleted from under us the way `zfs.*` just was.
 #:   * The same methods, in the same files, exist on 24.10 through 26. One code
 #:     path, no version conditionals.
 #:   * Both spellings take `recursive`, so ONE call sweeps the whole tree instead
 #:     of ~250 individual deletes, any of which could be missed.
 MIDDLEWARE_CALLS = [
    MiddlewareCall(
-        "call-zfs-dataset-query", NESTED, "zfs.dataset.query",
+        "call-snapshot-delete", NESTED, "pool.snapshot.delete",
-        "plugins/zfs_/dataset.py",
+        "plugins/pool_/snapshot.py",
-        why="SNAPSHOT_BLOCK enumerates FILESYSTEM datasets to build the staging plan",
+        also=[("zfs.snapshot.delete", "plugins/zfs_/snapshot.py")],
    ),
    MiddlewareCall(
        "call-zfs-snapshot-delete", NESTED, "zfs.snapshot.delete",
        "plugins/zfs_/snapshot.py",
        why="delete_snapshot_tree() sweeps the recursive snapshot. Without it every "
            "run orphans one snapshot per descendant dataset (250 on a real pool)",
    ),
    MiddlewareCall(
        "call-zfs-snapshot-query", NESTED, "zfs.snapshot.query",
        "plugins/zfs_/snapshot.py",
        why="delete_snapshot_tree()'s fallback sweep enumerates the tree by name",
    ),
 ]
 # There is deliberately NO entry here for a dataset or snapshot QUERY.
 #
 # The patch used to call `zfs.dataset.query` / `zfs.snapshot.query` (private, and
 # deleted in TrueNAS 26). The obvious port was to the public `pool.dataset.query` /
 # `pool.snapshot.query` -- and that port was WRONG in a way no source check could
 # ever have caught, because the methods are all present and correctly shaped.
 #
 # They are simply filtered. On a real box they return 205 of 274 datasets and 205
 # of 274 snapshots, hiding `ix-apps/*`, `.system/*` and `.ix-virt/*` -- 84 of 270
 # on the production pool, including live application data. Staging from that view
 # silently omits them; sweeping from it orphans one snapshot per hidden dataset,
 # forever.
 #
 # So the module enumerates from ZFS itself and there is no middleware assumption
 # left to check. That is the point: the fewer things we assume about middleware,
 # the less there is for iX to break. Only the MUTATION is still a middleware call,
 # and that is the one entry above.
 def check_call(c: MiddlewareCall, src: str | None,
               method: str | None = None, path: str | None = None,
               ) -> tuple[str, str | None]:
    """Is `method` still registered by middlewared?
    `method`/`path` name WHICH spelling of the call is being tried -- a call may
    have several equally acceptable ones (see MiddlewareCall.also). They default
    to the preferred spelling.
    """
    method = method or c.method
    path = path or c.path
    namespace = MiddlewareCall.namespace_of(method)
    name = MiddlewareCall.name_of(method)
 def check_call(c: MiddlewareCall, src: str | None) -> tuple[str, str | None]:
    """Is `c.method` still registered by middlewared?"""
    if src is None:
        return "broken", (
-            f"{c.path} no longer exists, so `{c.method}` is gone"
+            f"{path} no longer exists, so `{method}` is gone"
        )
    try:
        tree = ast.parse(_stock(src))
    except SyntaxError as e:
-        return "unknown", f"{c.path} does not parse: {e}"
+        return "unknown", f"{path} does not parse: {e}"
-    # namespace = 'zfs.snapshot' on some Service class in this file...
+    # Find the CLASS that declares this namespace, and look for the method THERE.
-    namespaces = {
+    #
-        n.value.value
+    # Not anywhere in the file. `ast.walk` over the whole module made *any* function
-        for n in ast.walk(tree)
+    # called `delete` satisfy the check -- one on an unrelated class, or even a nested
-        if isinstance(n, ast.Assign)
+    # local function inside `do_query`. That is a FALSE OK, and it breaks the one
-        and isinstance(n.value, ast.Constant)
+    # invariant this checker and the runtime share: `_defines_delete()` looks in
-        and isinstance(n.value.value, str)
+    # `vars(klass)` for a PLUGIN class on the service's MRO. If iX gutted
-        and any(isinstance(t, ast.Name) and t.id == "namespace" for t in n.targets)
+    # `PoolSnapshotService.do_delete` while some other class in the same file still had
-    }
+    # a `delete`, compat would say ok, apply.sh would patch, and the runtime would then
-    if c.namespace not in namespaces:
+    # correctly refuse `pool.snapshot`, fall through to a `zfs.snapshot` that does not
-        return "broken", (
+    # exist on 26, and fail every nested backup on a box the preflight called healthy.
-            f"{c.path} no longer declares namespace {c.namespace!r} "
+    #
-            f"(found: {sorted(namespaces) or 'none'}), so `{c.method}` is gone"
+    # Same question on both sides: does the class that OWNS this namespace define the
-        )
+    # method?
    # ...and it defines the method.
    #
    # A CRUDService exposes `create`/`update`/`delete` from methods NAMED
-    # `do_create`/`do_update`/`do_delete`. Both spellings are live right now:
+    # `do_create`/`do_update`/`do_delete`. Both spellings are live: 24.10 and 25.04
-    # 24.10 and 25.04 declare `do_delete`, 25.10 renamed it to `delete`, and all
+    # declare `do_delete`, 25.10 renamed it to `delete`, and all answer to
-    # three answer to `zfs.snapshot.delete`. Accepting only the literal name reported
+    # `<ns>.delete`. Accepting only the literal name reported working releases as
-    # the two older releases as broken -- a false BROKEN that would have switched off
+    # broken.
-    # nested snapshots on boxes where they work.
+    owners = []
-    defined = {
+    all_namespaces = set()
-        n.name for n in ast.walk(tree)
+    for cls in (n for n in ast.walk(tree) if isinstance(n, ast.ClassDef)):
-        if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef)
+        declared = {
-    }
+            n.value.value
-    if c.name not in defined and f"do_{c.name}" not in defined:
+            for n in ast.walk(cls)
-        return "broken", f"{c.path} no longer defines `{c.method}`"
+            if isinstance(n, ast.Assign)
            and isinstance(n.value, ast.Constant)
            and isinstance(n.value.value, str)
            and any(isinstance(t, ast.Name) and t.id == "namespace" for t in n.targets)
        }
        all_namespaces |= declared
        if namespace in declared:
            owners.append(cls)
-    return "ok", None
+    if not owners:
        return "broken", (
            f"{path} no longer declares namespace {namespace!r} "
            f"(found: {sorted(all_namespaces) or 'none'}), so `{method}` is gone"
        )
    wanted = accepted_spellings(name)
    for cls in owners:
        # Direct members of the class, not its nested scopes: a `def delete` inside
        # another method is a local function, not a service method.
        if any(
            isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef) and n.name in wanted
            for n in cls.body
        ):
            return "ok", None
    return "broken", (
        f"{path} still declares namespace {namespace!r}, but its class no longer "
        f"defines `{'` or `'.join(wanted)}` -- so `{method}` is gone"
    )
 #: Things that mean iX has done the job themselves and the module should RETIRE,
@@ -403,7 +518,10 @@ def check_source(a: Assumption, src: str | None) -> tuple[str, str | None]:
    breaks a box that was working.
    """
    if src is None:
-        return "broken", f"{a.path} does not exist"
+        # Name the SYMBOL, not just the file. Whoever reads the bug report needs to
        # know what the patch can no longer reach, and "utils/plugins.py does not
        # exist" does not tell them that `get_service` is gone.
        return "broken", f"{a.path} does not exist, so `{a.symbol}` is gone"
    try:
        tree = ast.parse(src)
@@ -538,6 +656,7 @@ def check(loader, modules=None) -> dict:
            out[a.module]["unknown"] = True
            out[a.module]["problems"].append({
                "id": a.id, "detail": f"could not read {a.path}: {e}", "why": a.why,
                "state": "unknown",
            })
            continue
@@ -545,37 +664,57 @@ def check(loader, modules=None) -> dict:
        if status == "broken":
            out[a.module]["ok"] = False
            out[a.module]["problems"].append({
-                "id": a.id, "detail": detail, "why": a.why,
+                "id": a.id, "detail": detail, "why": a.why, "state": "broken",
            })
        elif status == "unknown":
            out[a.module]["unknown"] = True
            out[a.module]["problems"].append({
-                "id": a.id, "detail": detail, "why": a.why,
+                "id": a.id, "detail": detail, "why": a.why, "state": "unknown",
            })
    # The methods the injected code CALLS, not just the symbols it wraps.
    #
    # A call may have several equally acceptable spellings, because no single
    # snapshot namespace spans every supported release (24.10 has `zfs.snapshot`,
    # 26 has only `pool.snapshot`). It is satisfied if ANY of them is present --
    # exactly as the runtime resolves it -- and BROKEN only when they all vanish.
    for c in MIDDLEWARE_CALLS:
        if c.module not in out:
            continue
-        try:
+
-            text = src(c.path)
+        satisfied, unknown, details = False, False, []
-        except Unreadable as e:
+        for method, path in c.options:
-            out[c.module]["unknown"] = True
+            try:
-            out[c.module]["problems"].append({
+                text = src(path)
-                "id": c.id, "detail": f"could not read {c.path}: {e}", "why": c.why,
+            except Unreadable as e:
-            })
+                unknown = True
                details.append(f"could not read {path}: {e}")
                continue
            status, detail = check_call(c, text, method, path)
            if status == "ok":
                satisfied = True
                break
            if status == "unknown":
                unknown = True
            details.append(detail)
        if satisfied:
            continue
-        status, detail = check_call(c, text)
+        # Every spelling failed. If we could not READ one of them we do not know
-        if status == "broken":
+        # that it is broken -- a rate-limited fetch is not a regression.
-            out[c.module]["ok"] = False
+        if unknown:
            out[c.module]["problems"].append({
                "id": c.id, "detail": detail, "why": c.why,
            })
        elif status == "unknown":
            out[c.module]["unknown"] = True
            out[c.module]["problems"].append({
-                "id": c.id, "detail": detail, "why": c.why,
+                "id": c.id, "detail": "; ".join(details), "why": c.why,
                "state": "unknown",
            })
        else:
            out[c.module]["ok"] = False
            out[c.module]["problems"].append({
                "id": c.id, "detail": "; ".join(details), "why": c.why,
                "state": "broken",
            })
    for module, (path, phrase, native_when_present) in NATIVE_PROBES.items():
@@ -732,6 +871,24 @@ def discover_refs(remote: str = REPO) -> list[str]:
      * UNRELEASED comes from the BRANCHES, because that is where a beta appears
        first: `release/26.0.0-BETA.3` had no tag yet while it was the newest beta.
        Catching breakage here, before it ships, is the whole point of this file.
    THE NEXT MAINTENANCE RELEASE IS ALSO A BRANCH, and it used to fall through both
    sieves. `release/25.10.5` is branched but not yet tagged, and 25.10 has already
    shipped -- so it is not in `shipped` (no tag) and it was excluded from `upcoming`
    (its line is in `shipped_lines`). It was invisible. That is the one ref a 25.10.4
    user is actually about to be upgraded onto, so a break there reaches people
    BEFORE the daily check ever looks at it -- the exact hole this file exists to
    close, on the only line anybody is running.
    The rule that separates it from the two things we must NOT report:
      * `release/24.10-RC.2` -- a prerelease of an already-shipped line. It is
        history, not a warning; it sorts OLDER than TS-24.10.2.4, so it is dropped.
      * `release/25.20.2.2` -- iX's typo branch. 25.20 never shipped, so it has no
        TS tag, so it is not a line at all and is dropped.
    ...which is: a plain `release/X.Y.Z` branch counts only if its line HAS shipped
    and it sorts NEWER than that line's newest tag. Both exclusions fall out of it.
    """
    tags = _ls_remote(remote, "--tags")
    heads = _ls_remote(remote, "--heads")
@@ -739,26 +896,45 @@ def discover_refs(remote: str = REPO) -> list[str]:
    shipped = _newest_per_line([
        t for t in tags if t.startswith("TS-") and "-BETA" not in t and "-RC" not in t
    ])
    newest_shipped = {_version_of(t)[0][:2]: _version_of(t) for t in shipped}
    release_heads = [h for h in heads if h.startswith("release/")]
    # A prerelease of a line that has ALREADY shipped is history, not a warning:
    # release/24.10-RC.2 still exists, and the nested module does not apply to it,
    # but 24.10 shipped long ago and TS-24.10.2.4 is fine. Reporting it would be a
    # standing red row in the matrix for a version nobody can install.
    shipped_lines = {_version_of(t)[0][:2] for t in shipped}
    upcoming = [
        h for h in _newest_per_line([
-            h for h in heads
+            h for h in release_heads if "-BETA" in h or "-RC" in h
            if h.startswith("release/") and ("-BETA" in h or "-RC" in h)
        ])
-        if _version_of(h)[0][:2] not in shipped_lines
+        if _version_of(h)[0][:2] not in newest_shipped
    ]
-    return [*shipped, *upcoming, "master"]
+    # The next maintenance release of a line that HAS shipped: branched, untagged,
    # and the very next thing those users get. See the docstring.
    pending = []
    for h in _newest_per_line([
        h for h in release_heads if "-BETA" not in h and "-RC" not in h
    ]):
        v = _version_of(h)
        tagged = newest_shipped.get(v[0][:2])
        if tagged and v > tagged:
            pending.append(h)
    return [*shipped, *pending, *upcoming, "master"]
 def is_unreleased(ref: str) -> bool:
-    """master and any BETA/RC. Breakage here is early warning, not an outage."""
+    """Anything iX has not TAGGED. Breakage here is early warning, not an outage.
-    return ref == "master" or "-BETA" in ref or "-RC" in ref
+
    Keyed on where the ref came from, not on its name: `discover_refs` takes shipped
    releases from `TS-*` TAGS and everything else from BRANCHES, so a `release/*` ref
    is by construction something iX has not released yet. Testing for `-BETA`/`-RC`
    instead would call `release/25.10.5` SHIPPED, and a break there would fail the
    build as a live outage -- on a version nobody is running yet.
    """
    return ref == "master" or ref.startswith("release/")
 def matrix(refs=None, remote: str = REPO) -> list[dict]:
@@ -802,7 +978,16 @@ def is_broken(r: dict) -> bool:
 #: is static analysis of iX's source, which proves the patch's assumptions hold --
 #: a strictly weaker claim than "a restore worked". Add a row only after doing it.
 HARDWARE_VERIFIED = {
-    "25.10.4": "nested + providers; 252-snapshot recursive backup of /mnt/Tap, 18m",
+    "25.10.4": (
        "v0.7.0: 3 live tasks — 191-dataset nested backup of /mnt/Tap, a "
        "215-filesystem/2-zvol backup of /mnt/Tank/backups, and a non-nested one; "
        "0 orphans, 0 leaked mounts, byte-identical restore; the collector also "
        "reclaimed a real orphan the pool had been carrying"
    ),
    "26.0.0-BETA.1": (
        "v0.7.0: 274-snapshot recursive backup of a 292-dataset pool; restored a "
        "4-deep child dataset byte-identical; zvol-orphan case reproduced then closed"
    ),
 }
 _LEGEND = """
@@ -815,6 +1000,16 @@ _LEGEND = """
 "ok" means *the patch's assumptions hold*, checked automatically against iX's
 source. It does not mean a human ran a backup on it — that is the
 **Hardware-verified** column, which is filled in by hand and only by doing it.
 **`master` is not the next release.** iX branches each major off to its own
 `release/` line and master rolls straight on to the one after — so master is
 `27-dev` while 26 is still in beta. A **BROKEN** master means iX has changed
 something that will reach users *a major release from now*, not in the version you
 are about to install. Read the numbered rows for that.
 A row like `25.10.5 _(unreleased)_` is the next maintenance release: branched by iX,
 not tagged yet, and the very next thing a 25.10.4 box gets. It is checked precisely
 because it is the one unshipped ref that reaches real users without warning.
 """
@@ -862,17 +1057,42 @@ def update_readme(rows: list[dict], path: str = README) -> bool:
    return True
 def dev_label(rows: list[dict]) -> str:
    """What `master` is a development line FOR -- e.g. "27-dev".
    `master` is NOT the next release, and labelling it "master _(unreleased)_" said
    it was. On 2026-07-14 every recent commit on master targeted 27.0.0-BETA.1 while
    26 was still in beta on its own `release/26.0.0-BETA.*` branches: master had
    already rolled over to the major AFTER the one that has not shipped yet. So a red
    `master` row read as "the version you are about to install is broken" when the
    breakage was a year out, on a line nobody can even download. That is a false alarm
    aimed squarely at the person deciding whether to trust this with their backups.
    Derived, not hardcoded: the newest major we can see anywhere, plus one. iX branches
    `release/N.0.0-BETA.1` off master and master immediately becomes N+1 -- so when 27
    betas appear, this says 28-dev on its own.
    """
    majors = [
        v[0][0] for v in (_version_of(r["ref"]) for r in rows if r["ref"] != "master")
        if v
    ]
    return f"{max(majors) + 1}-dev" if majors else "unreleased"
 def render_markdown(rows: list[dict]) -> str:
    """The matrix, for the README."""
    out = [
        "| TrueNAS | B2/S3 providers | Nested snapshots | Hardware-verified |",
        "| --- | --- | --- | --- |",
    ]
    dev = dev_label(rows)
    for row in rows:
        m = row["modules"]
        ref = row["ref"]
        label = ref.removeprefix("TS-").removeprefix("release/")
-        if row["unreleased"]:
+        if ref == "master":
            label = f"master _({dev})_"
        elif row["unreleased"]:
            label = f"{label} _(unreleased)_"
        cells = []
@@ -892,6 +1112,96 @@ def render_markdown(rows: list[dict]) -> str:
    return "\n".join(out) + "\n" + _LEGEND
 FINGERPRINT = "<!-- compat-fingerprint:"
 def fingerprint(rows: list[dict]) -> str:
    """A stable digest of WHAT IS BROKEN, and nothing else.
    The bug report must be updated when the findings change and stay silent when they
    do not. Without this the workflow commented on every run -- it left **11 identical
    3,000-character comments** on one issue in a single day, which is not a warning
    system, it is a mute button with extra steps.
    Deliberately excludes anything that moves on its own: the matrix's `ok` rows, the
    hardware-verified column, and the exact TrueNAS point-release (`TS-25.10.4` ->
    `TS-25.10.5` is not news). Only the broken (ref, module, problem-id) triples count.
    """
    findings = sorted(
        (r["ref"], mod, p["id"])
        for r in rows
        for mod, m in r["modules"].items()
        if is_broken(m)
        for p in m["problems"]
        # `unknown` problems are things we could not READ (a 429, an EACCES), not
        # things iX changed. On a ref that is broken for some other reason they would
        # otherwise join the digest, so one transient network blip rewrites the issue
        # body and the next clean run rewrites it back. That is the daily-noise
        # failure this fingerprint exists to prevent, wearing a different hat.
        if p.get("state", "broken") == "broken"
    )
    return hashlib.sha256(repr(findings).encode()).hexdigest()[:16]
 def extract_fingerprint(body: str) -> str | None:
    """The fingerprint a previous run left in the issue body, if any."""
    if not body:
        return None
    i = body.find(FINGERPRINT)
    if i == -1:
        return None
    return body[i + len(FINGERPRINT):].split("-->", 1)[0].strip() or None
 def render_issue(rows: list[dict]) -> str:
    """The bug report body: what is broken, why it matters, and nothing else up front.
    Short by design. The full matrix and the healthy versions go in a fold -- somebody
    opening this wants to know what broke and whether it can hurt them, not to re-read
    a table they can see in the README.
    """
    broken = [r for r in rows if any(is_broken(m) for m in r["modules"].values())]
    out = [
        "`tools/compat.py` checks what this patch assumes about middlewared against "
        "iXsystems' actual source, every day. Those assumptions no longer hold on the "
        "versions below.",
        "",
        "**This does not break anyone today.** `apply.sh` re-checks on every boot and "
        "**declines to apply** a module whose assumptions fail, so TrueNAS is left "
        "stock rather than half-patched. The cost is the module's feature, not a "
        "broken backup.",
        "",
    ]
    for r in broken:
        out.append(f"### `{r['ref']}`")
        out.append("")
        for mod, m in sorted(r["modules"].items()):
            if not is_broken(m):
                continue
            out.append(f"**{mod}**")
            out.append("")
            for p in m["problems"]:
                out.append(f"- {p['detail']}")
                out.append(f"  <br><sub>{p['why']}</sub>")
            out.append("")
    out += [
        "<details><summary>Full support matrix</summary>",
        "",
        render_markdown(rows),
        "</details>",
        "",
        "_Filed and kept up to date by "
        "[`compat.yml`](.github/workflows/compat.yml). It edits this body when the "
        "findings change, and stays quiet when they do not._",
        "",
        f"{FINGERPRINT} {fingerprint(rows)} -->",
    ]
    return "\n".join(out)
 def render_matrix(rows: list[dict]) -> str:
    """A support table.
@@ -900,7 +1210,13 @@ def render_matrix(rows: list[dict]) -> str:
    hardware-verified column lives in COMPATIBILITY.md and is maintained by hand,
    because nothing else can honestly fill it in.
    """
-    w = max((len(r["ref"]) for r in rows), default=10)
+    dev = dev_label(rows)
    # Same relabel as the README: a red `master` is a warning about the major AFTER
    # next, and "master" alone reads as "the release you are about to install".
    names = {r["ref"]: (f"master ({dev})" if r["ref"] == "master" else r["ref"])
             for r in rows}
    w = max((len(n) for n in names.values()), default=10)
    lines = [
        f"{'TrueNAS'.ljust(w)}  {'providers':<10}  {'nested':<10}",
        f"{'-' * w}  {'-' * 10}  {'-' * 10}",
@@ -908,7 +1224,7 @@ def render_matrix(rows: list[dict]) -> str:
    for row in rows:
        m = row["modules"]
        lines.append(
-            f"{row['ref'].ljust(w)}  "
+            f"{names[row['ref']].ljust(w)}  "
            f"{_verdict(m[PROVIDERS]):<10}  {_verdict(m[NESTED]):<10}"
        )
    return "\n".join(lines)
@@ -0,0 +1,185 @@
 #!/usr/bin/env python3
 """Keep ONE bug report in sync with what compat.py currently finds.
 WHY THIS IS NOT JUST "POST A COMMENT"
 -------------------------------------
 The first version commented on every run that found a break. In one day it left
 **11 identical 3,000-character comments** on the same issue. That is not a warning
 system; it is a mute button with extra steps. The next real finding would have been
 scrolled past, which defeats the entire point of building it.
 So:
  * **The issue body is the current truth.** It is edited in place, never appended to.
  * **Comments are a changelog of CHANGES.** A run whose findings are identical to the
    last one says nothing at all -- no comment, no edit, no notification.
  * A fingerprint of the findings (broken ref/module/problem triples only) is embedded
    in the body. It deliberately ignores things that move on their own -- healthy rows,
    the hardware-verified column, TrueNAS point releases -- so `TS-25.10.4` becoming
    `TS-25.10.5` is not news, and does not wake anybody up.
  * When everything is fixed, the issue is **closed** with a comment saying so.
 Works against GitHub and Gitea, which differ only in the auth header and the issue
 list URL. One implementation, so the two cannot drift.
    python3 tools/compat_publish.py --api <url> --token <tok> --matrix /tmp/matrix.json
 """
 from __future__ import annotations
 import argparse
 import json
 import sys
 import urllib.error
 import urllib.request
 sys.path.insert(0, __file__.rsplit("/", 1)[0])
 from compat import (  # noqa: E402
    extract_fingerprint,
    fingerprint,
    is_broken,
    render_issue,
 )
 TITLE = "TrueNAS compatibility: the patch's assumptions no longer hold"
 def _call(url, token, method="GET", data=None):
    req = urllib.request.Request(
        url, method=method,
        headers={
            # Gitea wants `token <t>`; GitHub accepts `Bearer <t>`. GitHub also
            # accepts `token <t>`, so one header serves both.
            "Authorization": f"token {token}",
            "Content-Type": "application/json",
            "Accept": "application/vnd.github+json",
        },
        data=json.dumps(data).encode() if data else None,
    )
    with urllib.request.urlopen(req) as r:  # noqa: S310
        return json.load(r) if r.length != 0 else {}
 def _same(a, b):
    """Is the issue body already what we would write?
    Compared after normalising line endings and trailing space: forges are free to
    round-trip `\r\n`, and a body that only "differs" by that would be rewritten on
    every single run -- a silent edit, but a pointless one that churns `updated_at`
    and makes the issue look freshly touched every morning.
    """
    def norm(s):
        return "\n".join(line.rstrip() for line in (s or "").replace("\r\n", "\n").split("\n")).strip()
    return norm(a) == norm(b)
 def find_issue(api, token, title):
    """The LOWEST-numbered issue with this title, open or closed.
    Lowest, not "whichever the API returns first": two issues with the same title
    existed once (an earlier version put the ref list in the title, so the identity
    changed whenever that set changed), and an order-dependent pick would alternate
    between them -- reopening one while commenting on the other.
    Both forges list PRs alongside issues, but they SAY SO DIFFERENTLY: GitHub omits
    the `pull_request` key on a plain issue, Gitea sends it as `null`. Testing for the
    KEY therefore discards every Gitea issue as if it were a PR -- so this returned
    None on every Gitea run, and the bot filed a brand-new duplicate report each time
    instead of editing the one it already had. Test the VALUE; it is the only form
    that is true on both.
    """
    issues = _call(f"{api}/issues?state=all&per_page=100&limit=100", token)
    mine = [
        i for i in issues
        if i.get("title") == title and not i.get("pull_request")
    ]
    return min(mine, key=lambda i: i["number"]) if mine else None
 def main(argv):
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--api", required=True, help="…/repos/<owner>/<repo>")
    ap.add_argument("--token", required=True)
    ap.add_argument("--matrix", required=True, help="compat.py --matrix --json output")
    args = ap.parse_args(argv[1:])
    with open(args.matrix, encoding="utf-8") as fh:
        rows = json.load(fh)
    broken = [r for r in rows if any(is_broken(m) for m in r["modules"].values())]
    issue = find_issue(args.api, args.token, TITLE)
    # ── everything is healthy ────────────────────────────────────────────────
    if not broken:
        if issue and issue["state"] == "open":
            _call(f"{args.api}/issues/{issue['number']}/comments", args.token, "POST",
                  {"body": "All of the patch's assumptions hold again on every "
                           "checked TrueNAS version. Closing."})
            _call(f"{args.api}/issues/{issue['number']}", args.token, "PATCH",
                  {"state": "closed"})
            print(f"closed #{issue['number']} — nothing is broken any more")
        else:
            print("nothing broken; no open report to close")
        return 0
    body = render_issue(rows)
    want = fingerprint(rows)
    # ── nothing to file yet ──────────────────────────────────────────────────
    if issue is None:
        made = _call(f"{args.api}/issues", args.token, "POST",
                     {"title": TITLE, "body": body})
        print(f"filed #{made['number']}")
        return 0
    have = extract_fingerprint(issue.get("body") or "")
    n = issue["number"]
    # ── two different questions, and they were being answered with one answer ────
    #
    #   * IS THE BODY STILL TRUE?     -> if not, rewrite it. Editing an issue body
    #     notifies NOBODY on either forge, so keeping it honest is free.
    #   * HAVE THE FINDINGS CHANGED?  -> only then comment. Comments DO notify, and a
    #     daily "still broken, same as yesterday" is what teaches everyone to ignore
    #     the one that finally matters.
    #
    # Conflating them meant an unchanged FINGERPRINT froze the BODY. The fingerprint
    # deliberately ignores everything that moves on its own -- healthy rows, the
    # hardware-verified column, point releases, how a row is LABELLED -- so none of
    # that could ever reach the report. Relabelling master `27-dev` (it is not the
    # next release; a red row there was reading as "the version you are about to
    # install is broken") would have shipped to the README and never to the issue
    # anybody actually opens.
    body_is_current = _same(issue.get("body"), body)
    if have == want and issue["state"] == "open" and body_is_current:
        print(f"#{n} is already current ({want}) — staying quiet")
        return 0
    _call(f"{args.api}/issues/{n}", args.token, "PATCH", {"body": body, "state": "open"})
    if have != want:
        refs = ", ".join(f"`{r['ref']}`" for r in broken)
        note = (
            "The findings changed — the report above has been updated.\n\n"
            f"Currently broken on: {refs}."
            if have else
            "This report is now kept up to date automatically: the body above always "
            "reflects the current findings, and a comment is only added when they "
            "change."
        )
        _call(f"{args.api}/issues/{n}/comments", args.token, "POST", {"body": note})
        print(f"updated #{n}: {have} -> {want}")
    elif issue["state"] != "open":
        print(f"reopened #{n}")
    else:
        # Same findings, new rendering. Silent by design: nothing has changed that
        # anybody needs waking up for, but the report should not be telling lies.
        print(f"#{n}: findings unchanged ({want}); body refreshed silently")
    return 0
 if __name__ == "__main__":
    sys.exit(main(sys.argv))
@@ -3,7 +3,7 @@
 set -euo pipefail
-VERSION="0.6.1"
+VERSION="0.8.0"
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 _HOOK_COMMENT='TrueCloud provider patch (S3/B2)'
@@ -124,6 +124,11 @@ if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]; then
    rm -f "$PATCH_DIR/nested_snapshots_enabled"
    echo "  Removed nested-snapshot opt-in marker."
 fi
 # Runtime breadcrumb recorded by apply.sh for wait_restart.sh. Harmless, but a
 # stale path left in an uninstalled tree is exactly the sort of thing that reads
 # as state later.
 rm -f "$PATCH_DIR/.mw_dir"
 echo ""
 if [ "$_restore_failed" -eq 1 ]; then
@@ -19,7 +19,7 @@
 set -euo pipefail
-VERSION="0.6.1"
+VERSION="0.8.0"
 PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
 _PREV_FILE="$PATCH_DIR/.update_previous"
@@ -0,0 +1,3 @@
 version = 1
 revision = 3
 requires-python = ">=3.12"