diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c71a4c2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,62 @@ +name: CI + +on: + push: + branches: [main, "feat/**", "fix/**"] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + shell: + name: shell (shellcheck + syntax) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: bash syntax check + run: | + fail=0 + while IFS= read -r f; do + bash -n "$f" || { echo "::error file=$f::bash syntax error"; fail=1; } + done < <(find . -name '*.sh' -not -path './.git/*') + exit $fail + + # Pinned to a release tag, not @master: a third-party action on a moving + # branch runs whatever that branch contains at the time CI fires. + - name: shellcheck + uses: ludeeus/action-shellcheck@2.0.0 + env: + SHELLCHECK_OPTS: -S warning -e SC1091 + + python: + name: python ${{ matrix.python }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # TrueNAS SCALE middleware runs 3.11+; keep the patch importable across + # the versions it may be injected into. + python: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: install dev deps + run: python -m pip install --upgrade pip pytest ruff + + - name: ruff + run: ruff check patch tests + + - name: pytest + run: pytest tests -v + + - name: verify injected middleware blocks compile + # Belt-and-braces: the *_BLOCK strings are appended into live middlewared + # modules. A syntax error there would break the box at boot. + run: pytest tests/test_apply_blocks.py -v diff --git a/.gitignore b/.gitignore index 9588eec..daf6914 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,12 @@ /apply.log.2 /hook_status.json /disabled +/nested_snapshots_enabled + +# Python +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +.venv/ +venv/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 218d5ce..859a99c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,135 @@ # Changelog +## v0.3.0 — 2026-07-12 + +### Added + +- **`snapshot = true` now works on datasets that have child datasets** — + **opt-in, off by default** (`install.sh --enable-nested-snapshots` / + `--disable-nested-snapshots`). It changes how backups read their source data, + so it is never enabled implicitly; with neither flag `install.sh` preserves + the existing setting, so a `git pull && bash install.sh` cannot silently flip + it. When disabled, `apply.sh` skips the patch entirely and the stock guard + remains. `uninstall.sh` tears down any staging mounts and removes the marker. + Stock TrueNAS refuses this with *"This option is only available for datasets + that have no further nesting"*, which makes the snapshot option unusable for + the single most common case on any box running Apps — every app is its own + dataset, often with `config`/`pgdata` children of its own. Without it, the + backup reads **live** files: databases are captured mid-write, and a busy app + rewriting its files can stall a backup indefinitely as restic chases a moving + target. + + The stock guard is **correct, and it is not an arbitrary limit.** + `plugins/cloud/snapshot.py` already takes a *recursive* ZFS snapshot, but it + then points the backup tool at the **parent** dataset's + `.zfs/snapshot//` directory — and ZFS does not expose child datasets + through a parent's snapshot directory: + + ``` + /mnt/Tap/.zfs/snapshot//apps/ -> 0 entries (children invisible) + /mnt/Tap/apps/lidarr/config/.zfs/snapshot// -> the real data + ``` + + So without the guard the backup tool would walk a near-empty tree, report + SUCCESS, and upload almost nothing. iX gate the config rather than ship a + backup that lies about succeeding. + + This release implements the missing half. After the (already recursive) + snapshot is taken, every descendant dataset's own `.zfs/snapshot/` is + bind-mounted into a **staging tree** mirroring the original layout, and the + backup tool is pointed at the staging root — a complete, consistent, + point-in-time view of the whole subtree. Only then is the guard relaxed. + + Safety properties, in order of importance: + + - **Staging failure is loud.** If any descendant cannot be staged, the backup + fails. A silently-incomplete backup is the exact outcome the stock guard + exists to prevent, and it would be worse than not having the feature. + - **A post-mount verification pass** asserts every planned target is really a + mountpoint and the staging root is non-empty, so this can never regress into + the empty-backup failure it is meant to fix. + - **The guard is relaxed last.** `apply.sh` installs the traversal, patches + `snapshot.py`, then `sync.py`, and only then `crud.py`. A partial failure + leaves the guard intact and the option merely unavailable — never + "guard removed, traversal missing". + - **The patch owns the whole snapshot lifecycle.** `zfs.snapshot.delete` + defaults to `recursive=False` and stock `restic_backup()` calls it with no + options. Stock gets away with that only because its validation means + `recursive` is never True in the field — but enabling nested datasets makes + recursive snapshots real, so the parent now has one child snapshot per + descendant dataset (160+ on a typical Apps pool). Relying on stock's delete + would therefore orphan every child snapshot **on every successful run**. + This patch sweeps the parent *and* all children, is idempotent against + stock's `finally` winning the race, records the snapshot in a sidecar file + (so a middlewared restart mid-backup cannot orphan it), reclaims the tree + left by a crashed run, and deletes the tree when staging fails — where + sync.py's own `finally` would otherwise delete nothing at all, because its + `snapshot` local never gets assigned. + - **The dataset list is enumerated *after* the snapshot, never before.** A + list read beforehand can miss a dataset created in the gap: the recursive + snapshot would capture it but the staging plan would not, silently omitting + its data. Read afterwards, an unsnapshotted dataset trips the staging check + and fails the run loudly instead. + - **Every injected block no-ops** if `_truecloud_nested` is absent. + - Datasets that cannot contribute to a file tree (`mountpoint=none|legacy`, + unmounted/locked, encrypted-and-locked) are skipped and **reported** — + never dropped silently. + - Scoped to `cloud_backup` only. Cloud Sync (rclone) shares the same + validation mixin but has no staging teardown wired in, so its guard is left + in place deliberately. + + Side benefit: the staging root is a **stable** path per task, so restic can + find its parent snapshot between runs. Stock's + `.zfs/snapshot/-/` path changes every run, which defeats + restic's parent detection and forces a full re-scan each time. + +- **CI** (GitHub Actions): shellcheck + `bash -n` on every script, ruff, and + pytest on Python 3.11/3.12/3.13. Includes tests that `compile()` the + `*_BLOCK` strings — they are Python source appended to live middlewared + modules, so a syntax error there would break the box at boot, and nothing + previously checked them. + +### Changed + +- **The patch is now two independent modules, and each retires on its own.** + Previously the native-support check looked only for native B2 restic support + and, on finding it, set the kill switch and disabled *everything*. With a + second capability in the patch that would silently take a still-needed module + down with the superseded one — TrueNAS is likely to ship one of these long + before the other. + + `apply.sh` now detects each separately (`providers`: does `B2RcloneRemote` + carry a real `get_restic_config()`; `nested`: is the *"no further nesting"* + validation still in `plugins/cloud/crud.py`), skips just the superseded one, + and only sets the kill switch once **both** are done. The UI patch belongs to + `providers` and is skipped with it. The deferred middlewared restart now fires + when *any* still-needed module landed — keying it off `providers` alone would + have left a freshly-patched `nested` module on disk and never loaded on a + native-B2 box. `hook_status.json` reports each module with an `active` flag and + a reason. + +- README rewritten to be less alarmist: dropped the warning boxes and the + disclaimer's fear-bulleting in favour of plain statements, and documented the + two-module design. The one caveat kept as a plain sentence: the `mount --bind` + staging step has not yet been exercised by a live backup run. + +- Version strings in `install.sh`, `uninstall.sh`, and `recover.sh` were stale + at `0.0.4`; all scripts now report the same version. +- `patch_ui.py`: replaced a `try`/`except`/`pass` with `contextlib.suppress` + (no behaviour change; satisfies the new lint gate). + +### Removed + +- `patch/__pycache__/create_task.cpython-314.pyc` was committed to the + repository; it is now untracked and `__pycache__/` is gitignored. + +### Known issues + +- Stock `restic_backup()` deletes the ZFS snapshot in its own `finally`, which + fails with `EBUSY` while the staging bind mounts pin it. It logs one benign + `Error deleting snapshot ...` warning per run; the patch then unmounts and + deletes the snapshot for real. The warning is expected and harmless. + ## v0.2.1 — 2026-07-09 ### Fixed diff --git a/README.md b/README.md index 1041705..56f11b2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ # truenas-truecloud-patch -Extends TrueNAS SCALE's **TrueCloud Backup** feature to work with S3-compatible -providers and native Backblaze B2, instead of Storj only. +Extends TrueNAS SCALE's **TrueCloud Backup** feature to: + +- work with S3-compatible providers and native Backblaze B2, instead of Storj only; +- take **consistent snapshots of datasets that have child datasets** — which is + every box running Apps (see [Nested-dataset snapshots](#nested-dataset-snapshots)). --- @@ -24,31 +27,41 @@ costs a fraction of the new Storj price. --- -## ⚠ Disclaimer — please read before installing +## Before you install -**This project is unofficial, unsupported, and not affiliated with iXsystems -or the TrueNAS project in any way.** +This project is unofficial and not affiliated with iXsystems. A few things worth +knowing: -By installing this patch you accept the following: +- It targets **internal middleware APIs** with no stability contract, so a + TrueNAS update can break it. Every patch is fail-safe: if it can't apply, + middlewared starts normally and the reason is logged to `apply.log`. Check the + log after an update. +- If you file a TrueNAS bug report, **remove the patch first** and reproduce on a + stock system. +- **Test your restores.** That is true of any backup, but it matters more here — + see [Verifying it works](#verifying-it-works). +- Provided as-is, no warranty. See LICENSE. -- **Unsupported configuration.** TrueNAS support staff are not obligated to - help with any issue on a system running this patch. If you file a bug report, - remove the patch first and reproduce the issue on an unmodified system. +The patch is two independent modules — **providers** (B2/S3) and **nested** +(snapshots on nested datasets) — and each retires on its own once TrueNAS ships +that capability natively. See [Native support](#if-truenas-adds-native-support). -- **May break on TrueNAS updates.** The patch targets internal middleware APIs - that are not part of any public contract. They can change at any time. When - they do, the patch silently degrades to Storj-only behaviour rather than - breaking TrueNAS — but you should check the log after each update. +## Development -- **Your backups are your responsibility.** Verify that your backup jobs - complete successfully and that restores work before relying on them for - disaster recovery. +Parts of this project were written with AI assistance (Claude). All of it is +reviewed and tested before release; the test suite and CI exist in large part to +make that review meaningful. Bugs are mine. -- **No warranty.** This software is provided as-is. See the LICENSE file. +```bash +pip install pytest ruff +ruff check patch tests +pytest tests +``` -If TrueNAS adds native B2 or S3 support to TrueCloud Backup, the patch -detects it at boot, disables itself, and tells you to run `uninstall.sh` — -see [Native support](#if-truenas-adds-native-support) below. +CI runs shellcheck, `bash -n`, ruff, and pytest on Python 3.11–3.13. The tests +include a pass that `compile()`s the `*_BLOCK` strings in `patch/apply.sh` — +those are Python source appended into live `middlewared` modules, so a syntax +error there would break the box at boot. --- @@ -74,14 +87,168 @@ restarting. Expect one middlewared restart shortly after every boot; the UI and API are briefly unavailable while it happens, and running services are not affected. -| Layer | What changes | Technique | +| Module | What changes | Technique | |---|---|---| -| **Backend** | `B2RcloneRemote` gains `get_restic_config()` — skipped automatically if TrueNAS already provides one on the class. `restic.py` URL builder is fixed: strips the stray leading slash and converts the slash separator to a colon (`b2:bucket:path`), which is the format restic 0.16.x expects. URL wrapper is a no-op if the URL is already correctly formed. | File patch applied inside the overlayfs upper layer | -| **UI** | The Angular bundle's `filterByProviders` binding is widened from `["STORJ_IX"]` to `["STORJ_IX","S3","B2"]` | In-place text replacement in the compiled JS chunk; original is backed up before patching | +| **providers** | `B2RcloneRemote` gains `get_restic_config()` — skipped automatically if TrueNAS already provides one on the class. `restic.py` URL builder is fixed: strips the stray leading slash and converts the slash separator to a colon (`b2:bucket:path`), which is the format restic 0.16.x expects. URL wrapper is a no-op if the URL is already correctly formed. | File patch applied inside the overlayfs upper layer | +| **providers** (UI) | The Angular bundle's `filterByProviders` binding is widened from `["STORJ_IX"]` to `["STORJ_IX","S3","B2"]` | In-place text replacement in the compiled JS chunk; original is backed up before patching | +| **nested** (opt-in) | `_truecloud_nested.py` is installed into `plugins/cloud/`, and `plugins/cloud/{snapshot,crud}.py` + `plugins/cloud_backup/sync.py` are patched so `snapshot = true` works on a dataset that has child datasets. See [below](#nested-dataset-snapshots). | New module + file patches inside the overlayfs upper layer | -Both changes are **fail-safe**: if a patch cannot be applied (e.g. TrueNAS -restructured the relevant code), middlewared starts normally with Storj-only -support and the reason is logged to `apply.log` in your repo root. +All changes are **fail-safe**: if a patch cannot be applied (e.g. TrueNAS +restructured the relevant code), middlewared starts normally, the affected module +is simply inactive, and the reason is logged to `apply.log` in your repo root. The +two modules are independent — one failing or going native does not disable the +other. + +## Nested-dataset snapshots + +**Opt-in, off by default.** It changes how backups read their source data, so it +is never enabled implicitly: + +```bash +bash install.sh --enable-nested-snapshots +bash install.sh --disable-nested-snapshots +``` + +With neither flag `install.sh` leaves the setting alone, so `git pull && bash +install.sh` won't flip it. The providers module is unaffected either way. + +The planner and the snapshot lifecycle have been validated against a real +250-dataset pool. The `mount --bind` staging step has not yet been exercised by a +live backup run, so confirm your first backup actually contains child-dataset +data before relying on it — see [Verifying it works](#verifying-it-works). + +TrueCloud Backup's **Take Snapshot** option makes restic read from a frozen ZFS +snapshot instead of live files. Without it the backup reads data *while apps are +writing to it* — databases get captured mid-write, and an app that rewrites its +files continuously can stall a backup indefinitely as restic chases a moving +target. + +Stock TrueNAS refuses to enable it on most real-world paths: + +``` +[EINVAL] cloud_backup_update.snapshot: + This option is only available for datasets that have no further nesting +``` + +That rules out **any pool running Apps** — every app is its own dataset, usually +with `config`/`pgdata` children of its own. On a typical box that is 100+ nested +datasets, so the feature is effectively unusable exactly where it matters most. + +### Why stock refuses + +The guard is **correct**. `plugins/cloud/snapshot.py` already takes a *recursive* +ZFS snapshot — but it then points restic at the **parent** dataset's +`.zfs/snapshot//` directory, and ZFS does not expose child datasets +through a parent's snapshot directory: + +``` +/mnt/Tap/.zfs/snapshot//apps/ -> 0 entries (children invisible) +/mnt/Tap/apps/lidarr/config/.zfs/snapshot// -> the real data +``` + +So if you just remove the validation, restic walks a near-empty tree, reports +SUCCESS, and uploads almost nothing — a green backup job protecting no data. iX +gate the config rather than ship a backup that lies about succeeding. + +That is worth spelling out, because deleting those four lines in +`plugins/cloud/crud.py` is the obvious "fix" and it is the wrong one. The guard +is load-bearing: it has to be *replaced* with a working traversal, not removed. + +### What this patch does instead + +After the (already recursive) snapshot is taken, every descendant dataset's own +`.zfs/snapshot/` is bind-mounted into a **staging tree** that mirrors the +original layout, and restic is pointed at the staging root — a complete, +consistent, point-in-time view of the whole subtree. Only then is the guard +relaxed. + +Safety properties, in order of importance: + +- **Staging failure is loud.** If any descendant cannot be staged, the backup + *fails*. A silently-incomplete backup is precisely what the stock guard exists + to prevent, and it would be worse than not having the feature at all. +- **Post-mount verification** asserts every planned target really is a mountpoint + and the staging root is non-empty — so this cannot regress into the empty + backup it exists to fix. +- **The guard is relaxed last.** `apply.sh` installs the traversal, patches + `snapshot.py`, then `sync.py`, and only then `crud.py`. Any partial failure + leaves the guard intact and the option merely unavailable — never + "guard removed, traversal missing". +- Datasets that cannot contribute to a file tree (`mountpoint=none|legacy`, + unmounted, locked/encrypted) are skipped and **reported to the log** — never + dropped silently. +- Scoped to **cloud_backup only**. Cloud Sync (rclone) shares the same + validation mixin but has no staging teardown wired in, so its guard is + deliberately left in place. + +Side benefit: the staging root is a **stable path per task**, so restic can find +its parent snapshot between runs. Stock's `.zfs/snapshot/-/` +path changes every run, defeating restic's parent detection and forcing a full +re-scan each time. + +### Snapshot lifecycle + +`zfs.snapshot.delete` defaults to **`recursive=False`**, and stock +`restic_backup()` calls it with no options. Stock is safe only because its +validation means a *recursive* snapshot never actually happens in the field. +Enabling nested datasets makes them real: on a 250-dataset pool, +`zfs snapshot -r` creates **250 snapshots**, and stock's delete removes only the +parent — orphaning **249 on every successful run** (measured, not theorised). + +So the patch owns the whole lifecycle: + +- **Sweeps the parent and every child**, and is idempotent against stock's + `finally` winning the race once the mounts are released. +- **Records the snapshot in a sidecar file before mounting anything**, so a + middlewared restart mid-backup cannot orphan the tree (this patch *schedules* + a restart at boot, so that is not hypothetical). +- **Reclaims the tree left by a crashed run** instead of overwriting the record. +- **Deletes the tree when staging fails** — sync.py's own `finally` deletes + *nothing* in that case, because its `snapshot` local never gets assigned. +- **Enumerates datasets *after* the snapshot, never before.** A list read + beforehand can miss a dataset created in the gap, which the recursive snapshot + *would* capture but the staging plan would not — a silent omission. + +**Expected log noise:** stock's delete fails with `EBUSY` while the staging +mounts pin the snapshot. You will see one benign `Error deleting snapshot ...` +warning per run; the patch then unmounts and deletes the tree for real. + +### Verifying it works + +This feature exists because a backup can report SUCCESS while containing +nothing, so check the contents rather than the exit status: + +```bash +# 1. Does the restic snapshot actually contain child-dataset data? +# Pick a path that lives in a CHILD dataset (e.g. an app's config). +midclt call cloud_backup.list_snapshots | head + +# 2. List a child-dataset path inside the newest restic snapshot. +# If this is empty, the staging tree did not work and you are backing up NOTHING. +midclt call cloud_backup.list_snapshot_directory "" "/apps/lidarr/config" +``` + +You should see the app's real files (`lidarr.db`, `config.xml`, …). An empty +listing means the child datasets were not staged; disable the feature and open an +issue. + +```bash +# 3. No snapshots may be left behind after a run. +zfs list -t snapshot -r | grep -c cloud_backup- # expect 0 between runs + +# 4. No staging mounts may be left behind. +mount | grep truecloud-nested # expect no output +``` + +### Troubleshooting + +| Symptom | Cause | +|---|---| +| `This option is only available for datasets that have no further nesting` | Feature not enabled. Run `install.sh --enable-nested-snapshots`, then restart middlewared. | +| Backup fails: `dataset '…' has no snapshot '…'; refusing to back up an incomplete tree` | Working as designed — a descendant dataset was not covered by the snapshot. The backup is refused rather than silently omitting that data. | +| Backup fails: `snapshot '…' cannot be read (Permission denied)` | The snapshot exists but is unreadable. Middleware runs as root, so this indicates a real permissions problem, not a missing snapshot. | +| `cloud_backup-*` snapshots accumulating | The sweep is not running. Check `apply.log` for the nested patch applying, and confirm `sync.py` carries the `TRUECLOUD_PATCH` block. | +| Stale mounts under `/run/truecloud-nested` | A crashed run. The next run tears them down; `uninstall.sh` also cleans them. | ## Supported providers after patching @@ -239,25 +406,35 @@ and restores the original UI bundle from backup. ## If TrueNAS adds native support -`apply.sh` checks at every boot whether TrueNAS has shipped native B2 restic -support (by inspecting `B2RcloneRemote.__dict__`). If it has: +The patch is **two independent modules**, and each retires on its own — TrueNAS +is likely to ship one of these natively long before the other, and a module +going native must not take the other one down with it. -1. The kill switch (`disabled` file) is set — no patching on any future boot. -2. Any active overlays are unmounted immediately. -3. The following message is written to `apply.log`: +| Module | What it does | Detected as native when | +|---|---|---| +| **providers** | B2/S3 credentials for TrueCloud Backup (`b2.py`, `restic.py`, UI dropdown) | `B2RcloneRemote` carries a real `get_restic_config()` | +| **nested** | Snapshots on datasets with child datasets (`plugins/cloud/*`) | the *"no further nesting"* validation is gone from `plugins/cloud/crud.py` | -``` -NOTICE: TrueNAS now provides native B2 restic support — truecloud-patch is no longer needed. -NOTICE: Setting kill switch; patching will be skipped on all future boots. -NOTICE: Run the following to fully remove the patch: -NOTICE: bash /mnt/tank/truenas-truecloud-patch/uninstall.sh -``` +At every boot `apply.sh` checks both: + +- **One module goes native** → that module is skipped and logged; the other keeps + working, and the patch stays installed. +- **Both are done** (native, or nested was never enabled) → the kill switch + (`disabled` file) is set, overlays are unmounted, and `apply.log` tells you to + run `uninstall.sh`. + +So on a box using only the provider patch, native B2 support retires the whole +thing as before. On a box that also uses nested snapshots, native B2 support +retires *just* that half. Check the log after any TrueNAS update: ```bash -cat /mnt/tank/truenas-truecloud-patch/apply.log | tail -20 +tail -20 /mnt/tank/truenas-truecloud-patch/apply.log ``` +`hook_status.json` reports each module separately (`module.providers`, +`module.nested_snapshots`) with an `active` flag and a reason. + **Scenarios where the auto-detect may not fire** (manual check needed): | Scenario | What happens | Action | diff --git a/install.sh b/install.sh index 5c635ff..56facbe 100755 --- a/install.sh +++ b/install.sh @@ -18,11 +18,50 @@ set -euo pipefail -VERSION="0.0.4" +VERSION="0.3.0" # The directory containing install.sh is the permanent install location. PATCH_DIR="$(cd "$(dirname "$0")" && pwd)" _HOOK_COMMENT='TrueCloud provider patch (S3/B2)' +_NESTED_MARKER="$PATCH_DIR/nested_snapshots_enabled" + +# ── Options ─────────────────────────────────────────────────────────────────── +# Nested-dataset snapshot support is OPT-IN and off by default. It changes how +# backups read their source data, so an unattended re-run (e.g. after a +# `git pull`) must never flip it on or off by itself: with neither flag given, +# whatever was chosen previously is preserved. +_nested_choice="" + +usage() { + cat <&2 + echo "" >&2 + usage >&2 + exit 1 + ;; + esac + shift +done if [ ! -f "$PATCH_DIR/patch/apply.sh" ]; then echo "ERROR: patch files not found at $PATCH_DIR/patch/" >&2 @@ -104,6 +143,42 @@ if [ -f "$PATCH_DIR/disabled" ]; then echo "" fi +# ── Nested-dataset snapshot support (opt-in) ────────────────────────────────── + +case "$_nested_choice" in + on) + touch "$_NESTED_MARKER" + echo "Nested-dataset snapshots: ENABLED" + echo " The \"Take Snapshot\" option will be allowed on datasets that have" + echo " child datasets. Backups then read from a frozen, complete staging" + echo " tree instead of live files." + echo "" + echo " This changes how your backups read their source data. Verify that a" + echo " backup completes AND that its restic snapshot actually contains" + echo " child-dataset data before you rely on it." + ;; + off) + if [ -f "$_NESTED_MARKER" ]; then + rm -f "$_NESTED_MARKER" + echo "Nested-dataset snapshots: DISABLED (stock guard restored)." + echo " Any task that already has snapshot=true on a nested dataset will" + echo " fail validation on its next edit. Turn the option off on those" + echo " tasks, or re-run with --enable-nested-snapshots." + else + echo "Nested-dataset snapshots: already disabled." + fi + ;; + *) + if [ -f "$_NESTED_MARKER" ]; then + echo "Nested-dataset snapshots: enabled (unchanged)." + else + echo "Nested-dataset snapshots: disabled (default)." + echo " Enable with: bash install.sh --enable-nested-snapshots" + fi + ;; +esac +echo "" + # ── Apply now ───────────────────────────────────────────────────────────────── echo "Applying patches ..." diff --git a/patch/__pycache__/create_task.cpython-314.pyc b/patch/__pycache__/create_task.cpython-314.pyc deleted file mode 100644 index e606434..0000000 Binary files a/patch/__pycache__/create_task.cpython-314.pyc and /dev/null differ diff --git a/patch/apply.sh b/patch/apply.sh index e2d52d7..59a408c 100755 --- a/patch/apply.sh +++ b/patch/apply.sh @@ -7,13 +7,21 @@ # is already up and has already imported the stock modules — the on-disk # patch alone cannot reach the running process. # -# TrueNAS updates replace /usr/ entirely; this script re-applies two patches: +# TrueNAS updates replace /usr/ entirely; this script re-applies three patches: # # 1. Backend — b2.py and restic.py are patched directly in the overlay. # On a boot run, a single detached middlewared restart is scheduled # (Step 3) so the patched modules actually get loaded. # -# 2. Angular JS bundle — Widens the TrueCloud Backup credential dropdown +# 2. Nested-dataset snapshots — installs _truecloud_nested.py and patches +# plugins/cloud/{snapshot,crud}.py + plugins/cloud_backup/sync.py so the +# "Take Snapshot" option works on a dataset that has child datasets. +# Stock middleware refuses that config, because it points the backup tool +# at the PARENT's .zfs/snapshot/ where children are invisible — it would +# silently back up a near-empty tree. We stage a complete tree of +# per-dataset bind mounts and only then relax the guard. +# +# 3. Angular JS bundle — Widens the TrueCloud Backup credential dropdown # from Storj-only to include S3 and B2. Served from # disk per request, so no restart is needed for it. # @@ -24,7 +32,7 @@ # Derive PATCH_DIR from this script's location (parent of the patch/ directory). PATCH_DIR="$(cd "$(dirname "$0")/.." && pwd)" LOG="$PATCH_DIR/apply.log" -VERSION="0.2.1" +VERSION="0.3.0" # Rotate log at 512 KB to avoid unbounded growth on a system volume. # Keep two prior generations (.1 and .2) so the last three boots are always available. @@ -101,10 +109,20 @@ echo "Using Python: $PYTHON" # Combines what were previously four separate Python invocations into one to # avoid repeated interpreter startup overhead under the PREINIT timeout budget. +# The patch has two independent modules, and each retires on its own: +# +# providers — B2/S3 credentials for TrueCloud Backup (b2.py, restic.py, UI) +# nested — snapshots on datasets that have child datasets (cloud/*.py) +# +# TrueNAS may well ship one natively long before the other, so a single +# all-or-nothing kill switch would silently take a still-needed module down with +# the superseded one. Each module is detected separately and skipped on its own; +# the global kill switch fires only once BOTH are native. + _tc_info=$("$PYTHON" -c " import inspect, os, sys -result = {'native': 'no', 'site_pkg': '', 'mw_dir': ''} +result = {'native_b2': 'no', 'native_nested': 'no', 'site_pkg': '', 'mw_dir': ''} try: import middlewared @@ -118,6 +136,7 @@ except ImportError: except Exception: pass +# providers: does B2RcloneRemote already carry a real get_restic_config()? try: import middlewared.rclone.remote.b2 as _b2_mod from middlewared.rclone.remote.b2 import B2RcloneRemote @@ -129,21 +148,59 @@ try: except (OSError, TypeError): method_src = '' if 'NotImplementedError' not in method_src: - result['native'] = 'yes' + result['native_b2'] = 'yes' except Exception: pass -print(result['native']) +# nested: stock gates nested datasets with a validation in plugins/cloud/crud.py. +# We only ever append to that file, so the stock text survives our patch -- if the +# guard is gone, iX removed it, which means they implemented the traversal. +# If the file cannot be read we assume 'no' and keep patching: worst case the +# patch declines to apply and the option simply stays unavailable. +try: + crud = os.path.join(result['mw_dir'], 'plugins', 'cloud', 'crud.py') + with open(crud, encoding='utf-8', errors='replace') as fh: + if 'no further nesting' not in fh.read(): + result['native_nested'] = 'yes' +except Exception: + pass + +print(result['native_b2']) +print(result['native_nested']) print(result['site_pkg']) print(result['mw_dir']) -" 2>/dev/null || printf 'no\n\n\n') +" 2>/dev/null || printf 'no\nno\n\n\n') -_tc_native=$(printf '%s' "$_tc_info" | sed -n '1p') -SITE_PKG=$(printf '%s' "$_tc_info" | sed -n '2p') -_MW_DIR=$(printf '%s' "$_tc_info" | sed -n '3p') +_tc_native_b2=$(printf '%s' "$_tc_info" | sed -n '1p') +_tc_native_nested=$(printf '%s' "$_tc_info" | sed -n '2p') +SITE_PKG=$(printf '%s' "$_tc_info" | sed -n '3p') +_MW_DIR=$(printf '%s' "$_tc_info" | sed -n '4p') -if [ "$_tc_native" = "yes" ]; then - echo "NOTICE: TrueNAS now provides native B2 restic support — truecloud-patch is no longer needed." +# Nested support is opt-in; if it was never enabled, it cannot be the reason to +# keep the patch alive. +if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]; then + _NESTED_ENABLED=1 +else + _NESTED_ENABLED=0 +fi + +# Is either module still doing something useful? +_providers_needed=1 +[ "$_tc_native_b2" = "yes" ] && _providers_needed=0 + +_nested_needed=0 +if [ "$_NESTED_ENABLED" = "1" ] && [ "$_tc_native_nested" != "yes" ]; then + _nested_needed=1 +fi + +if [ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]; then + echo "NOTICE: Nothing left for truecloud-patch to do:" + [ "$_tc_native_b2" = "yes" ] && echo "NOTICE: - TrueNAS now provides native B2 restic support." + if [ "$_tc_native_nested" = "yes" ]; then + echo "NOTICE: - TrueNAS now handles snapshots on nested datasets natively." + elif [ "$_NESTED_ENABLED" = "0" ]; then + echo "NOTICE: - Nested-dataset snapshots are not enabled (opt-in)." + fi echo "NOTICE: Setting kill switch; patching will be skipped on all future boots." echo "NOTICE: Run the following to fully remove the patch:" echo "NOTICE: bash $PATCH_DIR/uninstall.sh" @@ -158,12 +215,21 @@ if [ "$_tc_native" = "yes" ]; then exit 0 fi +if [ "$_providers_needed" = "0" ]; then + echo "NOTICE: TrueNAS now provides native B2 restic support — the providers module" + echo "NOTICE: is superseded and will be skipped. The nested-snapshot module is still" + echo "NOTICE: active, so the patch stays installed." +fi +if [ "$_NESTED_ENABLED" = "1" ] && [ "$_tc_native_nested" = "yes" ]; then + echo "NOTICE: TrueNAS now handles nested-dataset snapshots natively — that module is" + echo "NOTICE: superseded and will be skipped. You can drop --enable-nested-snapshots." +fi + # ── Step 1: backend patch ───────────────────────────────────────────────────── echo "--- backend patch ---" -_b2_ok=0 -_restic_ok=0 +_backend_ok=0 if [ -z "$SITE_PKG" ]; then echo "WARNING: Cannot determine site-packages directory; skipping backend patch." @@ -175,12 +241,27 @@ elif [ -z "$_MW_DIR" ]; then else _B2_PY="$_MW_DIR/rclone/remote/b2.py" _RESTIC_PY="$_MW_DIR/plugins/cloud_backup/restic.py" + _CLOUD_DIR="$_MW_DIR/plugins/cloud" + _SYNC_PY="$_MW_DIR/plugins/cloud_backup/sync.py" + _NESTED_SRC="$PATCH_DIR/patch/truecloud_nested.py" - # ── patch b2.py + restic.py + hook_status.json (single subprocess) ────── - if "$PYTHON" - "$_B2_PY" "$_RESTIC_PY" "$PATCH_DIR/hook_status.json" << 'PYEOF' -import json, os, sys, time + # Each module is applied only if it is still needed. _providers_needed and + # _nested_needed were computed above (native-support detection + the opt-in + # marker), so one module going native never disables the other. + # ── patch b2.py + restic.py + nested-snapshot + hook_status.json ──────── + # (single subprocess: PREINIT has a tight timeout budget) + if "$PYTHON" - "$_B2_PY" "$_RESTIC_PY" "$PATCH_DIR/hook_status.json" \ + "$_CLOUD_DIR" "$_SYNC_PY" "$_NESTED_SRC" \ + "$_providers_needed" "$_nested_needed" "$_NESTED_ENABLED" \ + "$_tc_native_nested" << 'PYEOF' +import json, os, shutil, sys, time b2_path, restic_path, status_path = sys.argv[1], sys.argv[2], sys.argv[3] +cloud_dir, sync_path, nested_src = sys.argv[4], sys.argv[5], sys.argv[6] +providers_needed = sys.argv[7] == "1" +nested_needed = sys.argv[8] == "1" +nested_enabled = sys.argv[9] == "1" +nested_native = sys.argv[10] == "yes" B2_BLOCK = """ # TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh @@ -240,6 +321,133 @@ else: get_restic_config._truecloud_patched = True """ +# ── nested-dataset snapshot support ─────────────────────────────────────────── +# Stock middleware refuses `snapshot=true` on a path containing child datasets, +# because it points the backup tool at the PARENT dataset's .zfs/snapshot/, +# where child datasets are INVISIBLE -- it would silently back up a near-empty +# tree. That guard is correct. We implement the missing traversal (a staging +# tree of per-dataset bind mounts) and only then relax the guard. +# +# Fail-safe direction: if any of these three blocks fails to apply, the stock +# guard remains and the option simply stays unavailable. We never end up with +# the guard removed but the traversal missing -- that would be a silently empty +# backup, the worst possible outcome. + +SNAPSHOT_BLOCK = """ +# TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh +try: + from middlewared.plugins.cloud import _truecloud_nested as _tc_nested +except ImportError: + _tc_nested = None + +if _tc_nested is not None: + _tc_orig_create_snapshot = create_snapshot + + async def create_snapshot(middleware, path, name="cloud_task-onetime"): + # Stock takes the (already recursive) snapshot; we only replace the PATH. + snapshot, snap_path = await _tc_orig_create_snapshot(middleware, path, name) + + _logger = getattr(middleware, "logger", None) + try: + # Enumerate datasets AFTER the snapshot, never before. The snapshot is + # the point-in-time truth; a list read beforehand could miss a dataset + # created in the gap, which the recursive snapshot WOULD capture but + # our staging plan would not -- silently omitting it from the backup. + # Read afterwards, an unsnapshotted dataset instead trips the isdir() + # check in plan_staging and fails the run loudly. Loud beats silent. + datasets = await middleware.call( + "zfs.dataset.query", [["type", "=", "FILESYSTEM"]] + ) + dataset, nested = get_dataset_recursive(datasets, path) + + if not nested: + # No children: stock behaviour, untouched. Stock's `finally` owns + # the snapshot from here (its non-recursive delete is correct, + # because a non-nested snapshot has no children). + return snapshot, snap_path + + staging_root = await _tc_nested.stage_nested( + middleware, path, snapshot, + dataset["name"], dataset["properties"]["mountpoint"]["value"], + name, datasets, logger=_logger, + ) + except Exception: + # The snapshot exists, but this exception means sync.py never completes + # `snapshot, local_path = await create_snapshot(...)`, so its local + # `snapshot` stays None and its `finally` deletes NOTHING. Sweep the + # tree ourselves or leak the parent plus one snapshot per descendant + # dataset (160+ here) on every failed run. + await _tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger) + raise + + return snapshot, staging_root + + create_snapshot._truecloud_patched = True +""" + +CRUD_BLOCK = """ +# TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh +try: + from middlewared.plugins.cloud import _truecloud_nested as _tc_nested +except ImportError: + _tc_nested = None + +if _tc_nested is not None: + _tc_orig_validate = CloudTaskServiceMixin._validate + + async def _tc_validate(self, app, verrors, name, data): + await _tc_orig_validate(self, app, verrors, name, data) + + # Only cloud_backup: staging teardown is wired into cloud_backup.sync's + # finally. cloudsync would leak bind mounts, so leave its guard intact. + if getattr(getattr(self, "_config", None), "namespace", "") != "cloud_backup": + return + + # Drop ONLY the nested-dataset guard. If iX ever rewords the message the + # filter stops matching, the guard survives, and the option merely stays + # unavailable -- the safe direction to fail. + verrors.errors = [ + e for e in verrors.errors + if not ( + getattr(e, "attribute", "") == f"{name}.snapshot" + and "no further nesting" in getattr(e, "errmsg", "") + ) + ] + + CloudTaskServiceMixin._validate = _tc_validate + CloudTaskServiceMixin._validate._truecloud_patched = True +""" + +SYNC_BLOCK = """ +# TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh +try: + from middlewared.plugins.cloud import _truecloud_nested as _tc_nested +except ImportError: + _tc_nested = None + +if _tc_nested is not None: + _tc_orig_restic_backup = restic_backup + + async def restic_backup(middleware, job, cloud_backup, dry_run=False, rate_limit=None): + # Our bind mounts pin the ZFS snapshot, so stock's `finally` cannot + # destroy it (EBUSY) and logs one benign warning. We unmount here and + # then delete the snapshot for real. + try: + return await _tc_orig_restic_backup(middleware, job, cloud_backup, dry_run, rate_limit) + finally: + try: + await _tc_nested.cleanup_task( + middleware, + f"cloud_backup-{cloud_backup.get('id', 'onetime')}", + logger=getattr(middleware, "logger", None), + ) + except Exception as e: + middleware.logger.warning("truecloud-patch: staging cleanup failed: %r", e) + + restic_backup._truecloud_patched = True +""" + + def patch_file(path, block): with open(path, encoding="utf-8") as fh: content = fh.read() @@ -250,35 +458,102 @@ def patch_file(path, block): fh.write(base.rstrip("\n") + "\n" + block) b2_ok = restic_ok = False +nested_ok = False -if os.path.exists(b2_path): - try: - patch_file(b2_path, B2_BLOCK) - b2_ok = True - print(f"OK: Patched b2.py → {b2_path}") - except Exception as e: - print(f"WARNING: Failed to patch b2.py: {e}") +# ── module: providers (B2/S3) ───────────────────────────────────────────────── +# Skipped entirely once TrueNAS ships native B2 restic support. That must not +# take the nested module down with it, so the two are gated independently. +providers_detail = '' +if not providers_needed: + providers_detail = 'superseded: TrueNAS provides native B2 restic support' + print('INFO: Providers module skipped — TrueNAS now supports B2 natively.') else: - print(f"WARNING: b2.py not found at {b2_path}") + if os.path.exists(b2_path): + try: + patch_file(b2_path, B2_BLOCK) + b2_ok = True + print(f"OK: Patched b2.py → {b2_path}") + except Exception as e: + print(f"WARNING: Failed to patch b2.py: {e}") + else: + print(f"WARNING: b2.py not found at {b2_path}") -if os.path.exists(restic_path): - try: - patch_file(restic_path, RESTIC_BLOCK) - restic_ok = True - print(f"OK: Patched restic.py → {restic_path}") - except Exception as e: - print(f"WARNING: Failed to patch restic.py: {e}") + if os.path.exists(restic_path): + try: + patch_file(restic_path, RESTIC_BLOCK) + restic_ok = True + print(f"OK: Patched restic.py → {restic_path}") + except Exception as e: + print(f"WARNING: Failed to patch restic.py: {e}") + else: + print(f"WARNING: restic.py not found at {restic_path}") + providers_detail = ( + 'patched on disk in overlay at boot' if (b2_ok and restic_ok) + else 'b2.py/restic.py not found or write failed' + ) + +# ── module: nested-dataset snapshots ────────────────────────────────────────── +# Order matters: install the traversal machinery FIRST, relax the validation +# guard LAST. If anything fails partway, the guard is still in place and the +# option stays unavailable -- we never expose "guard removed, traversal missing". +nested_detail = '' +if not nested_enabled: + nested_detail = 'disabled (opt-in; enable with: install.sh --enable-nested-snapshots)' + print('INFO: Nested-dataset snapshot support is disabled (opt-in feature).') + print('INFO: Enable with: bash install.sh --enable-nested-snapshots') +elif nested_native: + nested_detail = 'superseded: TrueNAS handles nested-dataset snapshots natively' + print('INFO: Nested module skipped — TrueNAS now handles nesting natively.') +elif not nested_needed: + nested_detail = 'not needed' + print('INFO: Nested module skipped.') else: - print(f"WARNING: restic.py not found at {restic_path}") + try: + snapshot_py = os.path.join(cloud_dir, 'snapshot.py') + crud_py = os.path.join(cloud_dir, 'crud.py') + nested_dst = os.path.join(cloud_dir, '_truecloud_nested.py') + + missing = [p for p in (snapshot_py, crud_py, sync_path, nested_src) if not os.path.exists(p)] + if missing: + raise FileNotFoundError('missing: ' + ', '.join(missing)) + + shutil.copyfile(nested_src, nested_dst) # 1. traversal implementation + patch_file(snapshot_py, SNAPSHOT_BLOCK) # 2. build the staging tree + patch_file(sync_path, SYNC_BLOCK) # 3. tear it down afterwards + patch_file(crud_py, CRUD_BLOCK) # 4. ONLY NOW allow nested tasks + + nested_ok = True + nested_detail = 'nested-dataset snapshots enabled (staging tree)' + print(f'OK: Installed nested-snapshot support → {nested_dst}') + print(f'OK: Patched snapshot.py, sync.py, crud.py → {cloud_dir}') + except Exception as e: + nested_detail = f'not applied: {e}' + print(f'WARNING: Failed to apply nested-snapshot patch: {e}') + print('WARNING: Stock nesting guard remains; snapshot option stays unavailable') + print('WARNING: for nested datasets. Existing backups are unaffected.') patches = { + 'module.providers': { + 'ok': bool(b2_ok and restic_ok) or not providers_needed, + 'active': providers_needed, + 'detail': providers_detail, + }, + 'module.nested_snapshots': { + 'ok': nested_ok or not nested_needed, + 'active': nested_needed, + 'detail': nested_detail, + }, 'middlewared.rclone.remote.b2': { 'ok': b2_ok, - 'detail': 'patched on disk in overlay at boot' if b2_ok else 'b2.py not found or write failed', + 'detail': 'patched on disk in overlay at boot' if b2_ok else providers_detail, }, 'middlewared.plugins.cloud_backup.restic': { 'ok': restic_ok, - 'detail': 'patched on disk in overlay at boot' if restic_ok else 'restic.py not found or write failed', + 'detail': 'patched on disk in overlay at boot' if restic_ok else providers_detail, + }, + 'middlewared.plugins.cloud.nested_snapshot': { + 'ok': nested_ok, + 'detail': nested_detail, }, } payload = {'patched_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'patches': patches} @@ -291,35 +566,45 @@ try: except OSError as e: print(f'WARNING: Could not write hook_status.json: {e}') -sys.exit(0 if (b2_ok and restic_ok) else 1) +# Exit 0 only if every module that is still NEEDED applied cleanly. A module that +# was skipped (superseded or opt-out) is not a failure. +_providers_done = (not providers_needed) or (b2_ok and restic_ok) +_nested_done = (not nested_needed) or nested_ok +sys.exit(0 if (_providers_done and _nested_done) else 1) PYEOF then - _b2_ok=1 - _restic_ok=1 + _backend_ok=1 else - # Individual results already printed above; exit code 1 means at least one failed. - true + # Individual results already printed above; exit code 1 means at least + # one module that was still needed failed to apply. + _backend_ok=0 fi fi # ── Step 2: Angular bundle ──────────────────────────────────────────────────── +# Belongs to the providers module (it widens the credential dropdown), so it is +# skipped along with it once TrueNAS supports B2 natively. echo "--- UI patch ---" -# Ensure the webui directory is writable before patch_ui.py tries to create a -# backup and write the patched bundle. On immutable OS we mount an overlay. -_webui_dir="" -for _d in /usr/share/truenas/webui /usr/share/truenas-ui /var/www/truenas; do - if [ -d "$_d" ]; then - _webui_dir="$_d" - break +if [ "$_providers_needed" = "0" ]; then + echo "Skipped — providers module superseded by native B2 support." +else + # Ensure the webui directory is writable before patch_ui.py tries to create a + # backup and write the patched bundle. On immutable OS we mount an overlay. + _webui_dir="" + for _d in /usr/share/truenas/webui /usr/share/truenas-ui /var/www/truenas; do + if [ -d "$_d" ]; then + _webui_dir="$_d" + break + fi + done + if [ -n "$_webui_dir" ]; then + _ensure_writable "$_webui_dir" "ui" || true # non-fatal; patch_ui.py reports the error fi -done -if [ -n "$_webui_dir" ]; then - _ensure_writable "$_webui_dir" "ui" || true # non-fatal; patch_ui.py reports the error -fi -"$PYTHON" "$PATCH_DIR/patch/patch_ui.py" || echo "WARNING: patch_ui.py exited non-zero; UI dropdown may still show Storj only." + "$PYTHON" "$PATCH_DIR/patch/patch_ui.py" || echo "WARNING: patch_ui.py exited non-zero; UI dropdown may still show Storj only." +fi # ── Step 3: deferred middlewared restart (boot runs only) ───────────────────── # At boot this script is spawned by middlewared, which already imported the @@ -340,9 +625,14 @@ fi echo "--- deferred restart ---" +# Restart when ANY still-needed backend module landed. Keying this off the +# providers module alone would skip the restart on a box where B2 has gone native +# but the nested module was freshly patched — leaving it on disk and never loaded. if ! grep -aq middlewared "/proc/$PPID/cmdline" 2>/dev/null; then echo "Manual run (parent is not middlewared) — no restart scheduled." -elif [ "$_b2_ok" != "1" ] || [ "$_restic_ok" != "1" ]; then +elif [ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]; then + echo "No backend module active — no restart scheduled (nothing new to load)." +elif [ "${_backend_ok:-0}" != "1" ]; then echo "Backend patch incomplete — no restart scheduled (nothing new to load)." else # A failed unit from an earlier attempt this boot would block systemd-run. diff --git a/patch/patch_ui.py b/patch/patch_ui.py index 74cc673..78978fe 100644 --- a/patch/patch_ui.py +++ b/patch/patch_ui.py @@ -19,6 +19,7 @@ Safe to run multiple times — a marker string detects an already-patched file. Exits 0 in all cases (warnings are printed to stdout and logged by apply.sh). """ +import contextlib import os import re import shutil @@ -131,10 +132,8 @@ def main(): os.replace(tmp, path) except OSError as exc: print(f"[truecloud-patch] ERROR: Could not write {path}: {exc}") - try: + with contextlib.suppress(OSError): os.unlink(tmp) - except OSError: - pass return print(f"[truecloud-patch] UI bundle patched ({count} replacement(s)): {path}") diff --git a/patch/truecloud_nested.py b/patch/truecloud_nested.py new file mode 100644 index 0000000..df02c6a --- /dev/null +++ b/patch/truecloud_nested.py @@ -0,0 +1,497 @@ +"""Nested-dataset snapshot support for TrueCloud Backup. + +Why this exists +--------------- +Stock TrueNAS refuses ``snapshot = true`` when the backup path contains child +datasets:: + + This option is only available for datasets that have no further nesting + +That guard is *correct* and it is not laziness. ``plugins/cloud/snapshot.py`` +already takes a **recursive** ZFS snapshot, but it then points the backup tool +at the *parent* dataset's ``.zfs/snapshot//`` directory -- and ZFS does +not expose child datasets through a parent's snapshot directory:: + + /mnt/Tap/.zfs/snapshot//apps/ -> 0 entries + /mnt/Tap/apps/lidarr/config/.zfs/snapshot// -> the real data + +So without the guard, the backup tool would walk a near-empty tree, report +SUCCESS, and upload almost nothing. A backup that lies about succeeding is the +worst failure a backup system can have, so middleware refuses the config +instead. + +This module implements the missing half: after the (already recursive) snapshot +is taken, every descendant dataset's *own* ``.zfs/snapshot/`` directory is +bind-mounted into a staging tree that mirrors the original layout. The backup +tool is then pointed at the staging root, which is a complete, consistent, +point-in-time view of the whole subtree. + +Cardinal safety rule +-------------------- +**If the tree cannot be staged completely, fail loudly.** Never return a partial +tree. Silently backing up an incomplete tree is precisely the failure this +feature exists to prevent, and it would be worse than not having the feature. + +Snapshot lifecycle -- read this before changing anything +-------------------------------------------------------- +``zfs.snapshot.delete`` defaults to ``recursive=False``, and stock +``restic_backup()`` calls it with no options. Stock gets away with that because +its validation means ``recursive`` is never actually True in the field. Enabling +nested datasets makes recursive snapshots real, so the parent +(``Tap@snap``) has one child snapshot per descendant dataset (160+ here). +Deleting only the parent would orphan every child on **every successful run**. + +Therefore this module owns the whole lifecycle: + +* :func:`delete_snapshot_tree` sweeps the parent *and* every child snapshot, and + is idempotent -- it copes with stock's ``finally`` having already removed the + parent. +* The snapshot name is recorded in a sidecar file next to the staging root, not + only in memory, so a middlewared restart mid-backup cannot orphan it. +* Bind-mounting ``.zfs/snapshot/`` pins the snapshot, so stock's delete + fails with EBUSY and logs one benign warning; we unmount and then sweep. +""" + +from __future__ import annotations + +import contextlib +import os +import stat +import subprocess + +__all__ = [ + "ACTIVE", + "STAGING_BASE", + "StagingError", + "apply_plan", + "cleanup_task", + "current_mounts_under", + "delete_snapshot_tree", + "plan_staging", + "sidecar_for", + "snapshot_tree_names", + "stage_nested", + "staging_root_for", + "teardown", + "verify_staged", +] + +#: Where staging trees are assembled. tmpfs; bind mounts consume no space. +STAGING_BASE = "/run/truecloud-nested" + +#: staging_root -> zfs snapshot name. A cache; the sidecar file is the source of +#: truth, so that a middlewared restart cannot orphan a snapshot. +ACTIVE: dict[str, str] = {} + + +class StagingError(Exception): + """Staging could not produce a complete tree. The backup must not proceed.""" + + +# ── pure helpers ────────────────────────────────────────────────────────────── + + +def staging_root_for(name: str, base: str | None = None) -> str: + """Stable staging root for a task name (e.g. ``cloud_backup-5``). + + ``base`` defaults to :data:`STAGING_BASE` at CALL time, not at import time -- + a ``base=STAGING_BASE`` default would freeze the value into the function + object and silently ignore any later override. + """ + if base is None: + base = STAGING_BASE + safe = "".join(c if (c.isalnum() or c in "-_.") else "_" for c in name) + # A component of "." or ".." would escape STAGING_BASE once joined. + if not safe or safe.strip(".") == "": + safe = "task" + return os.path.join(base, safe) + + +def sidecar_for(staging_root: str) -> str: + """Path of the file recording which ZFS snapshot a staging tree pins.""" + return staging_root + ".snapshot" + + +def _write_sidecar(staging_root: str, snapshot: str) -> None: + """Record the pinned snapshot on disk. Blocking; call via run_in_thread.""" + with contextlib.suppress(OSError): + os.makedirs(os.path.dirname(staging_root), exist_ok=True) + with open(sidecar_for(staging_root), "w", encoding="utf-8") as fh: + fh.write(snapshot) + + +def _read_sidecar(staging_root: str) -> str | None: + """The snapshot a previous run recorded here, if any.""" + try: + with open(sidecar_for(staging_root), encoding="utf-8") as fh: + return fh.read().strip() or None + except OSError: + return None + + +def _remove_sidecar(staging_root: str) -> None: + with contextlib.suppress(OSError): + os.unlink(sidecar_for(staging_root)) + + +def _depth(path: str) -> int: + return len([p for p in path.split("/") if p]) + + +def snapshot_tree_names(snapshot: str, all_names) -> list[str]: + """Every snapshot produced by ``zfs snapshot -r @``. + + That is the parent plus one per descendant dataset, all sharing the same + name after the ``@``. Pure, so the sweep logic is testable without ZFS. + """ + dataset, _, snapname = snapshot.partition("@") + if not snapname: + return [] + parent = f"{dataset}@{snapname}" + prefix = dataset + "/" + suffix = "@" + snapname + return [ + n for n in all_names + if n == parent or (n.startswith(prefix) and n.endswith(suffix)) + ] + + +def _probe_snapdir(path): + """Classify a snapshot directory: ``ok``, ``missing``, or why it is unusable. + + ``os.path.isdir()`` collapses "does not exist" and "cannot stat" into the + same ``False``, so an EACCES would report itself as "has no snapshot" and + send someone hunting for a snapshot that is sitting right there. Both cases + still abort the backup -- but it has to say which one. + """ + try: + st = os.stat(path) + except FileNotFoundError: + return "missing" + except OSError as e: + return f"cannot be read ({e.strerror})" + return "ok" if stat.S_ISDIR(st.st_mode) else "is not a directory" + + +def plan_staging(base_dataset, base_mountpoint, path, snapshot_name, datasets, + staging_root, probe=_probe_snapdir): + """Compute the bind-mount plan for staging a nested tree. Pure function. + + ``datasets`` is a list of dicts shaped like ``zfs.dataset.query`` results: + ``{"name": str, "properties": {"mountpoint": {"value": str}, + "mounted": {"value": "yes"|"no"}}}``. + + Returns ``(mounts, skipped)`` where ``mounts`` is an ordered list of + ``(source, target)`` pairs (parents before children) and ``skipped`` is a + list of ``(dataset_name, reason)`` covering only datasets that are *in + scope* -- i.e. descendants of ``base_dataset``. Datasets elsewhere on the + system are ignored silently; reporting them would bury the ones that matter. + + Raises StagingError if an in-scope descendant holds data we would otherwise + silently omit. + """ + def snapdir(mountpoint): + return os.path.join(mountpoint, ".zfs", "snapshot", snapshot_name) + + # Root of the staging tree: the backup path as seen inside the base + # dataset's own snapshot. + rel = os.path.relpath(path, base_mountpoint) + root_src = snapdir(base_mountpoint) + if rel != ".": + root_src = os.path.join(root_src, rel) + + mounts = [(root_src, staging_root)] + skipped = [] + + ds_prefix = base_dataset.rstrip("/") + "/" + path_prefix = path.rstrip("/") + "/" + + for ds in datasets: + name = ds.get("name", "") + # Scope by DATASET NAME, not mountpoint: a dataset with no mountpoint + # cannot be scoped by path, and scoping by path first would drag in + # every mountpoint-less dataset on the box (all of Tank/.system/*, ...). + if not name.startswith(ds_prefix): + continue + + props = ds.get("properties", {}) + mp = props.get("mountpoint", {}).get("value", "") + + if not mp or mp in ("none", "legacy", "-"): + skipped.append((name, f"mountpoint is {mp or 'unset'}")) + continue + + if not mp.startswith(path_prefix): + # A descendant dataset mounted outside the backed-up path is + # genuinely not part of this tree. Not an omission. + continue + + if props.get("mounted", {}).get("value", "yes") == "no": + # An unmounted (e.g. locked/encrypted) dataset contributes nothing to + # the live tree either, so skipping matches stock semantics -- but it + # is a real gap and must be visible, never silent. + skipped.append((name, "dataset is not mounted (locked/encrypted?)")) + continue + + src = snapdir(mp) + status = probe(src) + if status != "ok": + # Either the recursive snapshot missed this dataset, or we cannot read + # it. Either way its data would be silently omitted. Refuse -- but say + # WHICH, because "no snapshot" and "permission denied" send you to + # completely different places. + detail = ( + f"has no snapshot {snapshot_name!r}" if status == "missing" + else f"snapshot {snapshot_name!r} {status}" + ) + raise StagingError( + f"dataset {name!r} {detail} at {src!r}; " + f"refusing to back up an incomplete tree" + ) + + mounts.append((src, os.path.join(staging_root, os.path.relpath(mp, path)))) + + # Parents before children, so each mountpoint exists before we mount onto it. + mounts.sort(key=lambda m: _depth(m[1])) + return mounts, skipped + + +def current_mounts_under(root, mounts_file="/proc/self/mounts"): + """Mountpoints at or under ``root``, deepest first. Used for teardown.""" + found = [] + try: + with open(mounts_file, encoding="utf-8") as fh: + for line in fh: + parts = line.split() + if len(parts) < 2: + continue + mp = parts[1].replace("\\040", " ").replace("\\011", "\t") + if mp == root or mp.startswith(root.rstrip("/") + "/"): + found.append(mp) + except OSError: + return [] + found.sort(key=_depth, reverse=True) + return found + + +# ── mount / unmount ─────────────────────────────────────────────────────────── + + +def _run(cmd): + return subprocess.run(cmd, capture_output=True, text=True, check=False) + + +def apply_plan(mounts, runner=_run, isdir=os.path.isdir): + """Execute the bind-mount plan. Blocking; call via ``run_in_thread``. + + Raises StagingError on the first failure, after rolling back what was + mounted -- a half-built tree must never be handed to the backup tool. + """ + if not mounts: + raise StagingError("empty staging plan") + + staging_root = mounts[0][1] + done = [] + try: + os.makedirs(staging_root, exist_ok=True) + for src, target in mounts: + if not isdir(target): + # Child mountpoint dirs come from the parent snapshot, which is + # read-only -- we cannot mkdir them. Only the root is ours. + raise StagingError(f"staging target {target!r} does not exist") + res = runner(["mount", "--bind", src, target]) + if res.returncode != 0: + raise StagingError( + f"bind-mount {src!r} -> {target!r} failed: " + f"{(res.stderr or '').strip() or res.returncode}" + ) + done.append(target) + except Exception: + for target in reversed(done): + runner(["umount", "-l", target]) + with contextlib.suppress(OSError): + os.rmdir(staging_root) + raise + return staging_root + + +def verify_staged(mounts, ismount=os.path.ismount, listdir=os.listdir): + """Assert the staged tree is real and complete. Raises StagingError if not. + + This is the anti-regression guard: it is what stops this feature from ever + degrading back into the silently-empty backup that the stock validation + refuses to allow. + """ + if not mounts: + raise StagingError("nothing was staged") + + staging_root = mounts[0][1] + for _src, target in mounts: + if not ismount(target): + raise StagingError(f"staging target {target!r} is not a mountpoint") + + try: + if not listdir(staging_root): + raise StagingError(f"staging root {staging_root!r} is empty") + except OSError as e: + raise StagingError(f"staging root {staging_root!r} unreadable: {e}") from e + + return True + + +def teardown(staging_root, runner=_run, mounts_file="/proc/self/mounts"): + """Unmount the staging tree (deepest first) and remove the root. + + Idempotent, and does not depend on an in-memory plan -- so it also cleans up + leftovers from a crashed run. + """ + errors = [] + for mp in current_mounts_under(staging_root, mounts_file=mounts_file): + res = runner(["umount", mp]) + if res.returncode != 0: + res = runner(["umount", "-l", mp]) # lazy: better than leaking + if res.returncode != 0: + errors.append(f"{mp}: {(res.stderr or '').strip()}") + with contextlib.suppress(OSError): + os.rmdir(staging_root) + return errors + + +# ── async orchestration (middleware is duck-typed; no middlewared import) ───── + + +async def delete_snapshot_tree(middleware, snapshot, logger=None): + """Delete the parent snapshot AND every child created by ``zfs snapshot -r``. + + ``zfs.snapshot.delete`` is non-recursive by default and stock calls it with + no options, so relying on stock would orphan one snapshot per descendant + dataset on every run. Idempotent: tolerates the parent already being gone + (stock's ``finally`` may have won the race once our mounts were released). + """ + dataset = snapshot.partition("@")[0] + + try: + snaps = await middleware.call( + "zfs.snapshot.query", [["name", "^", dataset]], {"select": ["name"]} + ) + # An empty result means the tree is already gone -- delete nothing, and + # do not fall back to the parent, which would only log a spurious + # "does not exist" warning on every clean run. + names = snapshot_tree_names(snapshot, [s["name"] for s in snaps]) + except Exception as e: # noqa: BLE001 - fall back to at least the parent + if logger: + logger.warning( + "truecloud-patch: could not enumerate snapshot tree for %s: %r", + snapshot, e, + ) + names = [snapshot] + + for name in names: + try: + await middleware.call("zfs.snapshot.delete", name) + except Exception as e: # noqa: BLE001 - already gone is fine + if logger: + logger.warning( + "truecloud-patch: could not delete snapshot %s: %r", name, e + ) + + +async def stage_nested(middleware, path, snapshot, base_dataset, base_mountpoint, + task_name, datasets, logger=None): + """Build a complete staging tree for `path` from the already-taken `snapshot`. + + `snapshot` is a full ZFS snapshot name ("Tap@cloud_backup-5-2026..."). + + `datasets` is the FILESYSTEM dataset list. **It MUST have been enumerated + AFTER `snapshot` was taken.** A list read beforehand can miss a dataset + created in the gap: the recursive snapshot would capture it, but the staging + plan would not, and its data would be silently omitted from the backup. + Enumerated afterwards, an unsnapshotted dataset instead trips the isdir() + check in plan_staging and fails the run loudly. + + Returns the staging root to hand to the backup tool. + + Raises StagingError if the tree cannot be staged completely -- the caller + must let that propagate so the backup fails instead of silently uploading a + partial tree. The caller is responsible for deleting `snapshot` in that case + (see SNAPSHOT_BLOCK in apply.sh). + """ + snapshot_name = snapshot.split("@", 1)[1] + staging_root = staging_root_for(task_name) + + # A previous run may have crashed mid-flight; never build on top of that. + await middleware.run_in_thread(teardown, staging_root) + + # ...and if it left a sidecar behind, that snapshot tree is still on disk and + # nothing else will ever reclaim it. Sweep it before we overwrite the record, + # or a single crashed run orphans 160+ snapshots permanently. + stale = await middleware.run_in_thread(_read_sidecar, staging_root) + if stale and stale != snapshot: + if logger: + logger.warning( + "truecloud-patch: reclaiming snapshot tree from an earlier " + "interrupted run: %s", stale, + ) + await delete_snapshot_tree(middleware, stale, logger=logger) + + # Record the snapshot BEFORE mounting anything, not after. middlewared can + # die at any point (this patch even schedules a restart at boot), and the + # sidecar is the only thing that survives it -- an in-process dict would take + # the sole record of a 160-snapshot tree with it. Writing it after apply_plan + # would leave exactly the crash window the sidecar exists to close. + await middleware.run_in_thread(_write_sidecar, staging_root, snapshot) + + try: + mounts, skipped = await middleware.run_in_thread( + plan_staging, base_dataset, base_mountpoint, path, snapshot_name, + datasets, staging_root, + ) + if logger: + for name, reason in skipped: + logger.warning( + "truecloud-patch: not staging dataset %r: %s", name, reason + ) + + await middleware.run_in_thread(apply_plan, mounts) + await middleware.run_in_thread(verify_staged, mounts) + except Exception: + await middleware.run_in_thread(teardown, staging_root) + await middleware.run_in_thread(_remove_sidecar, staging_root) + raise + + ACTIVE[staging_root] = snapshot + + if logger: + logger.info( + "truecloud-patch: staged %d dataset(s) from %s at %s", + len(mounts), snapshot, staging_root, + ) + return staging_root + + +async def cleanup_task(middleware, task_name, logger=None): + """Tear down a task's staging tree and delete the snapshot it pinned. + + Safe to call unconditionally: a no-op when the task was never staged. + """ + staging_root = staging_root_for(task_name) + sidecar = sidecar_for(staging_root) + + snapshot = ACTIVE.pop(staging_root, None) + if snapshot is None: + # Sidecar survives a middlewared restart; ACTIVE does not. + snapshot = _read_sidecar(staging_root) + + if snapshot is None and not os.path.isdir(staging_root): + return # never staged; nothing to do + + errors = await middleware.run_in_thread(teardown, staging_root) + if errors and logger: + for err in errors: + logger.warning("truecloud-patch: staging teardown: %s", err) + + if snapshot is not None: + await delete_snapshot_tree(middleware, snapshot, logger=logger) + + with contextlib.suppress(OSError): + os.unlink(sidecar) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dbd62e5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +# Tooling config only — this project is not a Python package. The patch modules +# are copied into middlewared's site-packages by patch/apply.sh at boot. + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "W", "B", "UP", "SIM"] +ignore = [ + "E501", # long lines in explanatory comments are fine +] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/recover.sh b/recover.sh index 37cf741..cf54b64 100755 --- a/recover.sh +++ b/recover.sh @@ -17,7 +17,7 @@ # bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh # systemctl restart middlewared -VERSION="0.0.4" +VERSION="0.3.0" PATCH_DIR="$(cd "$(dirname "$0")" && pwd)" diff --git a/tests/test_apply_blocks.py b/tests/test_apply_blocks.py new file mode 100644 index 0000000..c59e652 --- /dev/null +++ b/tests/test_apply_blocks.py @@ -0,0 +1,190 @@ +"""The *_BLOCK strings in apply.sh are Python source injected into middleware. + +A syntax error in one of them would be appended to a live middlewared module and +break the box at boot. They are string literals, so nothing type-checks them -- +these tests do. +""" + +import ast +import os +import re + +import pytest + +APPLY_SH = os.path.join(os.path.dirname(__file__), "..", "patch", "apply.sh") + +EXPECTED_BLOCKS = { + "B2_BLOCK", + "RESTIC_BLOCK", + "SNAPSHOT_BLOCK", + "CRUD_BLOCK", + "SYNC_BLOCK", +} + + +def heredoc_source(): + with open(APPLY_SH, encoding="utf-8") as fh: + src = fh.read() + m = re.search(r"<< 'PYEOF'\n(.*?)\nPYEOF", src, re.S) + assert m, "could not find the PYEOF heredoc in apply.sh" + return m.group(1) + + +def extract_blocks(): + tree = ast.parse(heredoc_source()) + blocks = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for tgt in node.targets: + if ( + isinstance(tgt, ast.Name) + and tgt.id.endswith("_BLOCK") + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ): + blocks[tgt.id] = node.value.value + return blocks + + +def test_heredoc_itself_compiles(): + compile(heredoc_source(), "apply.sh:PYEOF", "exec") + + +def test_all_expected_blocks_present(): + assert set(extract_blocks()) == EXPECTED_BLOCKS + + +@pytest.mark.parametrize("name", sorted(EXPECTED_BLOCKS)) +def test_injected_block_is_valid_python(name): + block = extract_blocks()[name] + compile(block, f"apply.sh:{name}", "exec") + + +@pytest.mark.parametrize("name", sorted(EXPECTED_BLOCKS)) +def test_injected_block_carries_the_idempotency_marker(name): + # patch_file() truncates each target file at "\n# TRUECLOUD_PATCH" before + # re-appending, so every block must start with that marker or repeated runs + # would stack duplicate copies into the middleware module. + assert extract_blocks()[name].lstrip("\n").startswith("# TRUECLOUD_PATCH") + + +@pytest.mark.parametrize("name", ["SNAPSHOT_BLOCK", "CRUD_BLOCK", "SYNC_BLOCK"]) +def test_nested_blocks_degrade_safely_without_the_module(name): + # If _truecloud_nested failed to install, every nested block must no-op. + # Critically this includes CRUD_BLOCK: relaxing the guard without the + # traversal in place would mean silently-empty backups. + block = extract_blocks()[name] + assert "_tc_nested = None" in block + assert "if _tc_nested is not None:" in block + + +class TestSnapshotLeak: + """zfs.snapshot.delete is non-recursive and stock calls it with no options. + + A recursive snapshot has one child per descendant dataset (160+ here), so + every path that creates one must also sweep the whole tree. + """ + + def test_staging_failure_deletes_the_snapshot_tree(self): + # On a staging failure, sync.py's `snapshot, local_path = await + # create_snapshot(...)` never completes, so its local `snapshot` stays + # None and its finally deletes nothing. We must sweep it ourselves. + block = extract_blocks()["SNAPSHOT_BLOCK"] + assert "except Exception:" in block + assert "delete_snapshot_tree" in block + assert "raise" in block + + def test_sync_block_cleans_up_on_every_path(self): + block = extract_blocks()["SYNC_BLOCK"] + assert "finally:" in block + assert "cleanup_task" in block + + +def test_crud_block_is_scoped_to_cloud_backup(): + # cloudsync has no staging teardown wired in, so its guard must stay. + assert '!= "cloud_backup"' in extract_blocks()["CRUD_BLOCK"] + + +class TestIndependentModules: + """The two modules must retire independently. + + TrueNAS may ship native B2 support long before (or after) it handles nested + datasets. A single all-or-nothing kill switch would silently take a + still-needed module down with the superseded one. + """ + + def _sh(self): + with open(APPLY_SH, encoding="utf-8") as fh: + return fh.read() + + def test_native_support_is_detected_per_module(self): + sh = self._sh() + assert "native_b2" in sh + assert "native_nested" in sh + assert "no further nesting" in sh, "nested native-support probe" + + def test_kill_switch_only_when_both_modules_are_done(self): + sh = self._sh() + assert '[ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]' in sh + # ...and that is the only place the kill switch is actually set. (Ignore + # comment lines, which mention the same path.) + code = [ln for ln in sh.splitlines() if not ln.lstrip().startswith("#")] + sets = [ln for ln in code if 'touch "$PATCH_DIR/disabled"' in ln] + assert len(sets) == 1, f"kill switch set in {len(sets)} places" + + def test_each_module_is_gated_separately(self): + src = heredoc_source() + assert "if not providers_needed:" in src + assert "elif nested_native:" in src + + def test_ui_patch_is_tied_to_the_providers_module(self): + # The UI change widens the credential dropdown; it is meaningless once B2 + # is native, but must NOT be skipped merely because nested is off. + sh = self._sh() + i = sh.index("--- UI patch ---") + assert '[ "$_providers_needed" = "0" ]' in sh[i:i + 400] + + def test_restart_fires_when_any_needed_module_landed(self): + # Keying the restart off providers alone would leave a freshly-patched + # nested module on disk and never loaded on a native-B2 box. + sh = self._sh() + i = sh.index("--- deferred restart ---") + tail = sh[i:] + assert '_backend_ok' in tail + assert '"$_b2_ok"' not in tail + + +class TestOptIn: + """Nested-snapshot support must be opt-in and must never self-enable.""" + + def test_heredoc_gates_on_the_opt_in_flag(self): + src = heredoc_source() + assert re.search(r"nested_enabled = sys\.argv\[\d+\] == \"1\"", src) + assert "if not nested_enabled:" in src + + def test_apply_sh_reads_the_marker_file(self): + with open(APPLY_SH, encoding="utf-8") as fh: + sh = fh.read() + assert 'if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]' in sh + assert '"$_NESTED_ENABLED"' in sh + + def test_patching_is_skipped_entirely_when_disabled(self): + # The guard-relaxing crud.py patch must be inside the enabled branch. + src = heredoc_source() + gate = src.index("if not nested_enabled:") + crud = src.index("patch_file(crud_py, CRUD_BLOCK)") + assert gate < crud, "crud.py patch must sit inside the opt-in branch" + + +def test_guard_is_relaxed_only_after_traversal_is_installed(): + # Ordering in apply.sh is a safety property: copy module -> patch snapshot.py + # -> patch sync.py -> patch crud.py. crud.py (which unlocks the feature) must + # come last, so a partial failure never leaves "guard removed, traversal gone". + src = heredoc_source() + order = [ + src.index("shutil.copyfile(nested_src, nested_dst)"), + src.index("patch_file(snapshot_py, SNAPSHOT_BLOCK)"), + src.index("patch_file(sync_path, SYNC_BLOCK)"), + src.index("patch_file(crud_py, CRUD_BLOCK)"), + ] + assert order == sorted(order), "crud.py must be patched last" diff --git a/tests/test_truecloud_nested.py b/tests/test_truecloud_nested.py new file mode 100644 index 0000000..217aa6f --- /dev/null +++ b/tests/test_truecloud_nested.py @@ -0,0 +1,514 @@ +"""Tests for nested-dataset snapshot staging. + +Two rules are under test above all else: + +1. A tree that cannot be staged completely must fail LOUDLY. A silently + incomplete backup is the exact failure that stock TrueNAS's "no further + nesting" guard exists to prevent. + +2. Every snapshot we cause to exist must be cleaned up. ``zfs.snapshot.delete`` + is non-recursive by default and stock calls it with no options, so a + recursive snapshot would otherwise orphan one snapshot per descendant dataset + on EVERY run. +""" + +import asyncio +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "patch")) + +from truecloud_nested import ( # noqa: E402 + ACTIVE, + StagingError, + apply_plan, + cleanup_task, + current_mounts_under, + delete_snapshot_tree, + plan_staging, + sidecar_for, + snapshot_tree_names, + staging_root_for, + teardown, + verify_staged, +) + +SNAP = "cloud_backup-5-20260712030000" +ROOT = "/run/truecloud-nested/cloud_backup-5" + + +def ds(name, mountpoint, mounted="yes"): + return { + "name": name, + "properties": { + "mountpoint": {"value": mountpoint}, + "mounted": {"value": mounted}, + }, + } + + +# Mirrors the real layout: apps are datasets, several with their own children. +DATASETS = [ + ds("Tap", "/mnt/Tap"), + ds("Tap/apps", "/mnt/Tap/apps"), + ds("Tap/apps/lidarr", "/mnt/Tap/apps/lidarr"), + ds("Tap/apps/lidarr/config", "/mnt/Tap/apps/lidarr/config"), + ds("Tap/apps/immich", "/mnt/Tap/apps/immich"), + ds("Tap/apps/immich/pgdata", "/mnt/Tap/apps/immich/pgdata"), +] + + +def yes(_path): + return "ok" + + +def plan(datasets=DATASETS, base_dataset="Tap", base_mp="/mnt/Tap", + path="/mnt/Tap", probe=yes): + return plan_staging(base_dataset, base_mp, path, SNAP, datasets, ROOT, probe=probe) + + +class TestPlanStaging: + def test_stages_every_descendant_dataset(self): + mounts, skipped = plan() + assert skipped == [] + assert len(mounts) == 6 # root + 5 descendants + assert mounts[0] == (f"/mnt/Tap/.zfs/snapshot/{SNAP}", ROOT) + + by_target = {t: s for s, t in mounts} + assert by_target[f"{ROOT}/apps"] == f"/mnt/Tap/apps/.zfs/snapshot/{SNAP}" + assert by_target[f"{ROOT}/apps/immich/pgdata"] == ( + f"/mnt/Tap/apps/immich/pgdata/.zfs/snapshot/{SNAP}" + ) + + def test_parents_are_mounted_before_children(self): + # A child's mountpoint dir only exists inside its parent's snapshot, so + # mounting a child first would fail. + mounts, _ = plan() + seen = set() + for _src, target in mounts: + if target != ROOT: + assert os.path.dirname(target) in seen + seen.add(target) + + def test_backup_path_below_dataset_root(self): + mounts, _ = plan(base_dataset="Tap/apps", base_mp="/mnt/Tap/apps", + path="/mnt/Tap/apps") + assert mounts[0] == (f"/mnt/Tap/apps/.zfs/snapshot/{SNAP}", ROOT) + targets = [t for _s, t in mounts] + assert f"{ROOT}/lidarr" in targets + assert f"{ROOT}/apps/lidarr" not in targets + + def test_base_dataset_is_not_a_descendant_of_itself(self): + mounts, _ = plan(datasets=[ds("Tap", "/mnt/Tap")]) + assert len(mounts) == 1 + + +class TestScoping: + def test_unrelated_datasets_are_ignored_silently(self): + # Regression: scoping by mountpoint first dragged in every + # mountpoint-less dataset on the box (all of Tank/.system/*), burying the + # warnings that actually matter. + noisy = DATASETS + [ + ds("Tank/.system", "none"), + ds("Tank/.system/cores", "legacy"), + ds("Tank/backups", "/mnt/Tank/backups"), + ] + mounts, skipped = plan(datasets=noisy) + assert len(mounts) == 6 + assert skipped == [], "datasets outside the base dataset must not be reported" + + def test_in_scope_dataset_without_mountpoint_is_reported(self): + datasets = DATASETS + [ds("Tap/apps/weird", "none")] + _mounts, skipped = plan(datasets=datasets) + assert ("Tap/apps/weird", "mountpoint is none") in skipped + + def test_unmounted_dataset_is_skipped_but_never_silently(self): + datasets = DATASETS + [ds("Tap/apps/vault", "/mnt/Tap/apps/vault", mounted="no")] + mounts, skipped = plan(datasets=datasets) + assert f"{ROOT}/apps/vault" not in [t for _s, t in mounts] + assert ("Tap/apps/vault", "dataset is not mounted (locked/encrypted?)") in skipped + + def test_descendant_mounted_outside_the_path_is_not_an_omission(self): + datasets = DATASETS + [ds("Tap/elsewhere", "/mnt/other")] + mounts, skipped = plan(datasets=datasets) + assert len(mounts) == 6 + assert skipped == [] + + +class TestSilentOmissionGuard: + """The whole point of the feature. These are the tests that matter.""" + + @staticmethod + def _missing_pgdata(path): + return "missing" if "/mnt/Tap/apps/immich/pgdata/" in path else "ok" + + @staticmethod + def _denied_pgdata(path): + if "/mnt/Tap/apps/immich/pgdata/" in path: + return "cannot be read (Permission denied)" + return "ok" + + def test_missing_snapshot_on_descendant_raises(self): + with pytest.raises(StagingError, match="incomplete tree"): + plan(probe=self._missing_pgdata) + + def test_error_names_the_offending_dataset(self): + with pytest.raises(StagingError, match="Tap/apps/immich/pgdata"): + plan(probe=self._missing_pgdata) + + def test_missing_and_unreadable_are_reported_differently(self): + # os.path.isdir() collapses both into False, which would report a + # permission problem as "has no snapshot" and send you hunting for a + # snapshot that is sitting right there. Both abort -- but say which. + with pytest.raises(StagingError, match="has no snapshot"): + plan(probe=self._missing_pgdata) + with pytest.raises(StagingError, match="Permission denied"): + plan(probe=self._denied_pgdata) + + +class TestSnapshotTreeNames: + """zfs.snapshot.delete is non-recursive; we must sweep children ourselves.""" + + ALL = [ + "Tap@cloud_backup-5-20260712030000", + "Tap/apps@cloud_backup-5-20260712030000", + "Tap/apps/lidarr/config@cloud_backup-5-20260712030000", + "Tap@auto-2026-07-12_03-00", # unrelated periodic snapshot + "Tap/apps@cloud_backup-9-20260712030000", # another task + "Tank/backups@cloud_backup-5-20260712030000", # different pool + ] + + def test_returns_parent_and_all_children(self): + got = snapshot_tree_names("Tap@cloud_backup-5-20260712030000", self.ALL) + assert set(got) == { + "Tap@cloud_backup-5-20260712030000", + "Tap/apps@cloud_backup-5-20260712030000", + "Tap/apps/lidarr/config@cloud_backup-5-20260712030000", + } + + def test_never_touches_periodic_or_other_tasks_or_other_pools(self): + got = snapshot_tree_names("Tap@cloud_backup-5-20260712030000", self.ALL) + assert "Tap@auto-2026-07-12_03-00" not in got + assert "Tap/apps@cloud_backup-9-20260712030000" not in got + assert "Tank/backups@cloud_backup-5-20260712030000" not in got + + def test_malformed_snapshot_name_yields_nothing(self): + assert snapshot_tree_names("Tap", self.ALL) == [] + + +class FakeMiddleware: + def __init__(self, snapshots=None): + self.snapshots = list(snapshots or []) + self.calls = [] + self.logger = None + + async def call(self, method, *args): + self.calls.append((method, args)) + if method == "zfs.snapshot.query": + return [{"name": n} for n in self.snapshots] + if method == "zfs.snapshot.delete": + if args[0] not in self.snapshots: + raise RuntimeError("does not exist") + self.snapshots.remove(args[0]) + return True + raise AssertionError(f"unexpected call {method}") + + async def run_in_thread(self, fn, *args): + return fn(*args) + + +class TestDeleteSnapshotTree: + def test_deletes_parent_and_every_child(self): + mw = FakeMiddleware([ + "Tap@snap", "Tap/apps@snap", "Tap/apps/lidarr@snap", "Tap@keepme", + ]) + asyncio.run(delete_snapshot_tree(mw, "Tap@snap")) + assert mw.snapshots == ["Tap@keepme"] + + def test_is_idempotent_when_stock_already_removed_the_parent(self): + # Stock's finally can win the race once our mounts are released. + mw = FakeMiddleware(["Tap/apps@snap", "Tap/apps/lidarr@snap"]) + asyncio.run(delete_snapshot_tree(mw, "Tap@snap")) + assert mw.snapshots == [] + + def test_survives_query_failure_by_deleting_at_least_the_parent(self): + class Broken(FakeMiddleware): + async def call(self, method, *args): + if method == "zfs.snapshot.query": + raise RuntimeError("boom") + return await super().call(method, *args) + + mw = Broken(["Tap@snap"]) + asyncio.run(delete_snapshot_tree(mw, "Tap@snap")) + assert mw.snapshots == [] + + def test_attempts_no_delete_when_the_tree_is_already_gone(self): + # A successful query returning nothing means there is nothing to do. + # Falling back to the parent here would log a spurious "does not exist" + # warning on every clean run. + mw = FakeMiddleware(["Tap@unrelated"]) + asyncio.run(delete_snapshot_tree(mw, "Tap@snap")) + assert [m for m, _a in mw.calls if m == "zfs.snapshot.delete"] == [] + assert mw.snapshots == ["Tap@unrelated"] + + +class TestStageNestedOrdering: + def setup_method(self): + ACTIVE.clear() + + def test_sidecar_is_written_before_anything_is_mounted(self, tmp_path, monkeypatch): + # middlewared can die at any moment. If the snapshot were recorded only + # after apply_plan, a crash in that window would orphan a 160-snapshot + # tree -- the precise failure the sidecar exists to prevent. + import truecloud_nested as tn + + monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path)) + order = [] + + class Recorder(FakeMiddleware): + async def run_in_thread(self, fn, *args): + order.append(fn.__name__) + if fn.__name__ == "plan_staging": + return ([("/src", str(tmp_path / "cloud_backup-5"))], []) + if fn.__name__ in ("apply_plan", "verify_staged", "teardown"): + return [] if fn.__name__ == "teardown" else True + return fn(*args) + + asyncio.run(tn.stage_nested( + Recorder(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap", + "cloud_backup-5", DATASETS, + )) + + assert order.index("_write_sidecar") < order.index("apply_plan") + + def test_reclaims_the_snapshot_tree_left_by_a_crashed_run(self, tmp_path, + monkeypatch): + # teardown() reclaims the crashed run's MOUNTS, but nothing else would + # ever reclaim its SNAPSHOTS -- and we are about to overwrite the only + # record of them. One crash would orphan 160+ snapshots permanently. + import truecloud_nested as tn + + monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path)) + root = tn.staging_root_for("cloud_backup-5") + os.makedirs(os.path.dirname(root), exist_ok=True) + with open(sidecar_for(root), "w", encoding="utf-8") as fh: + fh.write("Tap@old-crashed-run") + + mw = FakeMiddleware(["Tap@old-crashed-run", "Tap/apps@old-crashed-run"]) + + class Stub(FakeMiddleware): + def __init__(self, inner): + super().__init__() + self.inner = inner + + async def call(self, method, *args): + return await self.inner.call(method, *args) + + async def run_in_thread(self, fn, *args): + if fn.__name__ == "plan_staging": + return ([("/src", root)], []) + if fn.__name__ == "teardown": + return [] + if fn.__name__ in ("apply_plan", "verify_staged"): + return True + return fn(*args) + + asyncio.run(tn.stage_nested( + Stub(mw), "/mnt/Tap", "Tap@new", "Tap", "/mnt/Tap", + "cloud_backup-5", DATASETS, + )) + + assert mw.snapshots == [], "the crashed run's snapshot tree must be reclaimed" + + def test_sidecar_is_removed_when_staging_fails(self, tmp_path, monkeypatch): + import truecloud_nested as tn + + monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path)) + root = tn.staging_root_for("cloud_backup-5") + + class Failing(FakeMiddleware): + async def run_in_thread(self, fn, *args): + if fn.__name__ == "plan_staging": + raise StagingError("boom") + return fn(*args) + + with pytest.raises(StagingError): + asyncio.run(tn.stage_nested( + Failing(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap", + "cloud_backup-5", DATASETS, + )) + + assert not os.path.exists(sidecar_for(root)) + + +class TestCleanupTask: + def setup_method(self): + ACTIVE.clear() + + def test_recovers_snapshot_from_sidecar_after_middlewared_restart(self, tmp_path, + monkeypatch): + # ACTIVE is in-process; a restart wipes it. The sidecar is the source of + # truth, otherwise the snapshot tree is orphaned forever. + import truecloud_nested as tn + + monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path)) + root = tn.staging_root_for("cloud_backup-5", base=str(tmp_path)) + os.makedirs(root, exist_ok=True) + with open(sidecar_for(root), "w", encoding="utf-8") as fh: + fh.write("Tap@snap") + + ACTIVE.clear() # simulate the restart + mw = FakeMiddleware(["Tap@snap", "Tap/apps@snap"]) + monkeypatch.setattr(tn, "teardown", lambda *_a, **_k: []) + + asyncio.run(cleanup_task(mw, "cloud_backup-5")) + + assert mw.snapshots == [] + assert not os.path.exists(sidecar_for(root)) + + def test_is_a_noop_when_never_staged(self, tmp_path, monkeypatch): + import truecloud_nested as tn + + monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path / "nope")) + mw = FakeMiddleware(["Tap@snap"]) + asyncio.run(cleanup_task(mw, "cloud_backup-5")) + assert mw.calls == [] + assert mw.snapshots == ["Tap@snap"] + + +class TestVerifyStaged: + """Anti-regression guard: proves the staged tree is real before we back it up.""" + + def test_passes_when_every_target_is_mounted_and_root_non_empty(self): + mounts = [("/src", ROOT), ("/src/a", f"{ROOT}/a")] + assert verify_staged(mounts, ismount=lambda p: True, listdir=lambda p: ["apps"]) + + def test_raises_when_a_target_is_not_actually_mounted(self): + # This is the case that would produce a silently-empty backup. + mounts = [("/src", ROOT), ("/src/a", f"{ROOT}/a")] + with pytest.raises(StagingError, match="not a mountpoint"): + verify_staged(mounts, ismount=lambda p: p == ROOT, listdir=lambda p: ["apps"]) + + def test_raises_when_staging_root_is_empty(self): + with pytest.raises(StagingError, match="empty"): + verify_staged([("/src", ROOT)], ismount=lambda p: True, listdir=lambda p: []) + + def test_raises_on_empty_plan(self): + with pytest.raises(StagingError): + verify_staged([]) + + +class FakeRunner: + def __init__(self, fail_on=None): + self.fail_on = fail_on + self.calls = [] + + def __call__(self, cmd): + self.calls.append(cmd) + + class R: + returncode = 0 + stderr = "" + + if self.fail_on and cmd[0] == "mount" and cmd[2] == self.fail_on: + R.returncode = 32 + R.stderr = "mount failed" + return R + + +class TestApplyPlanRollback: + def test_rolls_back_mounts_when_one_fails(self, tmp_path): + # A half-built tree must never reach the backup tool. + root = str(tmp_path / "root") + mounts = [("/src", root), ("/src/a", root + "/a"), ("/src/b", root + "/b")] + + runner = FakeRunner(fail_on="/src/b") + with pytest.raises(StagingError, match="bind-mount"): + apply_plan(mounts, runner=runner, isdir=lambda _p: True) + + umounts = [c[-1] for c in runner.calls if c[0] == "umount"] + assert umounts == [root + "/a", root] + + def test_raises_when_target_missing(self, tmp_path): + root = str(tmp_path / "root") + with pytest.raises(StagingError, match="does not exist"): + apply_plan( + [("/src", root), ("/src/a", root + "/a")], + runner=FakeRunner(), + isdir=lambda p: p == root, + ) + + +class TestTeardown: + def test_unmounts_deepest_first(self, tmp_path): + mounts_file = tmp_path / "mounts" + mounts_file.write_text( + f"tmpfs {ROOT} tmpfs rw 0 0\n" + f"tmpfs {ROOT}/apps tmpfs rw 0 0\n" + f"tmpfs {ROOT}/apps/lidarr/config tmpfs rw 0 0\n" + f"tmpfs {ROOT}/apps/lidarr tmpfs rw 0 0\n" + "tmpfs /somewhere/else tmpfs rw 0 0\n" + ) + runner = FakeRunner() + teardown(ROOT, runner=runner, mounts_file=str(mounts_file)) + + order = [c[-1] for c in runner.calls if c[0] == "umount"] + assert order == [ + f"{ROOT}/apps/lidarr/config", + f"{ROOT}/apps/lidarr", + f"{ROOT}/apps", + ROOT, + ] + assert "/somewhere/else" not in order + + def test_is_idempotent_when_nothing_mounted(self, tmp_path): + mounts_file = tmp_path / "mounts" + mounts_file.write_text("tmpfs /somewhere/else tmpfs rw 0 0\n") + runner = FakeRunner() + assert teardown(ROOT, runner=runner, mounts_file=str(mounts_file)) == [] + assert runner.calls == [] + + def test_falls_back_to_lazy_umount(self, tmp_path): + mounts_file = tmp_path / "mounts" + mounts_file.write_text(f"tmpfs {ROOT} tmpfs rw 0 0\n") + + class Busy(FakeRunner): + def __call__(self, cmd): + self.calls.append(cmd) + + class R: + returncode = 0 if "-l" in cmd else 32 + stderr = "target is busy" + + return R + + runner = Busy() + assert teardown(ROOT, runner=runner, mounts_file=str(mounts_file)) == [] + assert ["umount", "-l", ROOT] in runner.calls + + +class TestCurrentMountsUnder: + def test_matches_only_the_staging_subtree(self, tmp_path): + mounts_file = tmp_path / "mounts" + # "cloud_backup-50" must NOT match "cloud_backup-5". + mounts_file.write_text( + f"tmpfs {ROOT} tmpfs rw 0 0\n" + "tmpfs /run/truecloud-nested/cloud_backup-50 tmpfs rw 0 0\n" + ) + assert current_mounts_under(ROOT, mounts_file=str(mounts_file)) == [ROOT] + + +class TestStagingRootFor: + def test_stable_per_task(self): + assert staging_root_for("cloud_backup-5") == "/run/truecloud-nested/cloud_backup-5" + + def test_sanitises_path_separators(self): + assert "/" not in staging_root_for("evil/name").rsplit("/", 1)[-1] + + @pytest.mark.parametrize("name", ["..", ".", "...", "/", ""]) + def test_dot_components_cannot_escape_the_staging_base(self, name): + # os.path.join(BASE, "..") normalises to /run — teardown would rmdir it. + root = staging_root_for(name) + assert os.path.normpath(root).startswith("/run/truecloud-nested/") diff --git a/uninstall.sh b/uninstall.sh index 34a640e..9c1e866 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -3,7 +3,7 @@ set -euo pipefail -VERSION="0.0.4" +VERSION="0.3.0" PATCH_DIR="$(cd "$(dirname "$0")" && pwd)" _HOOK_COMMENT='TrueCloud provider patch (S3/B2)' @@ -91,6 +91,52 @@ if [ "$_ov_found" -eq 0 ]; then fi echo "" +# ── Unmount nested-snapshot staging trees ───────────────────────────────────── +# These bind mounts pin their ZFS snapshots, so they must go before anything +# tries to destroy those snapshots. Deepest first. + +echo "Unmounting nested-snapshot staging trees (if any) ..." +_stage_found=0 +_stage_failed=0 +# Deepest FIRST, by path depth (slash count) — not string length, which would +# let a long shallow path jump ahead of a short deep one and leave a child +# mounted (and its ZFS snapshot pinned). +while IFS= read -r _mp; do + [ -n "$_mp" ] || continue + if umount "$_mp" 2>/dev/null || umount -l "$_mp" 2>/dev/null; then + echo " Unmounted: $_mp" + else + echo " WARNING: Could not unmount $_mp" + _stage_failed=1 + fi + _stage_found=1 +done < <(awk '$2 == "/run/truecloud-nested" || index($2, "/run/truecloud-nested/") == 1 { + n = gsub(/\//, "/", $2); print n, $2 + }' /proc/self/mounts 2>/dev/null | sort -rn | cut -d' ' -f2-) + +if [ "$_stage_found" -eq 0 ]; then + echo " None active." +fi + +# NEVER `rm -rf` here: if an unmount failed, that would recurse *through* a live +# bind mount into the ZFS snapshot behind it. Remove empty directories only. +if [ "$_stage_failed" -eq 0 ]; then + find /run/truecloud-nested -depth -type d -exec rmdir {} + 2>/dev/null || true + rm -f /run/truecloud-nested/*.snapshot 2>/dev/null || true + rmdir /run/truecloud-nested 2>/dev/null || true +else + echo " WARNING: staging mounts remain; leaving /run/truecloud-nested in place." + echo " Unmount them manually, then remove the directory." +fi + +# The opt-in marker lives in the repo dir; remove it so a later re-install +# starts from the safe default (feature off). +if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]; then + rm -f "$PATCH_DIR/nested_snapshots_enabled" + echo " Removed nested-snapshot opt-in marker." +fi +echo "" + if [ "$_restore_failed" -eq 1 ]; then echo "" echo "ERROR: One or more UI bundle backups could not be restored." >&2