Compare commits
49
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6cf91e8c12 | ||
|
|
14ef25e3c6 | ||
|
|
02ba653127 | ||
|
|
5f8d42f2cf | ||
|
|
3915f92dec | ||
|
|
d1db3a3f60 | ||
|
|
0fc994f676 | ||
|
|
520b2d3735 | ||
|
|
4371797f94 | ||
|
|
45c89fd001 | ||
|
|
670dbd25f6 | ||
|
|
827c4358fd | ||
|
|
ea00bd0685 | ||
|
|
5a3d4288a2 | ||
|
|
b364a17735 | ||
|
|
3e1de8ffd1 | ||
|
|
a7cbb3a994 | ||
|
|
d1216eeb0f | ||
|
|
6b7034a8cd | ||
|
|
6ce1206f01 | ||
|
|
e426045255 | ||
|
|
89eb3a16be | ||
|
|
862bdd3399 | ||
|
|
753c3f8cad | ||
|
|
52b11eada2 | ||
|
|
ca906f5ee4 | ||
|
|
30b9f18166 | ||
|
|
908b6e9f22 | ||
|
|
df412eeff7 | ||
|
|
086b20ed23 | ||
|
|
677c90481c | ||
|
|
8a41d7d7ef | ||
|
|
0fea5c40bd | ||
|
|
ce6998a935 | ||
|
|
928d0d1973 | ||
|
|
413cd60ed4 | ||
|
|
605231b39f | ||
|
|
7cc0826c2c | ||
|
|
82084b6806 | ||
|
|
c6b252ac6b | ||
|
|
841e0364fd | ||
|
|
0d04c2cd1c | ||
|
|
2b8ef107f7 | ||
|
|
b50567e9a7 | ||
|
|
1b2407f6e2 | ||
|
|
ab0b66c47d | ||
|
|
469a2e4651 | ||
|
|
518a22d87e | ||
|
|
8c1b4c45f5 |
@@ -0,0 +1,2 @@
|
||||
github: sudolulo
|
||||
ko_fi: sudolulo
|
||||
+70
-23
@@ -9,9 +9,31 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# ONE job, deliberately. This was four (shell + a 3-way python matrix) and they
|
||||
# started within the same second on the self-hosted Gitea runner, which is what
|
||||
# made CI unreliable in two separate ways:
|
||||
#
|
||||
# 1. The act action-cache race. `act` caches each ACTION as a single shared git
|
||||
# clone under /root/.cache/act/<hash> and re-pulls it per job, so concurrent
|
||||
# jobs using the same action fight over that directory and the loser dies
|
||||
# with `lstat /root/.cache/act/<hash>/<file>: no such file or directory` --
|
||||
# a red `main` with zero suite output, and a different victim each push
|
||||
# (3.12 on one, 3.11 on the next). Dropping one action only shrank the
|
||||
# surface: every job still used actions/checkout. Concurrency is the actual
|
||||
# ingredient, so removing it removes the whole class -- a single job cannot
|
||||
# race itself, no matter which actions it uses.
|
||||
#
|
||||
# 2. Docker Hub 429s. The runner force-pulls its base image per job, so four
|
||||
# jobs meant four anonymous pulls per push. A few pushes and re-runs in an
|
||||
# afternoon exhausted the anonymous limit and every job failed before it
|
||||
# started -- including the shell job, which nothing had touched. One job is
|
||||
# one pull.
|
||||
#
|
||||
# The cost is wall-clock parallelism, and this repo does not need it: the suite
|
||||
# is ~1.5s, so container start and interpreter downloads dominate either way.
|
||||
jobs:
|
||||
shell:
|
||||
name: shell (shellcheck + syntax)
|
||||
ci:
|
||||
name: ci (shell + python 3.11-3.13)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -31,32 +53,57 @@ jobs:
|
||||
env:
|
||||
SHELLCHECK_OPTS: -S warning -e SC1091
|
||||
|
||||
python:
|
||||
name: python ${{ matrix.python }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# TrueNAS SCALE middleware runs 3.11+; keep the patch importable across
|
||||
# the versions it may be injected into.
|
||||
python: ["3.11", "3.12", "3.13"]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
|
||||
- name: install dev deps
|
||||
run: python -m pip install --upgrade pip pytest ruff
|
||||
# uv-managed interpreters instead of actions/setup-python: the prebuilt-CPython
|
||||
# download path setup-python relies on does not work on the self-hosted Gitea
|
||||
# runner (all three matrix jobs failed at setup there while passing on GitHub);
|
||||
# uv works identically on both.
|
||||
#
|
||||
# Installed by a plain `run:` step rather than astral-sh/setup-uv: one fewer
|
||||
# action is one fewer thing to go wrong, and the action was only ever
|
||||
# fetching a binary -- the interpreter is chosen per command by `uvx
|
||||
# --python`, never by the action.
|
||||
#
|
||||
# Pinned for the same reason ruff is pinned below: an unpinned uv means any
|
||||
# upstream release can turn main red with no code change here.
|
||||
- name: install uv
|
||||
env:
|
||||
UV_VERSION: "0.11.21"
|
||||
run: |
|
||||
curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | sh
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: ruff
|
||||
run: ruff check patch tests tools
|
||||
# Pinned: an unpinned ruff means any upstream release can turn main red
|
||||
# with no code change.
|
||||
run: uvx ruff@0.16.1 check patch tests tools
|
||||
|
||||
# TrueNAS SCALE middleware runs 3.11+; keep the patch importable across the
|
||||
# versions it may be injected into. Every version runs even after one
|
||||
# fails -- that is what `fail-fast: false` bought when this was a matrix,
|
||||
# and losing it would mean a 3.11 break hides whether 3.12 and 3.13 are
|
||||
# fine, which is exactly the information you want at that moment.
|
||||
- name: pytest
|
||||
run: pytest tests -v
|
||||
env:
|
||||
PYTHONS: "3.11 3.12 3.13"
|
||||
run: |
|
||||
fail=0
|
||||
for v in $PYTHONS; do
|
||||
echo "::group::pytest on python $v"
|
||||
uvx --python "$v" pytest tests -v \
|
||||
|| { echo "::error::suite failed on python $v"; fail=1; }
|
||||
echo "::endgroup::"
|
||||
done
|
||||
exit $fail
|
||||
|
||||
- name: verify injected middleware blocks compile
|
||||
# Belt-and-braces: the *_BLOCK strings are appended into live middlewared
|
||||
# modules. A syntax error there would break the box at boot.
|
||||
run: pytest tests/test_apply_blocks.py -v
|
||||
env:
|
||||
PYTHONS: "3.11 3.12 3.13"
|
||||
run: |
|
||||
fail=0
|
||||
for v in $PYTHONS; do
|
||||
uvx --python "$v" pytest tests/test_apply_blocks.py -v \
|
||||
|| { echo "::error::injected blocks failed to compile on python $v"; fail=1; }
|
||||
done
|
||||
exit $fail
|
||||
|
||||
@@ -24,10 +24,18 @@ on:
|
||||
paths:
|
||||
# The manifest itself changed -- re-check immediately rather than waiting a day.
|
||||
- "tools/compat.py"
|
||||
# ...and so did the thing that PUBLISHES the finding. This was missing, and it
|
||||
# showed: the commit teaching the bot to refresh a stale report body touched only
|
||||
# compat_publish.py, so no run fired, and the report stayed stale until the next
|
||||
# scheduled one. A fix nobody runs is a fix nobody has.
|
||||
- "tools/compat_publish.py"
|
||||
- ".github/workflows/compat.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# write, because the matrix refresh pushes a branch and opens a PR. It does NOT get
|
||||
# to move `main` -- that is the whole reason it is a PR. See the refresh step below.
|
||||
contents: write
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
@@ -130,17 +138,27 @@ jobs:
|
||||
- name: matrix
|
||||
run: cat /tmp/matrix.md
|
||||
|
||||
# Keep the README's table true. A support matrix that quietly goes stale is not
|
||||
# a stale doc -- it is a false promise to somebody deciding whether to trust
|
||||
# this with their backups.
|
||||
# Keep the README's table true — as a PULL REQUEST, on the CANONICAL forge.
|
||||
#
|
||||
# Only ever touches the block between the COMPAT MATRIX markers, and only on
|
||||
# the canonical host (Gitea) so the two forges cannot race each other. The
|
||||
# `paths:` trigger above does not include README.md, so this cannot re-trigger
|
||||
# itself; and a README change is documentation-only, which by design raises no
|
||||
# update alert on anyone's box.
|
||||
- name: refresh the README matrix
|
||||
# Two things this gets right that the obvious version gets wrong:
|
||||
#
|
||||
# 1. It is a PR, not a push to main. This used to `git push origin HEAD:main`
|
||||
# from CI. An unattended write to main is exactly what the release barrier
|
||||
# exists to prevent — a bot that can move main can move it somewhere nobody
|
||||
# looked. Nothing lands by itself.
|
||||
#
|
||||
# 2. It runs on GITEA, not GitHub. GitHub is a one-way MIRROR: a PR merged there
|
||||
# would be silently clobbered by the next `fleet-repos mirror` push from Gitea.
|
||||
# A bot opening PRs against a mirror is a bot doing nothing, slowly.
|
||||
#
|
||||
# A stale support matrix is not a stale doc — it is a false promise to somebody
|
||||
# deciding whether to trust this with their backups. So it is refreshed daily; it
|
||||
# just asks first.
|
||||
- name: refresh the README matrix (PR on the canonical forge)
|
||||
if: ${{ github.event_name == 'schedule' && !contains(github.server_url, 'github.com') }}
|
||||
env:
|
||||
TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
|
||||
API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import json, sys
|
||||
@@ -151,66 +169,73 @@ jobs:
|
||||
print("changed" if compat.update_readme(rows) else "unchanged")
|
||||
PY
|
||||
|
||||
if ! git diff --quiet -- README.md; then
|
||||
git config user.name "truecloud-patch bot"
|
||||
git config user.email "bot@onetick.ninja"
|
||||
git add README.md
|
||||
git commit -m "docs: refresh the TrueNAS compatibility matrix"
|
||||
git push origin HEAD:main
|
||||
fi
|
||||
git diff --quiet -- README.md && { echo "matrix unchanged — nothing to propose"; exit 0; }
|
||||
|
||||
# A broken SHIPPED release is an outage: users are on it right now.
|
||||
git config user.name "truecloud-patch bot"
|
||||
git config user.email "bot@onetick.ninja"
|
||||
|
||||
BRANCH=bot/compat-matrix
|
||||
git checkout -B "$BRANCH"
|
||||
git add README.md
|
||||
git commit -m "docs: refresh the TrueNAS compatibility matrix"
|
||||
git push -f origin "$BRANCH"
|
||||
|
||||
# ONE long-lived PR, force-pushed in place — not a new one every morning.
|
||||
# (A daily PR is the same mistake as a daily comment, wearing a hat.)
|
||||
BRANCH="$BRANCH" python3 - <<'PY'
|
||||
import json, os, urllib.error, urllib.request
|
||||
|
||||
api, token, branch = os.environ["API"], os.environ["TOKEN"], os.environ["BRANCH"]
|
||||
h = {"Authorization": f"token {token}", "Content-Type": "application/json"}
|
||||
|
||||
def call(url, method="GET", data=None):
|
||||
r = urllib.request.Request(
|
||||
url, method=method, headers=h,
|
||||
data=json.dumps(data).encode() if data else None)
|
||||
with urllib.request.urlopen(r) as resp: # noqa: S310
|
||||
return json.load(resp) if resp.length != 0 else {}
|
||||
|
||||
existing = [
|
||||
p for p in call(f"{api}/pulls?state=open")
|
||||
if p["head"]["ref"] == branch
|
||||
]
|
||||
if existing:
|
||||
print(f"PR #{existing[0]['number']} already open; the force-push updated it")
|
||||
else:
|
||||
pr = call(f"{api}/pulls", "POST", {
|
||||
"head": branch, "base": "main",
|
||||
"title": "docs: refresh the TrueNAS compatibility matrix",
|
||||
"body": (
|
||||
"The daily compatibility check found that the support matrix in "
|
||||
"the README no longer matches iXsystems' actual middleware.\n\n"
|
||||
"This only touches the block between the `COMPAT MATRIX` markers. "
|
||||
"It is regenerated by `tools/compat.py --matrix --update-readme` "
|
||||
"and force-pushed, so it always reflects the latest run."
|
||||
),
|
||||
})
|
||||
print(f"opened PR #{pr['number']}")
|
||||
PY
|
||||
|
||||
# ONE bug report, kept in sync. It is edited in place when the findings change and
|
||||
# says NOTHING when they do not.
|
||||
#
|
||||
# The first version commented on every run and left 11 identical 3,000-character
|
||||
# comments on one issue in a single day. A bot that repeats itself daily gets
|
||||
# muted, and then the next real finding is scrolled past — which defeats the whole
|
||||
# reason for building it.
|
||||
#
|
||||
# Runs on whichever forge it lands on; compat_publish.py handles both, so the two
|
||||
# cannot drift.
|
||||
- name: file / update / close the bug report
|
||||
env:
|
||||
TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
|
||||
API: ${{ contains(github.server_url, 'github.com') && 'https://api.github.com' || format('{0}/api/v1', github.server_url) }}/repos/${{ github.repository }}
|
||||
run: python3 tools/compat_publish.py --api "$API" --token "$TOKEN" --matrix /tmp/matrix.json
|
||||
|
||||
# A broken SHIPPED release is an outage: users are on it right now. Fails LAST, so
|
||||
# the report is filed before the job goes red.
|
||||
- name: fail if a shipped release is broken
|
||||
if: ${{ steps.check.outputs.shipped_broken != '0' }}
|
||||
run: |
|
||||
echo "::error::The patch is broken on a SHIPPED TrueNAS release."
|
||||
exit 1
|
||||
|
||||
- name: file a bug report (GitHub)
|
||||
if: ${{ steps.report.outputs.broken == '1' && contains(github.server_url, 'github.com') }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TITLE: "Incompatible with upcoming TrueNAS: ${{ steps.report.outputs.refs }}"
|
||||
run: |
|
||||
# One issue per set of broken refs, reopened/updated rather than duplicated
|
||||
# daily -- a bot that files the same issue every morning gets muted, and
|
||||
# then it is not a warning system any more.
|
||||
existing="$(gh issue list --state all --search "$TITLE" \
|
||||
--json number,title \
|
||||
--jq '.[] | select(.title == env.TITLE) | .number' | head -1)"
|
||||
|
||||
if [ -n "$existing" ]; then
|
||||
gh issue comment "$existing" --body-file /tmp/issue.md
|
||||
gh issue reopen "$existing" 2>/dev/null || true
|
||||
else
|
||||
gh issue create --title "$TITLE" --body-file /tmp/issue.md
|
||||
fi
|
||||
|
||||
- name: file a bug report (Gitea)
|
||||
if: ${{ steps.report.outputs.broken == '1' && !contains(github.server_url, 'github.com') }}
|
||||
env:
|
||||
TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
|
||||
API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
|
||||
TITLE: "Incompatible with upcoming TrueNAS: ${{ steps.report.outputs.refs }}"
|
||||
run: |
|
||||
body="$(jq -Rs . < /tmp/issue.md)"
|
||||
title="$(printf '%s' "$TITLE" | jq -Rs .)"
|
||||
|
||||
# Same title => same issue. Comment on it instead of filing a new one.
|
||||
number="$(curl -sf -H "Authorization: token $TOKEN" \
|
||||
"$API/issues?state=all&type=issues" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title == $t) | .number' | head -1)"
|
||||
|
||||
if [ -n "$number" ]; then
|
||||
curl -sS -X POST "$API/issues/$number/comments" \
|
||||
-H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||
-d "$(printf '{"body":%s}' "$body")" -o /dev/null -w 'comment -> %{http_code}\n'
|
||||
curl -sS -X PATCH "$API/issues/$number" \
|
||||
-H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||
-d '{"state":"open"}' -o /dev/null -w 'reopen -> %{http_code}\n'
|
||||
else
|
||||
curl -sS -X POST "$API/issues" \
|
||||
-H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||
-d "$(printf '{"title":%s,"body":%s}' "$title" "$body")" \
|
||||
-o /dev/null -w 'create -> %{http_code}\n'
|
||||
fi
|
||||
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
echo "--- release body ---"
|
||||
cat /tmp/notes.md
|
||||
|
||||
# This repo is canonically hosted on Gitea (git.onetick.ninja) and mirrored to
|
||||
# This repo is canonically hosted on Gitea (git.arch.fyi) and mirrored to
|
||||
# GitHub, and BOTH run this workflow -- Gitea reads .github/workflows too. So
|
||||
# the publish step has to work on whichever forge it lands on. Everything
|
||||
# above is forge-agnostic; only the "create a release" API differs.
|
||||
@@ -164,27 +164,45 @@ jobs:
|
||||
*-rc*|*-beta*|*-alpha*) prerelease=true ;;
|
||||
esac
|
||||
|
||||
# jq -Rs so the notes are JSON-encoded properly: the changelog is full of
|
||||
# quotes, backticks and newlines, and hand-built JSON would mangle them.
|
||||
body="$(jq -Rs . < /tmp/notes.md)"
|
||||
payload="$(printf '{"tag_name":%s,"name":%s,"body":%s,"prerelease":%s}' \
|
||||
"$(printf '%s' "$TAG" | jq -Rs .)" \
|
||||
"$(printf '%s' "$TAG" | jq -Rs .)" \
|
||||
"$body" "$prerelease")"
|
||||
# python3, not jq. The changelog is full of quotes, backticks and newlines,
|
||||
# so the body must be properly JSON-encoded -- but `jq` is not guaranteed on
|
||||
# a self-hosted Gitea runner, and a publish step that dies on a missing tool
|
||||
# leaves a tag with no release behind it. python3 is guaranteed: setup-python
|
||||
# ran above.
|
||||
python3 - "$TAG" "$API" "$TOKEN" "$prerelease" <<'PY'
|
||||
import json, sys, urllib.error, urllib.request
|
||||
|
||||
existing="$(curl -sf -H "Authorization: token $TOKEN" \
|
||||
"$API/releases/tags/$TAG" 2>/dev/null || true)"
|
||||
tag, api, token, prerelease = sys.argv[1:5]
|
||||
with open("/tmp/notes.md", encoding="utf-8") as fh:
|
||||
body = fh.read()
|
||||
|
||||
if [ -n "$existing" ]; then
|
||||
id="$(printf '%s' "$existing" | jq -r .id)"
|
||||
echo "Release $TAG exists (id=$id) — updating notes."
|
||||
curl -sS -X PATCH "$API/releases/$id" \
|
||||
-H "Authorization: token $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" -o /dev/null -w 'PATCH -> %{http_code}\n'
|
||||
else
|
||||
curl -sS -X POST "$API/releases" \
|
||||
-H "Authorization: token $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" -o /dev/null -w 'POST -> %{http_code}\n'
|
||||
fi
|
||||
payload = {
|
||||
"tag_name": tag, "name": tag, "body": body,
|
||||
"prerelease": prerelease == "true",
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def call(url, method, data=None):
|
||||
req = urllib.request.Request(
|
||||
url, method=method, headers=headers,
|
||||
data=json.dumps(data).encode() if data else None)
|
||||
with urllib.request.urlopen(req) as r: # noqa: S310
|
||||
return r.status, json.load(r) if r.length != 0 else {}
|
||||
|
||||
try:
|
||||
_, existing = call(f"{api}/releases/tags/{tag}", "GET")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
raise
|
||||
existing = None
|
||||
|
||||
if existing:
|
||||
status, _ = call(f"{api}/releases/{existing['id']}", "PATCH", payload)
|
||||
print(f"updated release {tag} -> {status}")
|
||||
else:
|
||||
status, _ = call(f"{api}/releases", "POST", payload)
|
||||
print(f"created release {tag} (prerelease={payload['prerelease']}) -> {status}")
|
||||
PY
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
/apply.log.1
|
||||
/apply.log.2
|
||||
/hook_status.json
|
||||
# The resolved middlewared directory, recorded by apply.sh so wait_restart.sh can
|
||||
# check whether the patched modules are still on the live path without
|
||||
# re-deriving site-packages.
|
||||
/.mw_dir
|
||||
/disabled
|
||||
/nested_snapshots_enabled
|
||||
# Written by apply.sh when a module's assumptions no longer fit the installed
|
||||
|
||||
+387
@@ -6,6 +6,349 @@ is deliberate: see [Releasing](docs/releasing.md). Twelve releases were cut on
|
||||
live, every one of those interrupts every user. An alert people learn to ignore is
|
||||
worse than no alert, because one day it carries a security fix.
|
||||
|
||||
## v0.8.0 — 2026-08-26
|
||||
### Changed
|
||||
|
||||
- **CI's python matrix is green on the self-hosted Gitea runner again.** The real
|
||||
failure was that the Gitea runner image executes jobs as root, and the two
|
||||
unreadable-sidecar tests build their scenario with `chmod(0)` — which cannot make
|
||||
a file unreadable for root (`CAP_DAC_OVERRIDE`). Those two tests now skip as root
|
||||
with that reason; GitHub's non-root runner still exercises them. The matrix also
|
||||
moved to uv-managed interpreters (one toolchain across both runners) and ruff is
|
||||
pinned to 0.16.1 so an upstream ruff release can't turn `main` red without a code
|
||||
change.
|
||||
- **README badges point at the public GitHub mirror** (workflow status and
|
||||
releases) instead of the private forge. The release badge had also been reading
|
||||
the stale Gitea v0.6.1 release instead of the current v0.7.0 on GitHub.
|
||||
|
||||
- **`master` is now labelled `27-dev`, because it is not the next release.** iX
|
||||
branches each major onto its own `release/` line and master rolls straight on to the
|
||||
one after — on 2026-07-14 every recent commit on master targeted `27.0.0-BETA.1`
|
||||
while 26 was still in beta. So a **BROKEN** master row, rendered as
|
||||
"master _(unreleased)_", read as *"the version you are about to install is broken"*
|
||||
when the breakage was a major release away on a line nobody can download. In a table
|
||||
whose entire job is helping somebody decide whether to trust this with their backups,
|
||||
that is a false alarm in the worst possible place. The label is derived from the
|
||||
newest major in the matrix plus one, so it rolls over to `28-dev` by itself once 27
|
||||
branches.
|
||||
|
||||
For the record, the breakage is `NAS-141498` (2026-06-24), "Convert cloud_backup
|
||||
plugin to the typesafe pattern": it re-signatures `restic_backup` and
|
||||
`get_restic_config`, splitting `entry`/`credentials` out of the `cloud_backup` dict.
|
||||
It is deliberately not being chased while the 27 line is still churning.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **CI turned `main` red on two of three pushes without running a single test,
|
||||
then stopped running at all.** Two failures, one cause: four concurrent jobs on
|
||||
one self-hosted runner.
|
||||
|
||||
`act`, the engine behind the Gitea runner, caches each *action* as one shared
|
||||
git clone under `/root/.cache/act/<hash>` and re-pulls it per job. Jobs
|
||||
starting in the same second fight over that directory, and the loser dies with
|
||||
`lstat /root/.cache/act/<hash>/.npmrc: no such file or directory` — before any
|
||||
suite output exists, with a different victim each push (3.12 on one, 3.11 on
|
||||
the next). Separately, the runner force-pulls its base image per job, so four
|
||||
jobs meant four anonymous Docker Hub pulls per push; a few pushes and re-runs
|
||||
in one afternoon hit `429 Too Many Requests` and *every* job began failing
|
||||
before it started — including the shell job, which nothing had touched.
|
||||
|
||||
CI is now a single job. Dropping one action (uv is installed by a `run:` step
|
||||
rather than `astral-sh/setup-uv`, which was only ever fetching a binary) only
|
||||
shrank the surface, because every job still used `actions/checkout`.
|
||||
Concurrency is the actual ingredient, so removing it removes the whole class:
|
||||
one job cannot race itself whatever actions it uses, and one job is one pull.
|
||||
The Python sweep moved inside that job and still runs every version after one
|
||||
fails — that is what `fail-fast: false` bought, and losing it would mean a 3.11
|
||||
break hides whether 3.12 and 3.13 are fine. The cost is wall-clock
|
||||
parallelism, which this repo does not need: the suite is ~1.5s, so container
|
||||
start and interpreter downloads dominate either way.
|
||||
|
||||
A red gate that is usually noise is worse than no gate, because the one time
|
||||
it means something, nobody looks.
|
||||
|
||||
- **The patch survived being applied and then silently stopped existing, because
|
||||
something else remounted `/usr` four seconds later.** On a box running
|
||||
TrueNAS 25.10.6 the boot of 2026-08-19 went: 16:41:56 `apply.sh` mounts its
|
||||
overlay on `/usr/lib/python3/dist-packages`, patches `b2.py`/`restic.py`, logs
|
||||
every step `OK`; **16:42:00** a second PREINIT hook runs `systemd-sysext
|
||||
refresh` over `/usr` — `Unmerged '/usr'.` / `Merged extensions into '/usr'.` —
|
||||
and our overlay, which lives *inside* that hierarchy, is torn off with it;
|
||||
16:47:24 our own deferred restart fires exactly as designed and middlewared
|
||||
imports the **stock** modules. Every B2 TrueCloud Backup task then failed with
|
||||
`NotImplementedError` from stock `rclone/base.py` for nineteen hours, across
|
||||
four scheduled runs, while `apply.log` and `hook_status.json` both said the
|
||||
patch was active.
|
||||
|
||||
Nothing in the patch was wrong, which is the point: applying at PREINIT and
|
||||
restarting later is only sound if the patched files are still on the live path
|
||||
when middlewared re-imports them, and **that is not something PREINIT can
|
||||
guarantee**. Init scripts run sequentially in id order, so any hook registered
|
||||
after ours always wins. Worse, hook ordering cannot fix it either —
|
||||
middlewared's own `docker.configure_nvidia` merges a sysext over `/usr` at
|
||||
*runtime*, long after every PREINIT hook is finished.
|
||||
|
||||
So the deferred restart no longer trusts the PREINIT pass. `wait_restart.sh`
|
||||
now re-applies immediately before it restarts middlewared — after boot has
|
||||
settled, which is also after every sysext merge and docker nvidia
|
||||
configuration — and verifies the marker is genuinely on the live path before
|
||||
restarting. It is no longer `exec systemctl try-restart middlewared`, because
|
||||
something has to run afterwards.
|
||||
|
||||
What runs afterwards deliberately does **not** restart again. `try-restart`
|
||||
returns as soon as middlewared is READY, and middlewared then brings docker up
|
||||
— `docker.configure_nvidia` merges the stock nvidia sysext over `/usr` at that
|
||||
point, detaching the overlay *after* the patched modules have already been
|
||||
imported. A disk check there reports "missing" on a perfectly healthy system,
|
||||
and restarting on that signal would restart a correctly-patched middlewared
|
||||
straight back into the same race. So the overlay is re-mounted for the benefit
|
||||
of the next restart, and the question of whether *this* middlewared actually
|
||||
holds the patch is left to the one thing that can answer it exactly — the
|
||||
in-process alert below. That re-mount preserves `hook_status.json`'s
|
||||
`patched_at`: `create_task.py verify` decides "loaded" by comparing
|
||||
middlewared's start time against that stamp, so a re-apply running *after* the
|
||||
restart would have made the stamp newer than the process which correctly
|
||||
imported the patch, and `verify` would have reported FAIL forever on every
|
||||
boot where the sysext merge detaches the overlay. Caught on hardware while
|
||||
validating the candidate — a new lying status introduced by the fix for a
|
||||
lying status.
|
||||
|
||||
Two supporting fixes fell out of the same failure. `_ensure_writable` treated
|
||||
"one of our overlays is listed on this directory" as "already done" — but it
|
||||
only ever reaches that check when the directory is **not** writable, and a live
|
||||
overlay of ours always is. A shadowed overlay was therefore indistinguishable
|
||||
from a healthy one; it is now detached and re-mounted, reusing the same
|
||||
upperdir so everything patched earlier in the boot reappears intact, with a
|
||||
fresh workdir because overlayfs refuses one left behind by a detached mount.
|
||||
|
||||
- **middlewared now says so when it is running stock.** The gap that let this
|
||||
cost nineteen hours was not the remount, it was that nothing could tell the
|
||||
difference between "patched on disk" and "patched in the running process".
|
||||
`apply.log` can only ever report the first. A new CRITICAL alert asks the
|
||||
second question from inside middlewared, hourly, where it is exact: the patch
|
||||
stamps the objects it replaces, so a missing stamp means this interpreter
|
||||
imported stock code. It checks both halves — `restic.py`'s `_truecloud_patched`
|
||||
marker and whether `B2RcloneRemote.get_restic_config` is still the base class's
|
||||
— since either can go missing alone. It stays quiet when the kill switch is
|
||||
set or the providers module has been retired as native, and it is deliberately
|
||||
**not** silenced by `update_alerts_disabled`: that mutes release notifications,
|
||||
not a broken backup path.
|
||||
|
||||
Boot-time diagnosis also no longer depends on the journal. `wait_restart.sh`
|
||||
logged only to the journal, and journald retention on a busy box is easily
|
||||
shorter than the interval between reboots — the 2026-08-19 boot had already
|
||||
rotated away by the time it was investigated. It now writes to `apply.log`
|
||||
alongside everything else.
|
||||
|
||||
- **The next maintenance release was never checked, and it is the one that reaches
|
||||
users.** Shipped versions were discovered from `TS-*` tags and unreleased ones from
|
||||
`release/*` branches carrying `-BETA`/`-RC`. A branched-but-untagged *maintenance*
|
||||
release is neither: `release/25.10.5` has no tag, and its line has already shipped,
|
||||
so the "a prerelease of a shipped line is history" filter discarded it. It was
|
||||
invisible — and it is precisely what a 25.10.4 box gets on its next update. A break
|
||||
there would have reached real users before the daily check ever looked at it, on the
|
||||
only line anybody is actually running.
|
||||
|
||||
A plain `release/X.Y.Z` branch is now checked when its line **has** shipped and it
|
||||
sorts **newer** than that line's newest tag. Both things that must stay out fall out
|
||||
of the same rule: `release/24.10-RC.2` sorts older than `TS-24.10.2.4` (history, not
|
||||
a warning), and iX's typo branch `release/25.20.2.2` is on a line that has no tag at
|
||||
all, so it is not a release line. This immediately surfaced two refs that had never
|
||||
been checked — `release/25.10.5` and `release/24.10.2.5` — both of which pass.
|
||||
|
||||
`is_unreleased()` now keys off where a ref came from (branch = not yet shipped)
|
||||
rather than looking for `-BETA`/`-RC` in its name. Otherwise `release/25.10.5` would
|
||||
count as shipped and a break in it would fail the build as a live outage — on a
|
||||
version nobody is running yet.
|
||||
|
||||
- **An unchanged fingerprint froze the bug report's body, not just its comments.** Two
|
||||
questions were sharing one answer. *Have the findings changed?* gates **comments** —
|
||||
they notify, and a daily "still broken, same as yesterday" is what teaches everyone
|
||||
to ignore the one that finally matters. *Is the body still true?* gates the **body** —
|
||||
and editing an issue body notifies nobody on either forge, so keeping it honest is
|
||||
free. Conflated, the report could never be corrected while the findings held steady,
|
||||
and the fingerprint deliberately ignores everything that moves on its own — healthy
|
||||
rows, the hardware-verified column, point releases, and how a row is labelled. The
|
||||
`master` → `27-dev` relabel above would have reached the README and never the issue
|
||||
anybody actually opens. The body is now rewritten whenever it is out of date (after
|
||||
normalising line endings, so a forge round-tripping `\r\n` does not cause a rewrite
|
||||
every run) and comments remain strictly a changelog of real changes.
|
||||
|
||||
- **A change to the publisher did not re-run the check.** `compat.yml`'s `push:` paths
|
||||
listed `tools/compat.py` but not `tools/compat_publish.py` — so the very commit that
|
||||
taught the bot to refresh a stale report body triggered no run, and the report stayed
|
||||
stale until the next scheduled one. A fix nobody runs is a fix nobody has.
|
||||
|
||||
- **The compatibility bot filed a new duplicate bug report on every Gitea run.**
|
||||
`find_issue()` skipped pull requests by testing for the *presence* of the
|
||||
`pull_request` key. GitHub omits that key on a plain issue; Gitea sends it as
|
||||
`null`. So on Gitea every issue was discarded as a PR, the lookup always came back
|
||||
empty, and the bot took the "nothing filed yet" branch and opened a fresh report
|
||||
each run — **nine copies on the canonical forge**, four of them filed *after* the
|
||||
commit that was meant to stop precisely this. The mirror was fine, which is why it
|
||||
went unnoticed: GitHub's payload shape is the one the filter was written against.
|
||||
|
||||
It is the same failure the anti-spam fix was written to prevent, moved from
|
||||
comments to issues, and it survived because `find_issue` was the only function in
|
||||
`compat_publish.py` with no test. It now has one, per forge, and the daily cron —
|
||||
which had not yet run once — no longer accumulates a report a day.
|
||||
|
||||
The issue list is also requested with **both** paging parameters (`per_page` for
|
||||
GitHub, `limit` for Gitea). Each forge ignores the other's, and Gitea's default page
|
||||
is 30, so the lookup would have started missing the report again once the pile it
|
||||
was creating grew past one page.
|
||||
|
||||
## v0.7.0 — 2026-07-14
|
||||
### Added
|
||||
|
||||
- **TrueNAS 26 support, verified on a real TrueNAS 26 install.** 26 deletes
|
||||
`plugins/zfs_/` outright, taking the private `zfs.dataset.query`,
|
||||
`zfs.snapshot.query` and `zfs.snapshot.delete` with it. Every one of those was on
|
||||
the nested module's critical path, so nested snapshots were **BROKEN** on 26 and
|
||||
`apply.sh` correctly refused to apply the module there.
|
||||
|
||||
Snapshot **deletion** now resolves its namespace at runtime — `pool.snapshot` on
|
||||
25.10 and 26, `zfs.snapshot` on 24.10 and 25.04, because no single namespace spans
|
||||
every supported release. `tools/compat.py` checks the same list the runtime uses,
|
||||
so what CI verifies and what runs cannot drift apart.
|
||||
|
||||
Hardware-verified on TrueNAS 26.0.0-BETA.1: a 274-snapshot recursive backup of a
|
||||
292-dataset pool, then a **byte-identical restore of a four-level-deep child
|
||||
dataset**.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Enumeration no longer trusts middleware's dataset and snapshot queries — they
|
||||
are filtered.** This is the important one, and it is the bug that a test VM caught
|
||||
and no amount of source analysis ever could have.
|
||||
|
||||
The obvious port of the deleted private `zfs.dataset.query` was the public
|
||||
`pool.dataset.query`. It exists, it is documented, it is covered by iX's
|
||||
deprecation policy — and it is **not a like-for-like replacement**. It applies a
|
||||
*visibility policy*: it hides the datasets TrueNAS considers its own — `ix-apps/*`,
|
||||
`.system/*`, `.ix-virt/*`. On a real pool that is **84 of 270 datasets**, and
|
||||
`ix-apps` holds **live application data**.
|
||||
|
||||
Staging from that view would have silently omitted every one of them. Worse,
|
||||
`plan_staging()` would never have seen them, so they would not have appeared in its
|
||||
`skipped` list either — no warning, no failure, just a green backup quietly missing
|
||||
data. That is precisely the failure this module exists to prevent. The snapshot
|
||||
query lies the same way (205 of 274), so the sweep would have orphaned one snapshot
|
||||
per hidden dataset, on every run, forever.
|
||||
|
||||
The module now **reads the truth from ZFS and makes changes through middleware**:
|
||||
enumeration is `zfs list`, which no policy can filter and which behaves identically
|
||||
on every release; mutation stays a middleware call, so TrueNAS's own bookkeeping
|
||||
stays consistent. A failing `zfs list` raises rather than returning an empty list —
|
||||
"no datasets" and "the command broke" must never look the same.
|
||||
|
||||
**No shipped release is affected.** v0.6.1 and earlier call the *private*
|
||||
`zfs.dataset.query`, which returns all 270 datasets. The bug existed only in the
|
||||
unreleased TrueNAS 26 port.
|
||||
|
||||
- **The patch now owns the snapshot sweep even when it does not stage anything.**
|
||||
Stock decides whether to take a *recursive* snapshot by its own rule, and on
|
||||
TrueNAS 26 that rule stopped being ours.
|
||||
|
||||
Up to 25.10, stock's `create_snapshot` called `get_dataset_recursive()` — the same
|
||||
function this module vendors — so "stock went recursive" and "we have something to
|
||||
stage" were the *same question*, and stock's non-recursive delete was correct for
|
||||
everything the patch declined to stage. TrueNAS 26 uses `filesystem.statfs`:
|
||||
`recursive = (path == the dataset's mountpoint)`. The two rules now disagree for a
|
||||
dataset whose only descendants are **ZVOLs** or **legacy/none-mountpoint** datasets
|
||||
— stock snapshots it recursively, while the patch sees nothing to stage.
|
||||
|
||||
The patch then handed the snapshot back to stock, which destroys the parent only.
|
||||
With no staging tree there was no sidecar, and the garbage collector only ever ran
|
||||
from the staging path — so nothing on the box would ever have found the children.
|
||||
Reproduced on the test VM: one orphaned snapshot per zvol, on every run, forever,
|
||||
with the backup reporting success. Ownership of the sweep is no longer conditional
|
||||
on staging.
|
||||
|
||||
- **The runtime resolved a *namespace*; the checker verified a *method*.** Those are
|
||||
different questions, and the gap is a false "ok". `get_service()` only proves a
|
||||
namespace is registered — it says nothing about whether `delete` still exists on it.
|
||||
So if iX guts the method while keeping the service (they have already done exactly
|
||||
that to `pool.snapshot.do_update` on master), `tools/compat.py` would fall through
|
||||
to `zfs.snapshot`, report the box healthy, and let the patch apply — while the
|
||||
runtime picked `pool.snapshot` and failed *every* delete, orphaning the whole tree.
|
||||
Both sides now ask the same question, and a test binds the two lists together.
|
||||
|
||||
- `query_filesystems()` **dropped malformed `zfs list` rows silently** — the last
|
||||
remaining silent-omission path, and a direct contradiction of this module's cardinal
|
||||
rule. It raises now. A missing `zfs` binary raised `FileNotFoundError` rather than
|
||||
`ZfsError`; also fixed.
|
||||
|
||||
- The snapshot retry loop **discarded the delete error** and reported every survivor
|
||||
as "(still busy?)" — naming the one cause that is benign and self-healing, and
|
||||
hiding the ones that are permanent. It keeps and reports the real error.
|
||||
|
||||
- The staging-failure handler could **lose the original exception** if its own cleanup
|
||||
sweep raised. An error handler must not be able to lose the error.
|
||||
|
||||
- **A snapshot delete that returns cleanly is not proof that anything was deleted.**
|
||||
The recursive sweep's fast path took the call's word for it and returned "no
|
||||
survivors" — so `cleanup_task` read that as a clean sweep and removed the sidecar,
|
||||
the only record the tree ever existed. Roughly 250 snapshots would have been orphaned
|
||||
on every run, with nothing left able to find them, and the backup reporting success.
|
||||
|
||||
This is not a hypothetical about a well-behaved API: iX has already gutted
|
||||
`pool.snapshot.do_update` on master into a no-op whose body is commented out and
|
||||
which returns `None`. A source check still sees the `def`; a runtime check still sees
|
||||
a callable method. Only asking ZFS can tell. The sweep now confirms against ZFS, and
|
||||
where it *cannot* confirm it keeps owning the tree rather than claiming success — a
|
||||
false survivor self-heals on the next run, a lost record never does.
|
||||
|
||||
- `_write_sidecar` **swallowed `OSError`**. The sidecar is the only thing that survives
|
||||
a middlewared restart; failing to write it is not fatal, but it must never be
|
||||
invisible. `_read_sidecar` had the mirror bug — it conflated "there is no sidecar"
|
||||
with "I could not read the sidecar", and `cleanup_task` then took the empty branch
|
||||
and **unlinked the only record** of a tree it had failed to read.
|
||||
|
||||
- **A dataset from another tree, mounted inside the backup path, was omitted
|
||||
silently.** The staging plan scopes by dataset *name*, which is correct — a dataset
|
||||
with no mountpoint cannot be scoped by path at all. But ZFS lets any dataset mount
|
||||
anywhere, so one from an unrelated tree can sit inside the path:
|
||||
|
||||
Tank/photos mountpoint=/mnt/Tap/apps/photos
|
||||
|
||||
It holds data inside the backed-up path, and `zfs snapshot -r Tap@…` does **not**
|
||||
cover it: recursion follows the dataset tree, not the directory tree. So there is no
|
||||
snapshot of it to stage, and no way to capture it consistently with the rest. It fell
|
||||
out of the name filter and vanished — not staged, not in `skipped`, no error, backup
|
||||
green. Stock has the same blind spot, but stock also refuses the nested config
|
||||
outright; this patch is what relaxes that guard, so the hole is this patch's to close.
|
||||
It now refuses, and names the offending datasets.
|
||||
|
||||
## v0.6.1 — 2026-07-13
|
||||
### Fixed
|
||||
|
||||
- **A reboot mid-backup orphaned the entire snapshot tree, permanently.** The sidecar
|
||||
is the record of which snapshots a run pinned — and it lives in `/run`, which is
|
||||
**tmpfs**. A reboot (or a crash) between taking the recursive snapshot and cleaning
|
||||
it up destroyed that record, leaving one snapshot per descendant dataset — **250+ on
|
||||
a real pool** — with nothing left pointing at them. Nothing would ever have found
|
||||
them again.
|
||||
|
||||
`gc_stale_snapshots()` is the backstop: it identifies leftovers **by name**, so it
|
||||
works when the record is gone. It runs at the start of every backup, after the
|
||||
sidecar reclaim — the recorded path stays authoritative, and the collector only ever
|
||||
mops up what the record lost.
|
||||
|
||||
Because it deletes data on a *name match* — a weaker claim than a recorded fact — the
|
||||
selection is a **pure function** with the harshest tests in the suite. A snapshot is
|
||||
collected only if **all** of these hold:
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| name is exactly `<dataset>@<task>-<YYYYMMDDHHMMSS>` | so `cloud_backup-5` never matches `cloud_backup-50`, an `auto-*` periodic snapshot, or anything a human made |
|
||||
| it is not the current run's | parent *and* children are excluded |
|
||||
| **nothing is mounted from it** | an in-flight run pins its own snapshots — this, not the age guard, is what protects a concurrent backup |
|
||||
| it is **over an hour old** | covers the seconds-long window where a live run has snapshotted but not yet mounted |
|
||||
|
||||
Verified against the real pool: of **4,728** snapshots — including **2,341** periodic
|
||||
ones — it selects exactly the orphans of the task being run, and nothing else.
|
||||
|
||||
## v0.6.0 — 2026-07-13
|
||||
### Added
|
||||
|
||||
@@ -66,6 +409,50 @@ worse than no alert, because one day it carries a security fix.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A few snapshots leaked on every nested run, forever.** Found on real hardware, in
|
||||
the one place it could be: a 256-snapshot backup of `/mnt/Tap` swept 253 cleanly and
|
||||
left **3 behind** with `dataset is busy`.
|
||||
|
||||
The cause is ZFS's own automount. Reading anything under
|
||||
`<dataset>/.zfs/snapshot/<snap>/` makes ZFS **automount that snapshot**, and it stays
|
||||
mounted for `zfs_expire_snapshot` seconds (**300** by default) after the last access.
|
||||
`teardown()` unmounts *our* bind mounts — but not the automount underneath — so
|
||||
`zfs destroy` refuses for exactly the datasets restic read most recently. Then
|
||||
`cleanup_task()` removed the sidecar anyway, destroying the only record that those
|
||||
snapshots existed. Nothing would ever have reclaimed them.
|
||||
|
||||
Three changes, and the third is the one that makes it safe rather than merely
|
||||
unlikely:
|
||||
- `release_snapdirs()` unmounts ZFS's own `.zfs/snapshot` automounts (deepest first)
|
||||
before deleting, so the snapshots are not busy in the first place.
|
||||
- `delete_snapshot_tree()` **retries** the transient busy, and **returns the
|
||||
snapshots it could not delete** instead of swallowing them.
|
||||
- **The sidecar is now removed only on a confirmed-clean sweep** — including on the
|
||||
staging-failure path, which used to remove it *before* the caller swept. The
|
||||
asymmetry is deliberate: a sidecar left behind when the tree is already gone costs
|
||||
one no-op delete on the next run, while a sidecar removed while the tree still
|
||||
exists is unrecoverable. Survivors are reclaimed by the next run.
|
||||
|
||||
**Expect the occasional straggler, and expect it to clean itself up.** On a
|
||||
256-snapshot tree this reliably sweeps ~255 immediately and may leave **one**: it is
|
||||
whatever restic read last, so its 300-second window has barely opened. That one is
|
||||
logged, its sidecar is kept, and the next run reclaims it before doing anything else.
|
||||
The leak is bounded at a single cycle rather than growing without limit — which is
|
||||
the property that actually matters. Blocking a backup job for five minutes to chase
|
||||
the last snapshot would be a worse trade, so it is not made.
|
||||
|
||||
- **Installing the patch permanently blocked updating it.** `install.sh` does
|
||||
`chmod +x update.sh`, and git recorded `update.sh` as `100644` — so the chmod was a
|
||||
*tracked modification*, and `update.sh` refuses to run over a dirty tree. Install
|
||||
once and you could never update again; the error even told you to run
|
||||
`git checkout -- .`, which just undoes the exec bit so the next install can re-dirty
|
||||
it. A real box sat on an old version for exactly this reason.
|
||||
|
||||
Fixed on both sides: the scripts `install.sh` chmods are now executable in git (so
|
||||
the chmod is a no-op), and `update.sh`'s dirty check now looks at **content**, not
|
||||
file mode — `git diff --numstat` reports `0 0` for a mode-only change. A test
|
||||
asserts every script in `install.sh`'s chmod loop is already `100755` in git.
|
||||
|
||||
- **Nested snapshots were broken on TrueNAS 24.10 and 25.04, and had been all
|
||||
along.** `SYNC_BLOCK`'s wrapper spelled out the stock signature and forwarded five
|
||||
arguments — but those releases declare `restic_backup(middleware, job,
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
truenas-truecloud-patch
|
||||
|
||||
Parts of this project were written with AI assistance (Claude); all of it is
|
||||
reviewed and tested before release.
|
||||
@@ -1,5 +1,7 @@
|
||||
# truenas-truecloud-patch
|
||||
|
||||
[](https://git.arch.fyi/flan/truenas-truecloud-patch/actions)
|
||||
|
||||
Extends TrueNAS SCALE's **TrueCloud Backup** to:
|
||||
|
||||
- back up to **Backblaze B2 and any S3-compatible provider**, not just Storj;
|
||||
@@ -22,7 +24,7 @@ Clone it onto a **pool** (not the boot device — that is wiped on TrueNAS upgra
|
||||
then run `install.sh` as root:
|
||||
|
||||
```bash
|
||||
git clone https://git.onetick.ninja/flan/truenas-truecloud-patch.git \
|
||||
git clone https://github.com/sudolulo/truenas-truecloud-patch.git \
|
||||
/mnt/tank/truenas-truecloud-patch # replace `tank` with your pool
|
||||
cd /mnt/tank/truenas-truecloud-patch
|
||||
sudo bash install.sh
|
||||
@@ -59,9 +61,10 @@ If something is wrong, the reason is in `apply.log` — start at
|
||||
| --- | --- | --- | --- |
|
||||
| 24.10.2.4 | ok | ok | — |
|
||||
| 25.04.2.6 | ok | ok | — |
|
||||
| 25.10.4 | ok | ok | nested + providers; 252-snapshot recursive backup of /mnt/Tap, 18m |
|
||||
| 26.0.0-BETA.3 _(unreleased)_ | ok | **BROKEN** | — |
|
||||
| master _(unreleased)_ | **BROKEN** | **BROKEN** | — |
|
||||
| 25.10.7 | ok | ok | — |
|
||||
| 24.10.2.5 _(unreleased)_ | ok | ok | — |
|
||||
| 26.0.0-RC.1 _(unreleased)_ | ok | ok | — |
|
||||
| master _(27-dev)_ | **BROKEN** | **BROKEN** | — |
|
||||
|
||||
| verdict | meaning |
|
||||
| --- | --- |
|
||||
@@ -72,17 +75,44 @@ If something is wrong, the reason is in `apply.log` — start at
|
||||
"ok" means *the patch's assumptions hold*, checked automatically against iX's
|
||||
source. It does not mean a human ran a backup on it — that is the
|
||||
**Hardware-verified** column, which is filled in by hand and only by doing it.
|
||||
|
||||
**`master` is not the next release.** iX branches each major off to its own
|
||||
`release/` line and master rolls straight on to the one after — so master is
|
||||
`27-dev` while 26 is still in beta. A **BROKEN** master means iX has changed
|
||||
something that will reach users *a major release from now*, not in the version you
|
||||
are about to install. Read the numbered rows for that.
|
||||
|
||||
A row like `25.10.5 _(unreleased)_` is the next maintenance release: branched by iX,
|
||||
not tagged yet, and the very next thing a 25.10.4 box gets. It is checked precisely
|
||||
because it is the one unshipped ref that reaches real users without warning.
|
||||
<!-- END COMPAT MATRIX -->
|
||||
|
||||
The table is **regenerated daily by CI** against iXsystems' actual middleware source
|
||||
— it is not a claim somebody typed once and forgot.
|
||||
|
||||
**TrueNAS 26: nested snapshots are not supported yet, and upgrading will not break
|
||||
you.** 26 rewrites `cloud_backup` and deletes the ZFS methods this module calls. On
|
||||
26 `apply.sh` finds that the assumptions no longer hold and **does not apply the
|
||||
module**: TrueNAS is left stock, B2/S3 keeps working, nested datasets are simply not
|
||||
covered, and the reason is named in `apply.log`. A broken backup is worse than a
|
||||
missing feature. Details: [How it works](docs/how-it-works.md#truenas-26).
|
||||
It is also **static analysis**: it proves the patch's assumptions still hold, which is
|
||||
a weaker claim than "a backup ran and a restore came back". For what has actually been
|
||||
run — which tasks, on which hardware, and the md5 of the file that came back — see
|
||||
[docs/verification.md](docs/verification.md).
|
||||
|
||||
**TrueNAS 26 is supported** as of v0.7.0, and was verified on a real
|
||||
**26.0.0-BETA.1** install: a 274-snapshot recursive backup of a 292-dataset pool, and a
|
||||
byte-identical restore of a four-level-deep child dataset. The *Hardware-verified*
|
||||
column tracks the newest beta iX has tagged (currently BETA.3), so it does not carry
|
||||
that mark — a build nobody has actually run a backup on does not get credit for one.
|
||||
|
||||
26 rewrites `cloud_backup` from async to synchronous and deletes the private ZFS
|
||||
methods this module used to call, so getting there took real work: the patch now
|
||||
injects the wrapper flavour that matches the installed middleware, reads dataset and
|
||||
snapshot lists **from ZFS rather than middleware** (whose queries hide TrueNAS's own
|
||||
datasets — 84 of 270 on a real pool, including live app data), and owns the snapshot
|
||||
sweep even when it stages nothing (26 decides `recursive` by a rule this patch does not
|
||||
share, and would otherwise orphan one snapshot per zvol on every run).
|
||||
|
||||
**And if a future TrueNAS breaks it, you get a missing feature, not a broken backup.**
|
||||
`apply.sh` re-checks the patch's assumptions at every boot and **refuses to apply a
|
||||
module whose assumptions no longer hold** — TrueNAS is left stock, and the reason is
|
||||
named in `apply.log`. Details: [How it works](docs/how-it-works.md#truenas-26).
|
||||
|
||||
---
|
||||
|
||||
@@ -125,6 +155,17 @@ alert, which both take the newest plain `vX.Y.Z` tag. That is what lets debuggin
|
||||
happen in `-rc` tags instead of in your notification bell — see
|
||||
[Releasing](docs/releasing.md).
|
||||
|
||||
**A second alert reports the patch not being loaded**, and this one you cannot
|
||||
turn off with `--no-update-alerts` — it is CRITICAL, hourly, and it means B2/S3
|
||||
backup tasks are about to fail. Being patched *on disk* and being patched *in the
|
||||
running middlewared* are different facts, and only middlewared can answer the
|
||||
second one: the patch stamps the objects it replaces, so a missing stamp means
|
||||
the process imported stock code. It fires if something detaches the patch overlay
|
||||
(a `systemd-sysext` merge over `/usr`, for instance) and the self-healing re-apply
|
||||
in the deferred restart could not put it back. `bash install.sh` clears it. It
|
||||
stays quiet when the kill switch is set, or when the providers module has been
|
||||
retired because TrueNAS went native.
|
||||
|
||||
The changelog is read from whichever forge `origin` points at, derived from the
|
||||
remote rather than hard-coded. That is not cosmetic: when the changelog cannot be
|
||||
read, the alert deliberately fires **anyway** rather than risk hiding a security
|
||||
@@ -207,5 +248,7 @@ and restores the original UI bundle from backup.
|
||||
- Filing a TrueNAS bug? **Remove the patch first** and reproduce on a stock system.
|
||||
- Provided as-is, no warranty. See LICENSE.
|
||||
|
||||
Parts of this project were written with AI assistance (Claude); all of it is reviewed
|
||||
and tested before release. Bugs are mine.
|
||||
## Support
|
||||
|
||||
If truenas-truecloud-patch is useful to you, consider supporting development via
|
||||
[GitHub Sponsors](https://github.com/sponsors/sudolulo) or [Ko-fi](https://ko-fi.com/sudolulo).
|
||||
|
||||
+94
-22
@@ -74,22 +74,46 @@ Two different things must survive two different events:
|
||||
and creates a transient systemd unit (`truecloud-mw-restart`, via
|
||||
`systemd-run --no-block`) running `patch/wait_restart.sh` — detached so it
|
||||
cannot disrupt the remainder of the boot sequence.
|
||||
5. **Once boot has settled, middlewared restarts once** and imports the
|
||||
patched modules from the overlay. `wait_restart.sh` holds the restart until
|
||||
the systemd boot job queue has drained (so in-flight `ix-*` units like
|
||||
`ix-reporting` finish first) *and* middlewared's docker/apps startup has
|
||||
reached a terminal state — plain unit ordering cannot see either, and
|
||||
restarting middlewared while they run kills apps and dashboard reporting
|
||||
5. **Once boot has settled, the patch is re-applied and middlewared restarts
|
||||
once**, importing the patched modules from the overlay. `wait_restart.sh`
|
||||
holds the restart until the systemd boot job queue has drained (so in-flight
|
||||
`ix-*` units like `ix-reporting` finish first) *and* middlewared's docker/apps
|
||||
startup has reached a terminal state — plain unit ordering cannot see either,
|
||||
and restarting middlewared while they run kills apps and dashboard reporting
|
||||
for the whole boot. S3/B2 backup support is then active until the next
|
||||
reboot, when the cycle repeats.
|
||||
|
||||
The **re-apply** in that sentence is load-bearing, not a safety blanket. The
|
||||
overlay from step 3 sits *inside* `/usr`, so anything that remounts that
|
||||
hierarchy detaches it, and two ordinary things do exactly that after our hook
|
||||
has finished: another PREINIT script running `systemd-sysext merge`/`refresh`
|
||||
over `/usr` (an out-of-tree nvidia driver, say), and middlewared's own
|
||||
`docker.configure_nvidia` when it brings docker up. Init scripts run
|
||||
sequentially in id order, so a hook registered after ours always wins — and
|
||||
ordering them differently would still not help, because `docker.configure_nvidia`
|
||||
fires at runtime. `wait_restart.sh` therefore re-runs `apply.sh` at the point
|
||||
where boot has settled and every such remount is behind it, re-mounting the
|
||||
overlay if it was torn off (same upper layer, so files patched in step 3
|
||||
reappear intact), then verifies the patch is really on the live path, restarts,
|
||||
and verifies again — retrying once if it was lost in between.
|
||||
|
||||
This is the failure that made it necessary: on 2026-08-19 the overlay was
|
||||
mounted at 16:41:56 and a sysext refresh unmerged and remerged `/usr` four
|
||||
seconds later. The restart at 16:47:24 loaded stock modules, and every B2
|
||||
backup failed for nineteen hours while `apply.log` said `OK` — because
|
||||
`apply.log` can only report what was written to disk, never what the restart
|
||||
imported. That second question is now asked from inside middlewared by an
|
||||
hourly CRITICAL alert (see [Update alerts](../README.md#update-alerts)).
|
||||
|
||||
What you will observe: one middlewared restart shortly after every boot (a
|
||||
brief web UI/API blip; running services are unaffected). Between steps 3
|
||||
and 5 there is a short window — typically well under a minute — where the UI
|
||||
already shows S3/B2 (the JS bundle is read from disk per request) but the
|
||||
backend is still stock. A backup job that fires inside that window fails once
|
||||
with `NotImplementedError` and succeeds on its next run; see
|
||||
[Troubleshooting](recovery.md) if it persists beyond boot.
|
||||
[Troubleshooting](recovery.md) if it persists beyond boot. If the backend is
|
||||
still stock an hour after boot, middlewared raises the "installed but NOT
|
||||
loaded" alert rather than leaving you to notice via a failed backup.
|
||||
|
||||
Manual runs of `bash patch/apply.sh` never trigger the restart — that only
|
||||
happens in boot context. `install.sh` and `recover.sh` perform their own
|
||||
@@ -158,7 +182,7 @@ python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
|
||||
| What you see | What it means |
|
||||
|---|---|
|
||||
| `[OK] providers`, `[OK]`/`[SKIP] nested_snapshots` | Fine. Nothing to do. |
|
||||
| `WARNING: … pattern not found` (UI) | The Angular bundle changed. The UI dropdown reverts to Storj-only, but **backups keep working** — create tasks with `create_task.py` meanwhile, and [open an issue](https://git.onetick.ninja/flan/truenas-truecloud-patch/issues) with your TrueNAS version. |
|
||||
| `WARNING: … pattern not found` (UI) | The Angular bundle changed. The UI dropdown reverts to Storj-only, but **backups keep working** — create tasks with `create_task.py` meanwhile, and [open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues) with your TrueNAS version. |
|
||||
| `WARNING: truecloud-patch is NOT COMPATIBLE with this TrueNAS version` | This TrueNAS changed middleware underneath the patch, and the named module was **deliberately not applied** — see `incompatible.json` for exactly which assumption broke. TrueNAS is left stock, so nothing is half-patched. Check [TrueNAS compatibility](../README.md#truenas-compatibility), then `bash update.sh` once a release supports your version; it re-applies itself on the next boot. This is **not** the kill switch and needs no manual reset. |
|
||||
| `[FAIL] providers` | **Your B2/S3 backups will not run.** middlewared is fine, but the credential/URL handling is gone. Open an issue with your version. |
|
||||
| `[FAIL] nested_snapshots` | The stock guard is back, so tasks with `snapshot = true` on a nested dataset will fail validation. Turn the option off on those tasks until it's fixed. |
|
||||
@@ -187,23 +211,71 @@ backup-breaking**, and none of them is visible from the `cloud_backup` files:
|
||||
| `get_dataset_recursive()` **deleted** from `plugins/cloud/snapshot.py` | `NameError` — the injected block called it out of the host module's namespace |
|
||||
| `plugins/zfs_/dataset.py` and `zfs_/snapshot.py` **deleted** | `zfs.dataset.query`, `zfs.snapshot.query` and `zfs.snapshot.delete` all vanish. 26 uses `filesystem.statfs` and `zfs.resource.*` |
|
||||
|
||||
The first two are fixed: the patch reads which flavour of `cloud_backup` your box
|
||||
declares and injects the wrapper that matches (one implementation of the real logic,
|
||||
two thin wrappers), and it carries its own copy of the deleted helper.
|
||||
All three are fixed as of **v0.7.0**, and 26 is supported.
|
||||
|
||||
The third is **not** fixed, and is why 26 reports BROKEN. Porting it means rewriting
|
||||
the module's ZFS calls onto 26's new API, and no single API spans 24.10 through 26 —
|
||||
so it needs a real 26 box to verify against, not a plausible-looking diff. Shipping a
|
||||
port nobody has run is exactly the failure this project exists to avoid.
|
||||
The first two were straightforward: the patch reads which flavour of `cloud_backup`
|
||||
your box declares and injects the wrapper that matches (one implementation of the real
|
||||
logic, two thin wrappers), and it carries its own copy of the deleted helper.
|
||||
|
||||
It is also the row that would have hurt most. `zfs.snapshot.delete` is what sweeps the
|
||||
recursive snapshot; without it, **every run would orphan one snapshot per descendant
|
||||
dataset — 250 on a real pool — forever.** The compatibility check caught it only
|
||||
because it now asserts the middleware *methods the patch calls*, not just the symbols
|
||||
it wraps.
|
||||
The third was not, and it is the one that would have hurt most — `zfs.snapshot.delete`
|
||||
is what sweeps the recursive snapshot, and without it **every run would orphan one
|
||||
snapshot per descendant dataset (250 on a real pool), forever, while reporting
|
||||
success.**
|
||||
|
||||
`master` (development after 26) reports BROKEN too: iXsystems are still reshaping
|
||||
The obvious port is to the public `pool.dataset.query` / `pool.snapshot.query`. Those
|
||||
methods exist, are documented, and are covered by iX's deprecation policy — and they
|
||||
are **not like-for-like replacements**. They apply a *visibility policy*: they hide the
|
||||
datasets TrueNAS considers its own (`ix-apps/*`, `.system/*`, `.ix-virt/*`). On a real
|
||||
pool that is **84 of 270 datasets, including live application data.** Staging from that
|
||||
view would have omitted every one of them from the backup — and the planner would never
|
||||
have seen them, so they would not have appeared in its "skipped" list either. A green
|
||||
backup, quietly missing data. The snapshot query lies the same way, so the sweep would
|
||||
have orphaned one snapshot per hidden dataset.
|
||||
|
||||
No source analysis could have caught that. The methods are all present and correctly
|
||||
shaped. Only running it could, which is why it took a real 26 box.
|
||||
|
||||
So the module now follows one rule:
|
||||
|
||||
> **Read the truth from ZFS. Make changes through middleware.**
|
||||
|
||||
Enumeration is `zfs list` — no policy can filter it, and it behaves identically on every
|
||||
release, which also means one code path instead of a version conditional. Mutation stays
|
||||
a middleware call, so TrueNAS's own bookkeeping stays consistent; an exact-name delete
|
||||
works fine even on a dataset the query hides. It is only enumeration that lies.
|
||||
|
||||
The snapshot *delete* still needs a namespace, and no single one spans every release —
|
||||
24.10 and 25.04 have `zfs.snapshot`, 26 has only `pool.snapshot`, 25.10 has both. So it
|
||||
is resolved at runtime, by asking whether the namespace can actually delete. `tools/compat.py`
|
||||
asks the identical question against iX's source, and a test binds the two lists together,
|
||||
so what CI verifies and what runs cannot drift apart.
|
||||
|
||||
### The one 26 changed that nothing warned about
|
||||
|
||||
Stock decides whether to take a **recursive** snapshot by its own rule, and on 26 that
|
||||
rule stopped being ours:
|
||||
|
||||
| | decides `recursive` by |
|
||||
| --- | --- |
|
||||
| stock ≤ 25.10 | `get_dataset_recursive()` — the same function this patch vendors |
|
||||
| **stock 26** | `filesystem.statfs`: `recursive = (path == the dataset's mountpoint)` |
|
||||
| this patch | `get_dataset_recursive()` — is a mounted *filesystem* child under the path? |
|
||||
|
||||
Up to 25.10 those were the *same question*, so a snapshot the patch declined to stage
|
||||
provably had no children and stock's non-recursive delete was correct. On 26 they
|
||||
disagree: a dataset whose only descendants are **zvols** or **legacy-mountpoint**
|
||||
datasets gets a recursive snapshot, while the patch sees nothing to stage. Stock then
|
||||
destroys the parent only — and with no staging tree there was no sidecar, and the
|
||||
garbage collector only ever ran from the staging path. Nothing on the box would ever
|
||||
have found the children.
|
||||
|
||||
It was reproduced on a 26 VM (one orphan per zvol, every run, backup green) and closed:
|
||||
**ownership of the sweep is no longer conditional on staging.**
|
||||
|
||||
`master` (development after 26) **does** report BROKEN: iXsystems are still reshaping
|
||||
these functions there, renaming `middleware` → `context` and `cloud_backup` → `entry`
|
||||
and adding a required `credentials` parameter. That is a moving target and is
|
||||
deliberately not chased; the check keeps reporting it until it settles into a beta,
|
||||
which is when it becomes worth fixing.
|
||||
which is when it becomes worth fixing. Until then, a box running master would simply
|
||||
not get the modules — `apply.sh` refuses to apply a module whose assumptions no longer
|
||||
hold, and says why in `apply.log`.
|
||||
|
||||
@@ -95,6 +95,40 @@ re-scan each time.
|
||||
|
||||
### Snapshot lifecycle
|
||||
|
||||
> **Two mechanisms clean up, and the second exists because the first can be destroyed.**
|
||||
>
|
||||
> 1. **The sidecar** records exactly which snapshots a run pinned, and is removed only
|
||||
> on a confirmed-clean sweep. Precise, and it survives a middlewared restart.
|
||||
> 2. **The garbage collector** finds leftovers by *name*, so it still works when the
|
||||
> sidecar is gone — and it can be: **the sidecar lives in `/run`, which is tmpfs.** A
|
||||
> reboot mid-backup takes it, and with it the only record of a 250-snapshot tree.
|
||||
>
|
||||
> The collector runs at the start of every backup, after the sidecar reclaim. It will
|
||||
> only touch a snapshot named `<dataset>@<task>-<timestamp>` that is not the current
|
||||
> run's, has **nothing mounted from it** (which is what protects a concurrently-running
|
||||
> backup), and is **over an hour old**. Periodic `auto-*` snapshots, other tasks'
|
||||
> snapshots, and anything you made by hand are structurally out of reach.
|
||||
|
||||
|
||||
> **A snapshot may survive a run, and that is expected.** ZFS **automounts**
|
||||
> `<dataset>/.zfs/snapshot/<snap>` the moment it is read, and holds it for
|
||||
> `zfs_expire_snapshot` seconds (**300** by default) after the last access. So
|
||||
> whatever restic read *last* is still pinned when we try to destroy it, and
|
||||
> `zfs destroy` refuses with `dataset is busy`.
|
||||
>
|
||||
> The patch unmounts those automounts itself and retries, which clears ~255 of 256 on
|
||||
> a real pool. The one that remains is **logged, its sidecar is kept, and the next run
|
||||
> reclaims it before doing anything else** — so the leak is bounded at a single cycle
|
||||
> instead of growing forever. Seeing one `could not delete snapshot … it will be
|
||||
> reclaimed on the next run` in the log is normal. Seeing the count *grow* run over run
|
||||
> is not, and would be a bug.
|
||||
>
|
||||
> This is why the sidecar is removed **only on a confirmed-clean sweep**: it is the
|
||||
> only record those snapshots exist, and a run that dropped it while they were still
|
||||
> around would orphan them permanently. That is precisely what happened before this was
|
||||
> fixed.
|
||||
|
||||
|
||||
`zfs.snapshot.delete` defaults to **`recursive=False`**, and stock
|
||||
`restic_backup()` calls it with no options. Stock is safe only because its
|
||||
validation means a *recursive* snapshot never actually happens in the field.
|
||||
|
||||
+32
-5
@@ -109,7 +109,7 @@ If a module shows `[FAIL]`:
|
||||
risk.
|
||||
4. **If the detail says the module doesn't exist**, a TrueNAS update renamed
|
||||
or restructured the internal API.
|
||||
[Open an issue](https://git.onetick.ninja/flan/truenas-truecloud-patch/issues)
|
||||
[Open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues)
|
||||
with your TrueNAS version number and the full verify output.
|
||||
|
||||
---
|
||||
@@ -120,8 +120,9 @@ If a module shows `[FAIL]`:
|
||||
|
||||
The traceback ends in `rclone/base.py` → `raise NotImplementedError` and
|
||||
contains no `_tc_` frames: the running middlewared is executing stock code.
|
||||
Either the deferred restart never fired, or the patch never landed on disk
|
||||
this boot. Diagnose in this order:
|
||||
Either the deferred restart never fired, the patch never landed on disk this
|
||||
boot, or it landed and was then torn off before the restart. Diagnose in this
|
||||
order:
|
||||
|
||||
```bash
|
||||
# Did apply.sh run this boot, at which version, and did it schedule the restart?
|
||||
@@ -130,11 +131,21 @@ tail -40 /mnt/tank/truenas-truecloud-patch/apply.log
|
||||
# Full check — compares the running process against the patch timestamp
|
||||
python3 /mnt/tank/truenas-truecloud-patch/patch/create_task.py verify
|
||||
|
||||
# Did the deferred restart unit run, fail, or never get created?
|
||||
systemctl status truecloud-mw-restart.service
|
||||
# What the deferred restart did -- re-apply, restart, and what it verified.
|
||||
# apply.log is the durable record; journald retention on a busy box is often
|
||||
# shorter than the gap between reboots, so the journal may have nothing left.
|
||||
grep wait_restart /mnt/tank/truenas-truecloud-patch/apply.log | tail -20
|
||||
journalctl -u truecloud-mw-restart.service --no-pager | tail -20
|
||||
|
||||
# Did something remount /usr and detach the patch overlay?
|
||||
systemd-sysext status
|
||||
findmnt -o TARGET,SOURCE /usr/lib/python3/dist-packages
|
||||
```
|
||||
|
||||
`systemctl status truecloud-mw-restart.service` reporting *"could not be
|
||||
found"* is **normal** — the unit is transient and is collected once it exits.
|
||||
It is not evidence that the restart was skipped.
|
||||
|
||||
- `verify` reports the process started **before** the patch → the restart
|
||||
didn't happen. `systemctl restart middlewared` fixes it immediately; the
|
||||
journal output above tells you why it was missed.
|
||||
@@ -145,6 +156,22 @@ journalctl -u truecloud-mw-restart.service --no-pager | tail -20
|
||||
- `apply.log` header shows `[v0.0.3]` or older → update:
|
||||
`git pull && bash install.sh` (v0.0.4 fixed patches not loading after
|
||||
reboot).
|
||||
- `apply.log` says the patch applied, but `findmnt` shows no `truecloud-mw`
|
||||
overlay on the dist-packages path → something remounted `/usr` after our
|
||||
PREINIT hook and detached it. `systemd-sysext status` names the culprit if it
|
||||
is a sysext (the `SINCE` column will sit a few seconds *after* the `apply.log`
|
||||
timestamp). Releases from 2026-08-26 on re-apply and verify immediately before
|
||||
the restart, so this should self-heal; if you are seeing it, update first.
|
||||
|
||||
**TrueNAS raises "truecloud-patch is installed but NOT loaded"**
|
||||
|
||||
The definitive symptom, and it does not depend on a backup failing first: the
|
||||
running middlewared has stock cloud_backup modules even though the patch is
|
||||
installed and its providers module is meant to be active. `bash install.sh`
|
||||
re-applies and restarts. The alert clears within the hour. It is silent when the
|
||||
kill switch is set or the providers module has been retired as native, and it is
|
||||
deliberately not muted by `update_alerts_disabled` — that silences release
|
||||
notifications, not a broken backup path.
|
||||
|
||||
**Apply log** (check after each reboot or install):
|
||||
```bash
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ are worth calling out, because nothing else would catch what they catch:
|
||||
newest CHANGELOG entry. `VERSION=` had silently drifted to three different
|
||||
values across the scripts before anything checked.
|
||||
|
||||
The project is hosted on **Gitea** (`git.onetick.ninja/flan/truenas-truecloud-patch`)
|
||||
The project is hosted on **Gitea** (`git.arch.fyi/flan/truenas-truecloud-patch`)
|
||||
and mirrored to GitHub. Both run the same workflows — Gitea reads
|
||||
`.github/workflows/` too — so a change is checked twice, on two independent runners.
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# What has actually been run
|
||||
|
||||
The support matrix in the README is **static analysis**: it proves the patch's
|
||||
assumptions about middlewared still hold. That is a strictly weaker claim than "a
|
||||
backup ran and a restore came back". This file is the stronger claim, and it is
|
||||
maintained by hand, because the only way to fill it in is to do it.
|
||||
|
||||
If you are deciding whether to trust this with your backups, read this file, not the
|
||||
matrix.
|
||||
|
||||
---
|
||||
|
||||
## v0.7.0 — TrueNAS 25.10.4 (production hardware)
|
||||
|
||||
Six live TrueCloud tasks, all `snapshot = true`, backing up to Backblaze B2. Three were
|
||||
exercised end to end, chosen to cover the three shapes the code handles differently:
|
||||
|
||||
| Task | Path | Shape | Result |
|
||||
| --- | --- | --- | --- |
|
||||
| 5 | `/mnt/Tap` | 191 nested datasets staged, 282-snapshot recursive tree | SUCCESS |
|
||||
| 7 | `/mnt/Tank/backups` | 215 filesystems **+ 2 zvols** | SUCCESS |
|
||||
| 9 | `/mnt/Tank/flan` | **no** nested filesystem children | SUCCESS |
|
||||
|
||||
After every run: **0 orphaned snapshots, 0 leaked bind mounts, 0 stale sidecars.**
|
||||
|
||||
**The restore.** `apps/vaultwarden/data/config.json` — a file inside a *child* dataset,
|
||||
which is exactly what stock TrueNAS cannot capture — was restored from B2 and compared
|
||||
against the live file:
|
||||
|
||||
live f809df6ba231986b1ba824044228a03a 1808 bytes
|
||||
restored f809df6ba231986b1ba824044228a03a 1808 bytes
|
||||
=> byte-identical
|
||||
|
||||
**The collector earned its keep on real data.** The pool was already carrying an orphan:
|
||||
`Tap/apps/prometheus@cloud_backup-5-20260713202355`, left behind by an earlier run when
|
||||
ZFS's automount held the snapshot busy past all four retries. The first v0.7.0 run found
|
||||
it by name, reclaimed it, and the pool's snapshot count went 2148 → 2147. That is the
|
||||
garbage collector doing the job it was written for, against a leak that was already
|
||||
there and that nothing else would ever have found.
|
||||
|
||||
**Boot path.** `apply.sh` is registered as a PREINIT `initshutdownscript`; it was
|
||||
re-run against the live middleware and left exactly one `TRUECLOUD_PATCH` marker in
|
||||
each patched module (a second copy stacked into a live middlewared module would break
|
||||
the box at boot). It correctly detected the box as **async** (`cloud_backup is async
|
||||
(TrueNAS <= 25.10)`) and injected the matching wrappers.
|
||||
|
||||
**Upgrade path.** `update.sh` was used to move the box from the release candidate to
|
||||
the stable tag, in detached HEAD at `v0.7.0`, which is how a user's box actually
|
||||
upgrades.
|
||||
|
||||
## v0.7.0 — TrueNAS 26.0.0-BETA.1 (VM)
|
||||
|
||||
A throwaway VM whose pool reproduces the production pool's *shape* — 292 datasets, 26
|
||||
`legacy` mountpoints, nesting five deep — because every bug found on the real box came
|
||||
from the shape of the pool, not the bytes in it. MinIO was not used; `rclone serve s3`
|
||||
(already on the box) provided the S3 target, so no real B2 credential ever entered the
|
||||
VM.
|
||||
|
||||
* 274-snapshot recursive backup of the 292-dataset pool. 0 orphans, 0 leaked mounts.
|
||||
* Restored `ix-apps/app_mounts/vaultwarden/pgData` — **four levels deep, and a dataset
|
||||
that middleware's own `pool.dataset.query` hides from itself** — byte-identical.
|
||||
* The zvol-orphan case was **reproduced with the fix disabled** (one orphan per zvol,
|
||||
every run, backup green), then **closed with it enabled**. See the CHANGELOG entry
|
||||
for why TrueNAS 26 decides `recursive` by a different rule than this patch decides
|
||||
`nested`.
|
||||
|
||||
## What is NOT covered
|
||||
|
||||
* **24.10 and 25.04** are `ok` in the matrix — the assumptions hold, checked against
|
||||
iX's source — but nobody has run a backup on them. The matrix says so.
|
||||
* **master** is BROKEN, and correctly reports so: iX renamed the leading parameters of
|
||||
`get_restic_config` and `restic_backup`. It is not a shipped release; the daily
|
||||
compatibility bot files it, and `apply.sh` would refuse to apply the modules on a box
|
||||
running it.
|
||||
* A **reboot** of the production box has not been done on v0.7.0. `apply.sh` was
|
||||
re-executed by hand against the live middleware, which exercises the same code path,
|
||||
but the PREINIT ordering itself has only been proven on earlier versions.
|
||||
+3
-3
@@ -4,7 +4,7 @@
|
||||
# Prerequisites: run as root on TrueNAS SCALE with middlewared running.
|
||||
# Clone this repository to a persistent ZFS pool first:
|
||||
#
|
||||
# git clone https://git.onetick.ninja/flan/truenas-truecloud-patch \
|
||||
# git clone https://github.com/sudolulo/truenas-truecloud-patch \
|
||||
# /mnt/<pool>/truenas-truecloud-patch
|
||||
# cd /mnt/<pool>/truenas-truecloud-patch && bash install.sh
|
||||
#
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="0.6.0"
|
||||
VERSION="0.8.0"
|
||||
|
||||
# The directory containing install.sh is the permanent install location.
|
||||
PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
@@ -72,7 +72,7 @@ done
|
||||
if [ ! -f "$PATCH_DIR/patch/apply.sh" ]; then
|
||||
echo "ERROR: patch files not found at $PATCH_DIR/patch/" >&2
|
||||
echo "Run install.sh from a clone of the repository on a persistent pool:" >&2
|
||||
echo " git clone https://git.onetick.ninja/flan/truenas-truecloud-patch \\" >&2
|
||||
echo " git clone https://github.com/sudolulo/truenas-truecloud-patch \\" >&2
|
||||
echo " /mnt/<pool>/truenas-truecloud-patch" >&2
|
||||
echo " cd /mnt/<pool>/truenas-truecloud-patch && bash install.sh" >&2
|
||||
exit 1
|
||||
|
||||
+104
-2
@@ -21,6 +21,7 @@ the way a `git fetch` from middlewared (running as root) would.
|
||||
|
||||
import datetime
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@@ -47,8 +48,8 @@ _VERSION_RE = re.compile(r'^VERSION="([^"]+)"', re.M)
|
||||
#: owner/repo out of any of:
|
||||
#: git@github.com:sudolulo/repo.git
|
||||
#: https://github.com/sudolulo/repo.git
|
||||
#: ssh://git@git.onetick.ninja:55214/flan/repo.git
|
||||
#: https://git.onetick.ninja/flan/repo.git
|
||||
#: ssh://git@git.arch.fyi:55214/flan/repo.git
|
||||
#: https://git.arch.fyi/flan/repo.git
|
||||
#: The SSH port is deliberately not captured: it is not the web port.
|
||||
_REMOTE_RE = re.compile(
|
||||
r"^(?:\w+://)?(?:[^@/]+@)?([^:/]+)(?::\d+)?[:/]([^/]+)/([^/]+?)(?:\.git)?/?$"
|
||||
@@ -77,6 +78,107 @@ class TrueCloudPatchSecurityUpdateAlertClass(AlertClass):
|
||||
)
|
||||
|
||||
|
||||
class TrueCloudPatchNotLoadedAlertClass(AlertClass):
|
||||
category = AlertCategory.SYSTEM
|
||||
level = AlertLevel.CRITICAL
|
||||
title = "truecloud-patch is installed but NOT loaded"
|
||||
text = (
|
||||
"truecloud-patch patched middlewared on disk, but this middlewared is "
|
||||
"running the STOCK cloud_backup modules -- B2 and S3 TrueCloud Backup "
|
||||
"tasks will fail with NotImplementedError. Something remounted /usr "
|
||||
"after the patch was applied (a systemd-sysext merge, or "
|
||||
"docker.configure_nvidia), detaching the patch overlay. Re-apply with: "
|
||||
"bash %(dir)s/install.sh"
|
||||
)
|
||||
|
||||
|
||||
class TrueCloudPatchNotLoadedAlertSource(ThreadedAlertSource):
|
||||
"""Does the middlewared running this check actually have the patch in it?
|
||||
|
||||
This is the one question apply.log cannot answer. apply.sh reports what it
|
||||
wrote to disk; whether the restart that followed imported those files is a
|
||||
separate fact, and on 2026-08-19 the two disagreed silently for nineteen
|
||||
hours while every B2 backup task failed. Asking from inside the process is
|
||||
exact -- the patch stamps the objects it replaces, so a missing stamp means
|
||||
this interpreter imported stock code.
|
||||
|
||||
Deliberately NOT silenced by the update-alert marker: that mutes release
|
||||
notifications, not a broken backup path. Only the patch's own kill switch
|
||||
(the `disabled` file, meaning the operator turned the patch off) stops it.
|
||||
"""
|
||||
|
||||
schedule = IntervalSchedule(datetime.timedelta(hours=1))
|
||||
run_on_backup_node = False
|
||||
|
||||
def check_sync(self):
|
||||
try:
|
||||
return self._check()
|
||||
except Exception:
|
||||
# An alert source must never take middlewared down with it.
|
||||
logger.debug("truecloud-patch loaded check failed", exc_info=True)
|
||||
return None
|
||||
|
||||
# -- internals ------------------------------------------------------------
|
||||
|
||||
def _check(self):
|
||||
if os.path.exists(os.path.join(PATCH_DIR, "disabled")):
|
||||
return None
|
||||
|
||||
# Only the providers module puts B2/S3 on the restic path. If it was
|
||||
# never applied here, or TrueNAS went native and it was retired, then
|
||||
# "not loaded" is the correct state and not a fault.
|
||||
status = self._hook_status()
|
||||
if not status:
|
||||
return None
|
||||
providers = status.get("patches", {}).get("providers", {})
|
||||
if not providers.get("active"):
|
||||
return None
|
||||
|
||||
if self._providers_loaded():
|
||||
return None
|
||||
|
||||
return Alert(
|
||||
TrueCloudPatchNotLoadedAlertClass,
|
||||
{"dir": PATCH_DIR},
|
||||
key=None,
|
||||
)
|
||||
|
||||
def _hook_status(self):
|
||||
try:
|
||||
with open(os.path.join(PATCH_DIR, "hook_status.json")) as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def _providers_loaded(self):
|
||||
"""True when THIS interpreter holds the patched provider objects.
|
||||
|
||||
Two independent stamps, because the two halves are written separately
|
||||
and either can be missing on its own:
|
||||
|
||||
* restic.py -- apply.sh sets `_truecloud_patched` on the wrapper it
|
||||
installs over `get_restic_config`.
|
||||
* b2.py -- apply.sh binds a B2-specific `get_restic_config` onto
|
||||
`B2RcloneRemote`. Comparing it against the base implementation is
|
||||
exact and survives renames of the patch's own helper.
|
||||
"""
|
||||
try:
|
||||
from middlewared.plugins.cloud_backup.restic import get_restic_config
|
||||
except Exception:
|
||||
return False
|
||||
if not getattr(get_restic_config, "_truecloud_patched", False):
|
||||
return False
|
||||
|
||||
try:
|
||||
from middlewared.rclone.base import BaseRcloneRemote
|
||||
from middlewared.rclone.remote.b2 import B2RcloneRemote
|
||||
except Exception:
|
||||
return False
|
||||
base = getattr(BaseRcloneRemote, "get_restic_config", None)
|
||||
b2 = getattr(B2RcloneRemote, "get_restic_config", None)
|
||||
return b2 is not None and b2 is not base
|
||||
|
||||
|
||||
class TrueCloudPatchUpdateAlertSource(ThreadedAlertSource):
|
||||
schedule = IntervalSchedule(datetime.timedelta(hours=24))
|
||||
run_on_backup_node = False
|
||||
|
||||
+83
-16
@@ -32,7 +32,7 @@
|
||||
# Derive PATCH_DIR from this script's location (parent of the patch/ directory).
|
||||
PATCH_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
LOG="$PATCH_DIR/apply.log"
|
||||
VERSION="0.6.0"
|
||||
VERSION="0.8.0"
|
||||
|
||||
# Rotate log at 512 KB to avoid unbounded growth on a system volume.
|
||||
# Keep two prior generations (.1 and .2) so the last three boots are always available.
|
||||
@@ -64,17 +64,45 @@ _ensure_writable() {
|
||||
rm -f "$dir/.truecloud-probe"
|
||||
return 0
|
||||
fi
|
||||
# Already our overlay on this exact directory from an earlier run this boot?
|
||||
# Not writable, so any overlay of ours listed on this directory is a
|
||||
# SHADOWED leftover rather than a working mount: something remounted the
|
||||
# hierarchy above it -- a systemd-sysext merge/refresh over /usr, or
|
||||
# middlewared's own docker.configure_nvidia -- and buried it. A live
|
||||
# overlay of ours is always writable, so this must never be treated as
|
||||
# "already done"; doing so is what let a buried overlay pass for a healthy
|
||||
# one and left the backend patch on disk but never loaded.
|
||||
if mount | grep -qF "truecloud-${tag} on ${dir} "; then
|
||||
return 0
|
||||
echo "NOTICE: a previous truecloud-${tag} overlay on $dir is shadowed --"
|
||||
echo "NOTICE: the hierarchy above it was remounted. Detaching and re-mounting."
|
||||
umount -l "$dir" 2>/dev/null
|
||||
fi
|
||||
# Keep the SAME upperdir across re-mounts: it holds everything patched
|
||||
# earlier this boot, so re-mounting restores those files intact instead of
|
||||
# re-deriving them. The workdir is scratch and must be empty, so it is
|
||||
# recreated -- a stale one left behind by a detached mount fails the mount.
|
||||
local upper="/run/truecloud-${tag}-upper" work="/run/truecloud-${tag}-work"
|
||||
mkdir -p "$upper" "$work"
|
||||
mkdir -p "$upper"
|
||||
rm -rf "$work" 2>/dev/null
|
||||
mkdir -p "$work"
|
||||
if mount -t overlay "truecloud-${tag}" \
|
||||
-o "lowerdir=$dir,upperdir=$upper,workdir=$work" "$dir" 2>/dev/null; then
|
||||
echo "OK: Mounted writable overlay on $dir"
|
||||
return 0
|
||||
fi
|
||||
# A lazily-detached overlay releases its workdir only once its last user is
|
||||
# gone, and overlayfs refuses a workdir that is still in use. That would turn
|
||||
# the re-mount this function exists to perform into a hard failure, so retry
|
||||
# once on a private workdir. It is scratch in /run (tmpfs) and goes away at
|
||||
# the next boot; the upperdir, which holds the patched files, is unchanged.
|
||||
work="/run/truecloud-${tag}-work.$$"
|
||||
rm -rf "$work" 2>/dev/null
|
||||
mkdir -p "$work"
|
||||
if mount -t overlay "truecloud-${tag}" \
|
||||
-o "lowerdir=$dir,upperdir=$upper,workdir=$work" "$dir" 2>/dev/null; then
|
||||
echo "OK: Mounted writable overlay on $dir (fresh workdir)"
|
||||
return 0
|
||||
fi
|
||||
rmdir "$work" 2>/dev/null
|
||||
echo "WARNING: overlay mount failed on $dir — backend patch will be skipped."
|
||||
return 1
|
||||
}
|
||||
@@ -196,6 +224,13 @@ _tc_native_nested=$(printf '%s' "$_tc_info" | sed -n '2p')
|
||||
SITE_PKG=$(printf '%s' "$_tc_info" | sed -n '3p')
|
||||
_MW_DIR=$(printf '%s' "$_tc_info" | sed -n '4p')
|
||||
|
||||
# Record the resolved middlewared directory so patch/wait_restart.sh can check,
|
||||
# without re-deriving any of this, whether the patched modules are still on the
|
||||
# live filesystem path at the moment it restarts middlewared.
|
||||
if [ -n "$_MW_DIR" ]; then
|
||||
printf '%s\n' "$_MW_DIR" > "$PATCH_DIR/.mw_dir" 2>/dev/null
|
||||
fi
|
||||
|
||||
# Nested support is opt-in; if it was never enabled, it cannot be the reason to
|
||||
# keep the patch alive.
|
||||
if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]; then
|
||||
@@ -535,8 +570,8 @@ if _tc_nested is not None:
|
||||
# snapshot=true, not just ours. Two consequences, and the second is worse:
|
||||
#
|
||||
# * everything below is a NEW failure mode for tasks that worked before we
|
||||
# were installed. A `zfs.dataset.query` that errors would break a
|
||||
# CloudSync job we have no business touching.
|
||||
# were installed. A `zfs list` that errors would break a CloudSync job
|
||||
# we have no business touching.
|
||||
# * if a CloudSync task ever were staged, nothing would ever tear it down:
|
||||
# the teardown is wired into cloud_backup's restic_backup finally, and
|
||||
# CRUD_BLOCK deliberately leaves CloudSync's nesting guard intact. The
|
||||
@@ -556,18 +591,28 @@ if _tc_nested is not None:
|
||||
# our staging plan would not -- silently omitting it from the backup.
|
||||
# Read afterwards, an unsnapshotted dataset instead trips the isdir()
|
||||
# check in plan_staging and fails the run loudly. Loud beats silent.
|
||||
datasets = middleware.call_sync(
|
||||
"zfs.dataset.query", [["type", "=", "FILESYSTEM"]]
|
||||
)
|
||||
# query_filesystems() reads ZFS directly. It deliberately does NOT use
|
||||
# pool.dataset.query: that applies a visibility policy and hides
|
||||
# TrueNAS-internal datasets (ix-apps/*, .system/*, .ix-virt/*) -- 84 of
|
||||
# 270 on a real pool, including live app data. Staging from the filtered
|
||||
# view omits them silently, which is the one thing this must never do.
|
||||
datasets = _tc_nested.query_filesystems(middleware)
|
||||
# OUR copy of get_dataset_recursive, not the host module's: TrueNAS 26
|
||||
# deleted that helper (create_snapshot uses filesystem.statfs now), so
|
||||
# calling it out of the module namespace is a NameError there.
|
||||
dataset, nested = _tc_nested.get_dataset_recursive(datasets, path)
|
||||
|
||||
if not nested:
|
||||
# No children: stock behaviour, untouched. Stock's `finally` owns
|
||||
# the snapshot from here (its non-recursive delete is correct,
|
||||
# because a non-nested snapshot has no children).
|
||||
# Nothing to STAGE -- but we still own the SWEEP, and that is not a
|
||||
# formality. Stock decides `recursive` by its own rule, and on 26 that
|
||||
# rule is no longer ours: it snapshots recursively whenever the backup
|
||||
# path IS the dataset's mountpoint (filesystem.statfs), while
|
||||
# get_dataset_recursive() sees nothing to stage when the only
|
||||
# descendants are ZVOLs or legacy/none-mountpoint datasets. Stock then
|
||||
# deletes the PARENT ONLY. Without this, one snapshot per descendant is
|
||||
# orphaned on every run, forever, with no sidecar and no GC to find it --
|
||||
# and the backup still reports success.
|
||||
_tc_nested.own_snapshot(middleware, name, snapshot, logger=_logger)
|
||||
return snapshot, snap_path
|
||||
|
||||
staging_root = _tc_nested.stage_nested(
|
||||
@@ -581,7 +626,19 @@ if _tc_nested is not None:
|
||||
# stays None and its `finally` deletes NOTHING. Sweep the tree ourselves
|
||||
# or leak the parent plus one snapshot per descendant dataset (160+ here)
|
||||
# on every failed run.
|
||||
_tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger)
|
||||
#
|
||||
# The sweep is itself wrapped: a cleanup that raises would REPLACE the
|
||||
# original exception with its own, hiding why the backup actually failed.
|
||||
# An error handler must not be able to lose the error.
|
||||
try:
|
||||
_tc_nested.delete_snapshot_tree(middleware, snapshot, logger=_logger)
|
||||
except Exception as _tc_sweep_err:
|
||||
if _logger:
|
||||
_logger.error(
|
||||
"truecloud-patch: could not sweep %s after a staging failure "
|
||||
"(%r) -- it is orphaned and must be deleted by hand",
|
||||
snapshot, _tc_sweep_err,
|
||||
)
|
||||
raise
|
||||
|
||||
return snapshot, staging_root
|
||||
@@ -977,8 +1034,13 @@ fi
|
||||
# runs (install.sh, recovery) never trigger a restart.
|
||||
#
|
||||
# The unit runs wait_restart.sh, which blocks until boot has actually
|
||||
# settled (systemd job queue drained, docker/apps state terminal) before
|
||||
# restarting. systemd ordering alone (After=multi-user.target, ≤ v0.0.4)
|
||||
# settled (systemd job queue drained, docker/apps state terminal), then
|
||||
# RE-APPLIES this script before restarting. The re-apply is not belt-and-
|
||||
# braces: our overlay lives inside /usr, and a systemd-sysext merge or
|
||||
# middlewared's docker.configure_nvidia remounts /usr *after* PREINIT and
|
||||
# detaches it, so what we patch here can be gone by restart time (seen
|
||||
# 2026-08-19). wait_restart.sh re-mounts and re-verifies at the moment it
|
||||
# matters. systemd ordering alone (After=multi-user.target, ≤ v0.0.4)
|
||||
# fired while ix-reporting and the docker/apps startup were still in flight
|
||||
# and killed both — apps and dashboard stats stayed down until the next
|
||||
# boot. No Type=oneshot: a oneshot's start job would hold the boot queue
|
||||
@@ -993,7 +1055,12 @@ echo "--- deferred restart ---"
|
||||
#
|
||||
# "No module active at all" cannot reach here: that is the kill-switch branch
|
||||
# above, which exits.
|
||||
if ! grep -aq middlewared "/proc/$PPID/cmdline" 2>/dev/null; then
|
||||
if [ "${TRUECLOUD_REAPPLY:-0}" = "1" ]; then
|
||||
# Invoked by patch/wait_restart.sh as its pre-restart re-apply pass. That
|
||||
# unit already exists to do the restart and verifies the result, so
|
||||
# scheduling another one here would be a loop.
|
||||
echo "Re-apply pass from wait_restart.sh — that unit owns the restart."
|
||||
elif ! grep -aq middlewared "/proc/$PPID/cmdline" 2>/dev/null; then
|
||||
echo "Manual run (parent is not middlewared) — no restart scheduled."
|
||||
elif [ "$_backend_ok" != "1" ]; then
|
||||
echo "Nothing landed on disk — no restart scheduled (nothing new to load)."
|
||||
|
||||
@@ -52,7 +52,7 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
__version__ = "0.6.0"
|
||||
__version__ = "0.8.0"
|
||||
|
||||
_PATCH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
_STATUS_FILE = os.path.join(_PATCH_DIR, "hook_status.json")
|
||||
|
||||
+3
-3
@@ -111,7 +111,7 @@ def main():
|
||||
print(
|
||||
"[truecloud-patch] WARNING: filterByProviders pattern not found in any JS bundle.\n"
|
||||
"[truecloud-patch] The TrueNAS webui may have been restructured in this version.\n"
|
||||
"[truecloud-patch] File an issue at https://git.onetick.ninja/flan/truenas-truecloud-patch\n"
|
||||
"[truecloud-patch] File an issue at https://github.com/sudolulo/truenas-truecloud-patch\n"
|
||||
f"[truecloud-patch] TrueNAS version info: {_tnversion()}"
|
||||
)
|
||||
return
|
||||
@@ -136,7 +136,7 @@ def main():
|
||||
f"[truecloud-patch] WARNING: {count} replacement(s) in {path}; "
|
||||
f"expected exactly 1 — skipping write to avoid corrupting the bundle.\n"
|
||||
f"[truecloud-patch] File an issue at "
|
||||
f"https://git.onetick.ninja/flan/truenas-truecloud-patch"
|
||||
f"https://github.com/sudolulo/truenas-truecloud-patch"
|
||||
)
|
||||
return
|
||||
|
||||
@@ -154,7 +154,7 @@ def main():
|
||||
"[truecloud-patch] The UI is UNCHANGED and still works. This means the "
|
||||
"pattern no longer fits this TrueNAS build.\n"
|
||||
"[truecloud-patch] File an issue at "
|
||||
"https://git.onetick.ninja/flan/truenas-truecloud-patch"
|
||||
"https://github.com/sudolulo/truenas-truecloud-patch"
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
+924
-64
File diff suppressed because it is too large
Load Diff
+114
-1
@@ -27,6 +27,50 @@
|
||||
# --wait` below waits for that same queue to drain — the unit would deadlock
|
||||
# on itself until the timeout. apply.sh schedules this with the default
|
||||
# service type, whose start job completes at fork.
|
||||
#
|
||||
# THE RE-APPLY PASS (added 2026-08-26). Applying the patch at PREINIT and
|
||||
# restarting later is only sound if the patched files are still on the live
|
||||
# path at the moment middlewared re-imports them. They may not be: our patch
|
||||
# lives in an overlay mounted *inside* /usr, and anything that remounts the
|
||||
# hierarchy above it detaches or buries that overlay. Two things on a normal
|
||||
# TrueNAS box do exactly that, both AFTER our PREINIT hook has run:
|
||||
#
|
||||
# - `systemd-sysext merge/refresh` over /usr (an nvidia sysext, for
|
||||
# instance) — `Unmerged '/usr'` then `Merged extensions into '/usr'`;
|
||||
# - middlewared's own `docker.configure_nvidia`, which merges the stock
|
||||
# nvidia sysext over /usr when it brings docker up.
|
||||
#
|
||||
# PREINIT scripts run sequentially in id order, so a hook registered after
|
||||
# ours always wins the race, silently. Observed 2026-08-19: our overlay was
|
||||
# mounted at 16:41:56 and a sysext refresh tore /usr down four seconds later;
|
||||
# the restart at 16:47:24 then loaded stock modules and every B2 cloud_backup
|
||||
# job failed for the next nineteen hours while apply.log said "OK".
|
||||
#
|
||||
# Ordering the hooks cannot fix this — docker.configure_nvidia re-merges at
|
||||
# runtime, long after every PREINIT hook is done. So instead of trusting the
|
||||
# PREINIT pass, re-apply immediately before the restart (apply.sh is
|
||||
# idempotent and re-mounts a lost overlay, keeping the same upperdir so
|
||||
# already-patched files survive), verify the marker is really on the live
|
||||
# path, and verify again afterwards.
|
||||
|
||||
PATCH_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
LOG="$PATCH_DIR/apply.log"
|
||||
|
||||
_log() { echo "[wait_restart] $*" >> "$LOG" 2>/dev/null; }
|
||||
|
||||
# Is the providers patch visible on the live filesystem path -- i.e. would a
|
||||
# middlewared starting right now import it? Reads the marker apply.sh leaves
|
||||
# in restic.py. Returns 0 when patched, 1 when stock, 2 when we cannot tell
|
||||
# (no recorded middlewared dir yet, or the file is gone).
|
||||
_patch_visible() {
|
||||
local mw_dir restic_py
|
||||
mw_dir=$(cat "$PATCH_DIR/.mw_dir" 2>/dev/null)
|
||||
[ -n "$mw_dir" ] || return 2
|
||||
restic_py="$mw_dir/plugins/cloud_backup/restic.py"
|
||||
[ -f "$restic_py" ] || return 2
|
||||
grep -q "TRUECLOUD_PATCH" "$restic_py" 2>/dev/null && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
# 1. systemd layer: wait for the boot job queue to drain. This covers every
|
||||
# ix-* oneshot still activating, including ix-reporting's in-flight midclt
|
||||
@@ -39,6 +83,9 @@ timeout 900 systemctl is-system-running --wait > /dev/null 2>&1
|
||||
# transitional states (PENDING/INITIALIZING/STOPPING/MIGRATING — see
|
||||
# middlewared/plugins/docker/state_utils.py). An empty answer means
|
||||
# midclt could not respond at all; keep waiting. Cap at 10 minutes.
|
||||
# This also covers docker.configure_nvidia, the runtime /usr re-merge:
|
||||
# waiting for docker to reach a terminal state means the merge that would
|
||||
# bury our overlay has already happened by the time we re-apply below.
|
||||
for _ in $(seq 1 120); do
|
||||
_status=$(midclt call docker.status 2>/dev/null \
|
||||
| grep -oE '"status": "[A-Z_]+"' | cut -d'"' -f4)
|
||||
@@ -52,4 +99,70 @@ done
|
||||
# queryable state (smb.configure and friends). Bounded insurance.
|
||||
sleep 30
|
||||
|
||||
exec systemctl try-restart middlewared
|
||||
# 4. Re-apply pass. Boot has settled, so every sysext merge and docker nvidia
|
||||
# configuration that could bury our overlay is behind us. Re-running
|
||||
# apply.sh is cheap and idempotent: it re-mounts the overlay if it was
|
||||
# detached (same upperdir, so files patched at PREINIT reappear intact)
|
||||
# and re-patches anything that reverted to stock.
|
||||
_patch_visible
|
||||
case $? in
|
||||
0) _log "providers patch still visible on the live path before restart" ;;
|
||||
1) _log "PATCH LOST since PREINIT (something remounted /usr) — re-applying" ;;
|
||||
*) _log "cannot confirm patch state before restart — re-applying anyway" ;;
|
||||
esac
|
||||
|
||||
TRUECLOUD_REAPPLY=1 /bin/bash "$PATCH_DIR/patch/apply.sh"
|
||||
|
||||
if ! _patch_visible; then
|
||||
_log "WARNING: patch is STILL not on the live path after the re-apply pass;"
|
||||
_log "WARNING: restarting anyway, but middlewared will load stock modules."
|
||||
fi
|
||||
|
||||
# 5. The restart itself.
|
||||
systemctl try-restart middlewared
|
||||
|
||||
# 6. Record what the restart landed on -- but do NOT restart again on a miss.
|
||||
#
|
||||
# `try-restart` returns as soon as middlewared is READY; it then brings docker
|
||||
# up asynchronously, and `docker.configure_nvidia` merges the stock nvidia
|
||||
# sysext over /usr at that point. That detaches our overlay AFTER the new
|
||||
# middlewared has already imported the patched modules -- so a disk check here
|
||||
# can report "missing" on a perfectly healthy system. Restarting on that signal
|
||||
# would restart a correctly-patched middlewared and then hit the same race
|
||||
# again, so the disk is deliberately not treated as a verdict after the restart.
|
||||
#
|
||||
# The authoritative answer is whether the running process holds the patch, and
|
||||
# only middlewared can answer that. The alert source installed by apply.sh
|
||||
# checks exactly that, in-process and hourly, and is what reports a genuine
|
||||
# miss. What is still worth doing here is putting the overlay back, so the next
|
||||
# middlewared restart -- whenever and whyever it happens -- finds patched files.
|
||||
if _patch_visible; then
|
||||
_log "OK: providers patch present on the live path across the restart"
|
||||
else
|
||||
_log "overlay detached again after the restart (expected when docker's"
|
||||
_log "nvidia sysext merge follows it) -- re-mounting for the next restart."
|
||||
_log "Whether THIS middlewared loaded the patch is answered in-process by"
|
||||
_log "the 'installed but NOT loaded' alert, not by this check."
|
||||
|
||||
# Preserve hook_status.json's patched_at across this re-mount.
|
||||
#
|
||||
# create_task.py verify decides "loaded" by comparing middlewared's start
|
||||
# time against patched_at. This re-apply restores the SAME patch the boot
|
||||
# pass already applied, but it runs *after* the restart -- so letting it
|
||||
# re-stamp would make patched_at newer than the process that correctly
|
||||
# imported the patch, and verify would report FAIL forever, on every boot
|
||||
# where docker's sysext merge detaches the overlay. That is precisely the
|
||||
# lying-status failure this release exists to remove, so do not introduce a
|
||||
# new one. The snapshot lives in /run, never in the repo: a leftover file
|
||||
# there would leave the tree dirty and update.sh refuses to run over that.
|
||||
_saved_status=/run/truecloud-hook_status.pre
|
||||
cp -p "$PATCH_DIR/hook_status.json" "$_saved_status" 2>/dev/null
|
||||
|
||||
TRUECLOUD_REAPPLY=1 /bin/bash "$PATCH_DIR/patch/apply.sh"
|
||||
|
||||
if [ -f "$_saved_status" ]; then
|
||||
mv -f "$_saved_status" "$PATCH_DIR/hook_status.json" 2>/dev/null
|
||||
fi
|
||||
fi
|
||||
|
||||
_log "=== deferred restart complete ==="
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@
|
||||
# bash /mnt/tank/truenas-truecloud-patch/patch/apply.sh
|
||||
# systemctl restart middlewared
|
||||
|
||||
VERSION="0.6.0"
|
||||
VERSION="0.8.0"
|
||||
|
||||
PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
|
||||
Regular → Executable
+322
-14
@@ -140,19 +140,60 @@ class TestSnapshotLeak:
|
||||
every path that creates one must also sweep the whole tree.
|
||||
"""
|
||||
|
||||
def test_staging_failure_deletes_the_snapshot_tree(self):
|
||||
# On a staging failure, sync.py's `snapshot, local_path = await
|
||||
# create_snapshot(...)` never completes, so its local `snapshot` stays
|
||||
# None and its finally deletes nothing. We must sweep it ourselves.
|
||||
block = extract_blocks()["SNAPSHOT_ASYNC"]
|
||||
assert "except Exception:" in block
|
||||
assert "delete_snapshot_tree" in block
|
||||
assert "raise" in block
|
||||
# The behaviour these once asserted as substrings -- the sweep, the re-raise, the
|
||||
# teardown in the finally -- is now asserted STRUCTURALLY, against the parsed
|
||||
# block: see TestTheStagingFailurePathReallyReRaises and
|
||||
# TestTheSyncBlockAlwaysTearsDown. As substring checks they were satisfied by
|
||||
# COMMENTS ("a cleanup that raises...", "cleanup_task gets logger=None"), so
|
||||
# deleting the actual `raise` and the actual cleanup call both left the suite
|
||||
# green -- reinstating a silently-empty backup and ~250 orphans per run.
|
||||
|
||||
def test_sync_block_cleans_up_on_every_path(self):
|
||||
block = extract_blocks()["SYNC_ASYNC"]
|
||||
assert "finally:" in block
|
||||
assert "cleanup_task" in block
|
||||
def test_the_snapshot_block_still_owns_the_snapshot_when_not_staging(self):
|
||||
# The TrueNAS 26 zvol/legacy orphan: stock decides `recursive` by its own rule
|
||||
# (path == mountpoint) and deletes only the parent, so we must record the
|
||||
# snapshot even on the path where we stage nothing.
|
||||
for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
|
||||
stage = functions(tree_of(name), "_tc_stage")[0]
|
||||
assert calls_to(stage, "_tc_nested.own_snapshot"), (
|
||||
f"{name} hands an unstaged snapshot back to stock, whose delete is "
|
||||
f"non-recursive -- every zvol/legacy child is orphaned, every run"
|
||||
)
|
||||
|
||||
def test_the_staging_plan_is_enumerated_from_ZFS(self):
|
||||
for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
|
||||
stage = functions(tree_of(name), "_tc_stage")[0]
|
||||
assert calls_to(stage, "_tc_nested.query_filesystems"), (
|
||||
"the staging plan must come from query_filesystems() (which reads ZFS "
|
||||
"unfiltered); middleware's query hides ix-apps/*, .system/*, .ix-virt/*"
|
||||
)
|
||||
assert not calls_to(stage, "middleware.call_sync"), (
|
||||
"the block calls middleware directly again -- its dataset/snapshot "
|
||||
"queries are FILTERED and silently omit 84 of 270 datasets"
|
||||
)
|
||||
|
||||
def test_the_vendored_helper_is_used_not_the_host_module(self):
|
||||
# TrueNAS 26 DELETED get_dataset_recursive from plugins/cloud/snapshot.py, so
|
||||
# calling it out of the host module's namespace is a NameError there.
|
||||
for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
|
||||
stage = functions(tree_of(name), "_tc_stage")[0]
|
||||
assert calls_to(stage, "_tc_nested.get_dataset_recursive"), (
|
||||
"must call OUR vendored copy: TrueNAS 26 deleted the host's"
|
||||
)
|
||||
|
||||
def test_datasets_are_enumerated_AFTER_the_snapshot(self):
|
||||
# A dataset created between the listing and the snapshot would be captured by
|
||||
# the recursive snapshot but missing from the staging plan -- silently omitted.
|
||||
# Read afterwards, it instead trips plan_staging's probe and fails loudly.
|
||||
for name in ("SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"):
|
||||
src = extract_blocks()[name]
|
||||
code = "\n".join(
|
||||
ln for ln in src.splitlines() if not ln.lstrip().startswith("#")
|
||||
)
|
||||
# _tc_stage receives `snapshot` as a parameter -- i.e. it is taken by the
|
||||
# caller, before any of this runs. If the enumeration ever moves ahead of
|
||||
# create_snapshot it can only do so by leaving _tc_stage.
|
||||
assert "def _tc_stage(middleware, path, name, snapshot, snap_path)" in code
|
||||
assert "query_filesystems" in code
|
||||
|
||||
|
||||
def test_crud_block_is_scoped_to_cloud_backup():
|
||||
@@ -438,8 +479,275 @@ class TestOnlyOurOwnTasksAreTouched:
|
||||
# The point is to add NO new failure mode to a CloudSync task. If any
|
||||
# middleware call happened before the bail-out, we would already have broken
|
||||
# the thing we are trying not to touch.
|
||||
#
|
||||
# Checked against whichever interactions the block ACTUALLY contains, not a
|
||||
# fixed list: the dataset query moved behind `_tc_nested.query_filesystems()`
|
||||
# when it switched to the public pool.* API, and a hardcoded
|
||||
# `middleware.call_sync(` simply stopped being found -- a test that silently
|
||||
# stops testing is worse than no test.
|
||||
block = extract_blocks()[name]
|
||||
gate = block.index('if not name.startswith("cloud_backup"):')
|
||||
for call in ("middleware.call_sync(", "_tc_nested.stage_nested(",
|
||||
"_tc_nested.delete_snapshot_tree("):
|
||||
|
||||
interactions = [
|
||||
"middleware.call_sync(",
|
||||
"_tc_nested.query_filesystems(",
|
||||
"_tc_nested.stage_nested(",
|
||||
"_tc_nested.delete_snapshot_tree(",
|
||||
]
|
||||
present = [c for c in interactions if c in block]
|
||||
assert present, "found no middleware interaction at all -- the test is vacuous"
|
||||
for call in present:
|
||||
assert gate < block.index(call), f"{call} runs before the cloud_backup gate"
|
||||
|
||||
|
||||
# ── structural assertions ────────────────────────────────────────────────────
|
||||
#
|
||||
# `assert "raise" in block` was TRUE because a COMMENT in the block says "a cleanup
|
||||
# that raises would replace the original exception". `assert "cleanup_task" in block`
|
||||
# was TRUE because a comment says "cleanup_task gets logger=None". Deleting the actual
|
||||
# `raise`, and deleting the actual cleanup call from the `finally`, both left the suite
|
||||
# green -- while reinstating, respectively, a silently-empty backup and ~250 orphaned
|
||||
# snapshots per run.
|
||||
#
|
||||
# A test that a comment can satisfy is not a test. These parse the block and assert on
|
||||
# the CODE.
|
||||
|
||||
def tree_of(name):
|
||||
return ast.parse(textwrap.dedent(extract_blocks()[name]))
|
||||
|
||||
|
||||
def functions(tree, name):
|
||||
return [
|
||||
n for n in ast.walk(tree)
|
||||
if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef) and n.name == name
|
||||
]
|
||||
|
||||
|
||||
def calls_to(node, dotted):
|
||||
"""Every Call in `node` whose callee renders as `dotted` (e.g. a.b.c)."""
|
||||
out = []
|
||||
for n in ast.walk(node):
|
||||
if isinstance(n, ast.Call):
|
||||
try:
|
||||
if ast.unparse(n.func) == dotted:
|
||||
out.append(n)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
class TestTheStagingFailurePathReallyReRaises:
|
||||
"""If staging fails and we swallow it, restic backs up the UN-STAGED path.
|
||||
|
||||
That is the silently-empty backup this entire module exists to prevent: stock
|
||||
points the tool at the parent's `.zfs/snapshot/`, where child datasets are
|
||||
invisible. The exception MUST propagate.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
|
||||
def test_the_handler_sweeps_the_snapshot_and_re_raises(self, name):
|
||||
stage = functions(tree_of(name), "_tc_stage")
|
||||
assert stage, "_tc_stage is gone"
|
||||
|
||||
handlers = [
|
||||
h for t in ast.walk(stage[0]) if isinstance(t, ast.Try)
|
||||
for h in t.handlers
|
||||
]
|
||||
assert handlers, "the staging failure handler is gone"
|
||||
|
||||
sweeps = any(calls_to(h, "_tc_nested.delete_snapshot_tree") for h in handlers)
|
||||
assert sweeps, (
|
||||
"a staging failure no longer sweeps the snapshot. sync.py's `snapshot` "
|
||||
"local stays None, so ITS finally deletes nothing -- the whole tree leaks "
|
||||
"on every failed run."
|
||||
)
|
||||
|
||||
# A bare `raise` directly in the handler body -- not one nested inside the
|
||||
# defensive try/except that wraps the sweep.
|
||||
reraises = any(
|
||||
any(isinstance(s, ast.Raise) and s.exc is None for s in h.body)
|
||||
for h in handlers
|
||||
)
|
||||
assert reraises, (
|
||||
"the staging failure is SWALLOWED. restic then runs against the un-staged "
|
||||
"path and uploads a near-empty tree, reporting SUCCESS."
|
||||
)
|
||||
|
||||
|
||||
class TestTheSyncBlockAlwaysTearsDown:
|
||||
"""The teardown is what unmounts the staging tree and sweeps the snapshot.
|
||||
|
||||
It must run on EVERY exit from restic_backup -- success, failure, or exception --
|
||||
or the bind mounts pin the snapshot and the tree is orphaned.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("name", ["SYNC_ASYNC", "SYNC_SYNC"])
|
||||
def test_cleanup_runs_in_a_finally(self, name):
|
||||
fns = functions(tree_of(name), "restic_backup")
|
||||
assert fns, "the restic_backup wrapper is gone"
|
||||
|
||||
tries = [t for t in ast.walk(fns[0]) if isinstance(t, ast.Try) and t.finalbody]
|
||||
assert tries, "restic_backup no longer has a try/finally"
|
||||
|
||||
cleans = any(
|
||||
"cleanup_task" in ast.unparse(stmt)
|
||||
for t in tries for stmt in t.finalbody
|
||||
)
|
||||
assert cleans, (
|
||||
"cleanup_task is not called in the finally. The staging tree is never torn "
|
||||
"down, its bind mounts pin the snapshot, and ~250 snapshots leak per run."
|
||||
)
|
||||
|
||||
|
||||
class TestTheBlockingWorkNeverRunsOnTheEventLoop:
|
||||
"""`zfs list` and `call_sync` are BLOCKING. On <=25.10 these blocks are async.
|
||||
|
||||
Running them directly on middlewared's event loop stalls the whole daemon.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("name,fn", [
|
||||
("SNAPSHOT_ASYNC", "create_snapshot"),
|
||||
("SYNC_ASYNC", "restic_backup"),
|
||||
])
|
||||
def test_the_async_flavour_hops_to_a_thread(self, name, fn):
|
||||
fns = functions(tree_of(name), fn)
|
||||
assert fns and isinstance(fns[0], ast.AsyncFunctionDef)
|
||||
assert calls_to(fns[0], "middleware.run_in_thread"), (
|
||||
f"{name}.{fn} does the blocking work on the asyncio event loop"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("name,fn", [
|
||||
("SNAPSHOT_SYNC", "create_snapshot"),
|
||||
("SYNC_SYNC", "restic_backup"),
|
||||
])
|
||||
def test_the_sync_flavour_does_not(self, name, fn):
|
||||
# On 26 stock already runs this in the thread pool; hopping again would be
|
||||
# wrong (and there is no event loop to protect).
|
||||
fns = functions(tree_of(name), fn)
|
||||
assert fns and isinstance(fns[0], ast.FunctionDef)
|
||||
assert not calls_to(fns[0], "middleware.run_in_thread")
|
||||
|
||||
|
||||
def test_the_flavour_mapping_is_not_inverted():
|
||||
# `_snapshot_block = SNAPSHOT_ASYNC if _flavour else SNAPSHOT_SYNC` -- inverting it
|
||||
# injects an async wrapper on 26 (a coroutine gets unpacked as a tuple) or a sync
|
||||
# one on 25.10 (the event loop blocks). Every nested backup breaks, both ways.
|
||||
with open(APPLY_SH, encoding="utf-8") as fh:
|
||||
code = " ".join(
|
||||
ln for ln in fh.read().splitlines() if not ln.lstrip().startswith("#")
|
||||
)
|
||||
code = re.sub(r"\s+", " ", code) # the assignments are space-aligned
|
||||
for block in ("SNAPSHOT", "CRUD", "SYNC"):
|
||||
assert f"{block}_ASYNC if _flavour else {block}_SYNC" in code, (
|
||||
f"the {block} flavour mapping is missing or inverted: _flavour is True for "
|
||||
f"an ASYNC middleware, so it must select {block}_ASYNC"
|
||||
)
|
||||
|
||||
|
||||
# ── the compat preflight ─────────────────────────────────────────────────────
|
||||
#
|
||||
# This is the guard that stands between a broken middleware and a live NAS: at every
|
||||
# boot, apply.sh checks the patch's assumptions against the middlewared actually
|
||||
# installed, and REFUSES to apply a module whose assumptions no longer hold.
|
||||
#
|
||||
# It had no test. An audit turned it into a no-op eight different ways -- `verdict()`
|
||||
# always returning 'ok', the broken branch never firing, the kill switch never honoured
|
||||
# -- and the suite stayed green every time. The most consequential safety net in the
|
||||
# project was unguarded.
|
||||
|
||||
def preflight_heredoc():
|
||||
"""The preflight's Python, lifted out of apply.sh and made runnable.
|
||||
|
||||
Extracted, not reimplemented: a reimplementation would happily pass while the
|
||||
SHIPPED preflight stayed broken, which is exactly the failure being guarded.
|
||||
"""
|
||||
with open(APPLY_SH, encoding="utf-8") as fh:
|
||||
sh = fh.read()
|
||||
# Line-based: the compat heredoc opens with `<<'PYEOF'` on the _tc_compat line and
|
||||
# closes at the next bare PYEOF. (A regex that matched `<< 'PYEOF'` silently found
|
||||
# the OTHER heredoc and ran a different script entirely.)
|
||||
lines = sh.splitlines()
|
||||
start = next(
|
||||
i for i, ln in enumerate(lines)
|
||||
if ln.startswith("_tc_compat=$(") and "<<'PYEOF'" in ln
|
||||
)
|
||||
end = next(i for i in range(start + 1, len(lines)) if lines[i].strip() == "PYEOF")
|
||||
m = "\n".join(lines[start + 1:end])
|
||||
assert m, "could not find the compat preflight heredoc in apply.sh"
|
||||
return m
|
||||
|
||||
|
||||
def run_preflight(result, tmp_path):
|
||||
"""Run the SHIPPED preflight against a fake compat.check_tree result.
|
||||
|
||||
The heredoc does `import sys`, so a fake `sys` in the namespace is immediately
|
||||
rebound to the real module -- drive the real one instead.
|
||||
"""
|
||||
import contextlib
|
||||
import io
|
||||
import sys
|
||||
import types
|
||||
|
||||
src = preflight_heredoc()
|
||||
fake = types.ModuleType("compat")
|
||||
fake.check_tree = lambda _mw: result
|
||||
|
||||
saved_mod = sys.modules.get("compat")
|
||||
saved_argv = sys.argv
|
||||
sys.modules["compat"] = fake
|
||||
sys.argv = ["x", "/patch", "/mw", str(tmp_path / "compat.json")]
|
||||
|
||||
buf = io.StringIO()
|
||||
try:
|
||||
with contextlib.redirect_stdout(buf):
|
||||
exec(compile(src, "apply.sh:preflight", "exec"), {"__name__": "__main__"}) # noqa: S102
|
||||
except SystemExit:
|
||||
pass
|
||||
finally:
|
||||
sys.argv = saved_argv
|
||||
if saved_mod is not None:
|
||||
sys.modules["compat"] = saved_mod
|
||||
else:
|
||||
sys.modules.pop("compat", None)
|
||||
return buf.getvalue().splitlines()
|
||||
|
||||
|
||||
def _mod(ok=True, native=False, unknown=False, problems=()):
|
||||
return {"ok": ok, "native": native, "unknown": unknown, "problems": list(problems)}
|
||||
|
||||
|
||||
class TestTheBootPreflightRefusesABrokenMiddleware:
|
||||
def test_a_healthy_tree_is_ok(self, tmp_path):
|
||||
out = run_preflight({"providers": _mod(), "nested": _mod()}, tmp_path)
|
||||
assert out[:2] == ["ok", "ok"]
|
||||
|
||||
def test_a_broken_module_is_reported_broken(self, tmp_path):
|
||||
out = run_preflight({
|
||||
"providers": _mod(),
|
||||
"nested": _mod(ok=False, problems=[
|
||||
{"id": "x", "detail": "gone", "why": "orphans every run"},
|
||||
]),
|
||||
}, tmp_path)
|
||||
assert "broken" in out, (
|
||||
"the preflight did not report a module whose assumptions FAILED. It would "
|
||||
"be injected into a middleware it does not fit -- broken backups, "
|
||||
"discovered at restore time."
|
||||
)
|
||||
|
||||
def test_a_module_that_went_NATIVE_is_also_not_applied(self, tmp_path):
|
||||
# 'native' answers "do we still need it?", 'ok' answers "is it safe to inject?".
|
||||
# Applying a module TrueNAS now implements itself is not safe either.
|
||||
out = run_preflight({
|
||||
"providers": _mod(),
|
||||
"nested": _mod(ok=False, native=True),
|
||||
}, tmp_path)
|
||||
assert "broken" in out
|
||||
|
||||
def test_an_UNKNOWN_verdict_is_not_reported_as_broken(self, tmp_path):
|
||||
# A network error or an unreadable file is not iX deleting our symbols. Calling
|
||||
# it broken would switch a working module off on a healthy box.
|
||||
out = run_preflight({
|
||||
"providers": _mod(unknown=True),
|
||||
"nested": _mod(unknown=True),
|
||||
}, tmp_path)
|
||||
assert "broken" not in out
|
||||
|
||||
+488
-41
@@ -15,6 +15,7 @@ dangerous thing this file can say -- it means "TrueNAS does this now, retire the
|
||||
module" -- and it rests on nothing more than a substring match.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
@@ -23,6 +24,7 @@ import pytest
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
|
||||
|
||||
import compat # noqa: E402
|
||||
import compat_publish # noqa: E402
|
||||
from compat import ( # noqa: E402
|
||||
NESTED,
|
||||
PROVIDERS,
|
||||
@@ -51,24 +53,44 @@ GOOD = {
|
||||
"async def restic_backup(middleware, job, cloud_backup, dry_run=False, "
|
||||
"rate_limit=None):\n pass\n"
|
||||
),
|
||||
# The middlewared METHODS the injected code calls. TrueNAS 26 deleted both of
|
||||
# these files, taking zfs.dataset.query / zfs.snapshot.query / zfs.snapshot.delete
|
||||
# with them -- see TestMiddlewareMethodsWeCall.
|
||||
"plugins/zfs_/dataset.py": (
|
||||
"class ZFSDataset(CRUDService):\n"
|
||||
# The middlewared METHODS the injected code calls. These now go through the
|
||||
# PUBLIC pool.* API: TrueNAS 26 deleted plugins/zfs_/ outright, taking the whole
|
||||
# private zfs.* service with it -- see TestMiddlewareMethodsWeCall.
|
||||
#
|
||||
# This default tree is a MODERN box (25.10/26): it has pool.snapshot and no
|
||||
# zfs.snapshot. The older shape is built explicitly where it is tested.
|
||||
# Not a plugin: a method on the middleware OBJECT. `snapshot_service()` resolves
|
||||
# the snapshot namespace through it, so if it vanishes the module cannot sweep the
|
||||
# snapshot it just took.
|
||||
"utils/plugins.py": (
|
||||
"class LoadPluginsMixin:\n"
|
||||
" def get_service(self, name):\n pass\n"
|
||||
),
|
||||
"plugins/pool_/dataset.py": (
|
||||
"class PoolDatasetService(CRUDService):\n"
|
||||
" class Config:\n"
|
||||
" namespace = 'zfs.dataset'\n"
|
||||
" namespace = 'pool.dataset'\n"
|
||||
" def query(self, filters, options):\n pass\n"
|
||||
),
|
||||
"plugins/zfs_/snapshot.py": (
|
||||
"class ZFSSnapshot(CRUDService):\n"
|
||||
"plugins/pool_/snapshot.py": (
|
||||
"class PoolSnapshotService(CRUDService):\n"
|
||||
" class Config:\n"
|
||||
" namespace = 'zfs.snapshot'\n"
|
||||
" namespace = 'pool.snapshot'\n"
|
||||
" def query(self, filters, options):\n pass\n"
|
||||
" def delete(self, id_, options={}):\n pass\n"
|
||||
),
|
||||
}
|
||||
|
||||
#: A 24.10/25.04 box: `pool.snapshot` does not exist yet and the snapshot CRUD
|
||||
#: service still answers to the (then-public) `zfs.snapshot`.
|
||||
ZFS_ERA_SNAPSHOT = (
|
||||
"class ZFSSnapshot(CRUDService):\n"
|
||||
" class Config:\n"
|
||||
" namespace = 'zfs.snapshot'\n"
|
||||
" def query(self, filters, options):\n pass\n"
|
||||
" def do_delete(self, id_, options={}):\n pass\n"
|
||||
)
|
||||
|
||||
|
||||
def loader(files):
|
||||
def load(path):
|
||||
@@ -284,9 +306,23 @@ class TestAsyncFlavour:
|
||||
broken["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429")
|
||||
assert compat.async_flavour(loader(broken)) is None
|
||||
|
||||
def test_the_real_truenas_versions(self):
|
||||
# Pinning the actual fact this whole port exists for.
|
||||
assert compat.async_flavour(loader(GOOD)) is True
|
||||
def test_it_reads_STOCK_source_not_our_own_injected_block(self):
|
||||
# This was a byte-identical copy of test_async_middleware_is_detected under a
|
||||
# name that promised more. The fact worth pinning: apply.sh re-runs on an
|
||||
# ALREADY-PATCHED overlay, so the probe must cut our block off first -- our own
|
||||
# SNAPSHOT_SYNC wrapper is a plain `def create_snapshot`, and reading it would
|
||||
# report a 25.10 box as synchronous and inject the wrong flavour.
|
||||
patched = dict(GOOD)
|
||||
patched["plugins/cloud/snapshot.py"] = (
|
||||
GOOD["plugins/cloud/snapshot.py"]
|
||||
+ "\n# TRUECLOUD_PATCH\n"
|
||||
+ 'def create_snapshot(middleware, path, name="x"):\n return "s", "p"\n'
|
||||
)
|
||||
assert compat.async_flavour(loader(patched)) is True, (
|
||||
"the flavour probe read our own injected block and concluded the box is "
|
||||
"synchronous -- it would then inject a sync wrapper into an async "
|
||||
"middleware, and every nested backup would break"
|
||||
)
|
||||
|
||||
|
||||
class TestMiddlewareMethodsWeCall:
|
||||
@@ -303,52 +339,86 @@ class TestMiddlewareMethodsWeCall:
|
||||
per descendant dataset (250 on a real pool) on every run, forever.
|
||||
"""
|
||||
|
||||
ZFS_SNAPSHOT = (
|
||||
"class ZFSSnapshot(CRUDService):\n"
|
||||
" class Config:\n"
|
||||
" namespace = 'zfs.snapshot'\n"
|
||||
" def query(self, filters, options):\n pass\n"
|
||||
" def delete(self, id_, options={}):\n pass\n"
|
||||
)
|
||||
ZFS_DATASET = (
|
||||
"class ZFSDataset(CRUDService):\n"
|
||||
" class Config:\n"
|
||||
" namespace = 'zfs.dataset'\n"
|
||||
" def query(self, filters, options):\n pass\n"
|
||||
)
|
||||
POOL_SNAPSHOT = GOOD["plugins/pool_/snapshot.py"]
|
||||
|
||||
def _tree(self, **over):
|
||||
files = dict(GOOD)
|
||||
files["plugins/zfs_/snapshot.py"] = self.ZFS_SNAPSHOT
|
||||
files["plugins/zfs_/dataset.py"] = self.ZFS_DATASET
|
||||
files.update(over)
|
||||
return files
|
||||
|
||||
#: A 26 box: pool.snapshot only.
|
||||
def _modern(self, **over):
|
||||
return self._tree(**over)
|
||||
|
||||
#: A 24.10/25.04 box: zfs.snapshot only -- plugins/pool_/snapshot.py does not
|
||||
#: exist yet.
|
||||
def _zfs_era(self, **over):
|
||||
return self._tree(**{
|
||||
"plugins/pool_/snapshot.py": None,
|
||||
"plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
|
||||
**over,
|
||||
})
|
||||
|
||||
def test_present_methods_are_ok(self):
|
||||
r = check_files(self._tree())
|
||||
r = check_files(self._modern())
|
||||
assert r[NESTED]["ok"], r[NESTED]["problems"]
|
||||
|
||||
def test_a_deleted_plugin_file_is_broken(self):
|
||||
# Literally TrueNAS 26: plugins/zfs_/snapshot.py does not exist.
|
||||
r = check_files(self._tree(**{"plugins/zfs_/snapshot.py": None}))
|
||||
def test_the_OLD_zfs_era_snapshot_service_also_satisfies_the_call(self):
|
||||
# 24.10 and 25.04 have no `pool.snapshot` at all -- the CRUD service is the
|
||||
# then-public `zfs.snapshot`. Pinning only the modern spelling marked both of
|
||||
# those releases BROKEN and would have switched nested snapshots OFF on boxes
|
||||
# where they work perfectly. The runtime picks the same way; see
|
||||
# pick_snapshot_service().
|
||||
r = check_files(self._zfs_era())
|
||||
assert r[NESTED]["ok"], r[NESTED]["problems"]
|
||||
|
||||
def test_it_is_broken_only_when_NEITHER_namespace_exists(self):
|
||||
# The real failure: middleware drops the last spelling we know how to call.
|
||||
r = check_files(self._tree(**{
|
||||
"plugins/pool_/snapshot.py": None,
|
||||
"plugins/zfs_/snapshot.py": None,
|
||||
}))
|
||||
assert is_broken(r[NESTED])
|
||||
details = " ".join(p["detail"] for p in r[NESTED]["problems"])
|
||||
assert "zfs.snapshot.delete" in details
|
||||
assert "pool.snapshot.delete" in details
|
||||
assert "zfs.snapshot.delete" in details, (
|
||||
"the report must say BOTH spellings were tried, or whoever reads it will "
|
||||
"think we simply never looked for the one their box has"
|
||||
)
|
||||
|
||||
def test_we_do_NOT_depend_on_a_middleware_dataset_query_at_all(self):
|
||||
# iX could delete plugins/pool_/dataset.py tomorrow and the patch would not
|
||||
# care, because the staging plan is enumerated from ZFS, not from middleware.
|
||||
#
|
||||
# That is deliberate, and it was expensive to learn. `pool.dataset.query`
|
||||
# exists and is correctly shaped -- and it LIES: it applies a visibility
|
||||
# policy that hides ix-apps/*, .system/* and .ix-virt/* (84 of 270 datasets
|
||||
# on the real pool, including live app data). No source check could ever
|
||||
# have caught that; only running it could. So there is no assumption here
|
||||
# left to break.
|
||||
r = check_files(self._modern(**{"plugins/pool_/dataset.py": None}))
|
||||
assert r[NESTED]["ok"], r[NESTED]["problems"]
|
||||
|
||||
ids = {c.id for c in compat.MIDDLEWARE_CALLS}
|
||||
assert not any("dataset" in i or "query" in i for i in ids), (
|
||||
"a dataset/snapshot QUERY assumption crept back into the manifest -- "
|
||||
"middleware's queries are filtered; enumerate from ZFS"
|
||||
)
|
||||
|
||||
def test_a_renamed_namespace_is_broken(self):
|
||||
r = check_files(self._tree(**{
|
||||
"plugins/zfs_/snapshot.py": self.ZFS_SNAPSHOT.replace(
|
||||
"'zfs.snapshot'", "'zfs.resource.snapshot'"),
|
||||
"plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace(
|
||||
"'pool.snapshot'", "'zfs.resource.snapshot'"),
|
||||
"plugins/zfs_/snapshot.py": None,
|
||||
}))
|
||||
assert is_broken(r[NESTED])
|
||||
|
||||
def test_the_CRUDService_do_prefix_is_accepted(self):
|
||||
# 24.10 and 25.04 declare `do_delete`; 25.10 renamed it to `delete`. BOTH
|
||||
# answer to zfs.snapshot.delete. Accepting only the literal name reported the
|
||||
# two older releases as broken -- a false BROKEN that would have switched off
|
||||
# nested snapshots on boxes where they work perfectly.
|
||||
r = check_files(self._tree(**{
|
||||
"plugins/zfs_/snapshot.py": self.ZFS_SNAPSHOT.replace(
|
||||
# A CRUDService exposes `delete` from a method NAMED `do_delete`. Both
|
||||
# spellings are live across the matrix. Accepting only the literal name
|
||||
# reported working releases as broken.
|
||||
r = check_files(self._modern(**{
|
||||
"plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace(
|
||||
"def delete(", "def do_delete("),
|
||||
}))
|
||||
assert r[NESTED]["ok"], r[NESTED]["problems"]
|
||||
@@ -356,6 +426,383 @@ class TestMiddlewareMethodsWeCall:
|
||||
def test_the_snapshot_delete_reason_names_the_orphan_risk(self):
|
||||
# If this ever regresses, whoever reads the bug report must understand that
|
||||
# it is not a cosmetic failure.
|
||||
r = check_files(self._tree(**{"plugins/zfs_/snapshot.py": None}))
|
||||
r = check_files(self._tree(**{
|
||||
"plugins/pool_/snapshot.py": None,
|
||||
"plugins/zfs_/snapshot.py": None,
|
||||
}))
|
||||
whys = " ".join(p["why"] for p in r[NESTED]["problems"])
|
||||
assert "orphan" in whys
|
||||
|
||||
|
||||
class TestTheMethodCheckIsNotJustANamespaceCheck:
|
||||
"""compat must verify the METHOD, not merely that the namespace still exists.
|
||||
|
||||
Deleting the method check entirely used to leave all 304 tests green -- so the
|
||||
"namespace AND method" claim was unenforced and silently revertible. It is the
|
||||
half of the predicate that catches iX gutting a method while keeping its service,
|
||||
which they have already done to `pool.snapshot.do_update` on master.
|
||||
"""
|
||||
|
||||
def test_a_namespace_that_no_longer_defines_delete_is_broken(self):
|
||||
gutted = (
|
||||
"class PoolSnapshotService(CRUDService):\n"
|
||||
" class Config:\n"
|
||||
" namespace = 'pool.snapshot'\n"
|
||||
" def query(self, filters, options):\n pass\n"
|
||||
# do_delete is GONE -- the service is still registered and still a
|
||||
# CRUDService, so it still INHERITS a callable `delete`.
|
||||
)
|
||||
r = check_files(with_(**{
|
||||
"plugins/pool_/snapshot.py": gutted,
|
||||
"plugins/zfs_/snapshot.py": None, # no fallback either
|
||||
}))
|
||||
assert is_broken(r[NESTED]), (
|
||||
"a namespace with no delete must be BROKEN. Checking only that the "
|
||||
"namespace exists would apply the patch to a box that cannot sweep its "
|
||||
"own snapshots."
|
||||
)
|
||||
|
||||
def test_the_alternative_still_saves_it_when_only_the_primary_is_gutted(self):
|
||||
gutted = (
|
||||
"class PoolSnapshotService(CRUDService):\n"
|
||||
" class Config:\n"
|
||||
" namespace = 'pool.snapshot'\n"
|
||||
" def query(self, filters, options):\n pass\n"
|
||||
)
|
||||
r = check_files(with_(**{
|
||||
"plugins/pool_/snapshot.py": gutted,
|
||||
"plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
|
||||
}))
|
||||
assert r[NESTED]["ok"], r[NESTED]["problems"]
|
||||
|
||||
|
||||
class TestUnreadableIsNeverOkAndNeverBroken:
|
||||
"""A rate limit is not a regression, and it is not a clean bill of health either.
|
||||
|
||||
compat runs ~30 unauthenticated GitHub requests per matrix; 429 is a real outcome.
|
||||
It also runs at BOOT against the installed tree, where a read can fail with EACCES.
|
||||
|
||||
* treating unreadable as BROKEN repaints the README, files a bug report, and
|
||||
makes apply.sh refuse the module on a box where it works.
|
||||
* treating it as OK injects a module whose delete may be gone.
|
||||
|
||||
Both mutations used to pass the whole suite.
|
||||
"""
|
||||
|
||||
def test_both_spellings_unreadable_is_unknown_not_broken(self):
|
||||
r = check_files(with_(**{
|
||||
"plugins/pool_/snapshot.py": Unreadable("HTTP 429"),
|
||||
"plugins/zfs_/snapshot.py": Unreadable("HTTP 429"),
|
||||
}))
|
||||
assert not is_broken(r[NESTED]), "a 429 is not iX deleting the snapshot service"
|
||||
assert r[NESTED]["unknown"]
|
||||
|
||||
def test_an_unreadable_primary_with_a_healthy_alternative_is_ok(self):
|
||||
r = check_files(with_(**{
|
||||
"plugins/pool_/snapshot.py": Unreadable("HTTP 429"),
|
||||
"plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT,
|
||||
}))
|
||||
assert r[NESTED]["ok"], r[NESTED]["problems"]
|
||||
assert not r[NESTED]["unknown"], (
|
||||
"one spelling answered the question; the other's 429 is irrelevant"
|
||||
)
|
||||
|
||||
def test_a_missing_primary_with_an_unreadable_alternative_is_unknown(self):
|
||||
# We cannot tell whether the box is broken. Saying either would be a guess.
|
||||
r = check_files(with_(**{
|
||||
"plugins/pool_/snapshot.py": None,
|
||||
"plugins/zfs_/snapshot.py": Unreadable("HTTP 429"),
|
||||
}))
|
||||
assert not is_broken(r[NESTED])
|
||||
assert r[NESTED]["unknown"]
|
||||
|
||||
|
||||
class TestGetServiceIsChecked:
|
||||
"""The runtime resolves the snapshot namespace through `middleware.get_service`.
|
||||
|
||||
It is not a plugin method, so the manifest had no way to express it and never
|
||||
checked it. If it vanishes, `_can_delete` reports BOTH namespaces unusable and
|
||||
every nested backup fails -- on a box the preflight had declared healthy.
|
||||
"""
|
||||
|
||||
def test_a_middleware_without_get_service_is_broken(self):
|
||||
r = check_files(with_(**{"utils/plugins.py": None}))
|
||||
assert is_broken(r[NESTED])
|
||||
details = " ".join(p["detail"] for p in r[NESTED]["problems"])
|
||||
assert "get_service" in details
|
||||
|
||||
|
||||
class TestATransientNetworkBlipDoesNotWakeAnybody:
|
||||
"""The fingerprint must digest what iX BROKE, not what GitHub failed to serve.
|
||||
|
||||
`unknown` problems (a 429 on one of ~30 unauthenticated fetches, an EACCES at boot)
|
||||
used to be folded into an already-broken module's problem list, so one blip flipped
|
||||
the fingerprint, `compat_publish` rewrote the issue body, and the next clean run
|
||||
rewrote it back. Daily churn is what teaches people to ignore the bot -- which is
|
||||
the whole thing this fingerprint exists to prevent.
|
||||
"""
|
||||
|
||||
def _rows(self, files):
|
||||
return [{"ref": "master", "modules": check_files(files)}]
|
||||
|
||||
def test_an_unreadable_file_does_not_change_the_fingerprint_of_a_broken_ref(self):
|
||||
# The blip must land in the SAME module that is broken. Put it in `providers`
|
||||
# (which is healthy) and `fingerprint()` skips the whole module via
|
||||
# `is_broken(m)` -- so the `state` filter under test never runs and the test
|
||||
# passes no matter what the code does. `nested` is the broken one here, so the
|
||||
# unreadable file goes in `nested` too.
|
||||
broken = with_(**{
|
||||
"plugins/cloud/snapshot.py":
|
||||
"async def create_snapshot(name, path, middleware):\n return 1, 2\n",
|
||||
})
|
||||
clean = compat.fingerprint(self._rows(broken))
|
||||
|
||||
blipped = dict(broken)
|
||||
blipped["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429") # nested
|
||||
assert compat.fingerprint(self._rows(blipped)) == clean, (
|
||||
"a rate-limited fetch changed the fingerprint, so the bot rewrites the "
|
||||
"issue body and then rewrites it back tomorrow"
|
||||
)
|
||||
|
||||
def test_a_REAL_new_finding_still_changes_it(self):
|
||||
# ...and the anti-noise measure must not have made it deaf.
|
||||
broken = with_(**{
|
||||
"plugins/cloud/snapshot.py":
|
||||
"async def create_snapshot(name, path, middleware):\n return 1, 2\n",
|
||||
})
|
||||
worse = dict(broken)
|
||||
worse["plugins/cloud_backup/restic.py"] = (
|
||||
"class ResticConfig:\n cmd: list\n\n"
|
||||
"def get_restic_config(entry, credentials):\n pass\n"
|
||||
)
|
||||
assert compat.fingerprint(self._rows(worse)) != compat.fingerprint(self._rows(broken))
|
||||
|
||||
|
||||
class TestTheBotFindsItsOwnIssueOnBOTHForges:
|
||||
"""`find_issue` decides "have I already filed this?" -- and it ran on two forges.
|
||||
|
||||
It used to skip pull requests with `"pull_request" not in i`. GitHub omits that key
|
||||
on a plain issue; **Gitea sends it as `null`**. So on Gitea every issue looked like
|
||||
a PR, the match list was always empty, and the bot took the "nothing filed yet"
|
||||
branch on EVERY run: nine duplicate copies of the same report on the canonical
|
||||
forge, four of them filed after the commit that was supposed to stop exactly this.
|
||||
|
||||
It is the same failure the spam fix was written to prevent, moved from comments to
|
||||
issues -- and it survived because `find_issue` was the one function here with no
|
||||
test. So the payload shapes are pinned, per forge, by hand.
|
||||
"""
|
||||
|
||||
TITLE = compat_publish.TITLE
|
||||
|
||||
def _find(self, monkeypatch, payload):
|
||||
monkeypatch.setattr(compat_publish, "_call", lambda *a, **k: payload)
|
||||
return compat_publish.find_issue("https://forge/api", "tok", self.TITLE)
|
||||
|
||||
def test_gitea_sends_pull_request_as_null_and_the_issue_is_still_found(self, monkeypatch):
|
||||
found = self._find(monkeypatch, [
|
||||
{"number": 7, "title": self.TITLE, "state": "open", "pull_request": None},
|
||||
{"number": 1, "title": self.TITLE, "state": "open", "pull_request": None},
|
||||
])
|
||||
assert found is not None, (
|
||||
"find_issue missed a Gitea issue, so the bot files a NEW duplicate report "
|
||||
"every run -- which is how nine of them piled up"
|
||||
)
|
||||
assert found["number"] == 1, "lowest-numbered wins"
|
||||
|
||||
def test_github_omits_the_key_entirely_and_the_issue_is_still_found(self, monkeypatch):
|
||||
found = self._find(monkeypatch, [
|
||||
{"number": 2, "title": self.TITLE, "state": "open"},
|
||||
])
|
||||
assert found is not None and found["number"] == 2
|
||||
|
||||
def test_a_real_PR_with_the_same_title_is_still_skipped_on_both(self, monkeypatch):
|
||||
# The reason the filter exists at all: both forges list PRs on /issues, and
|
||||
# commenting on a PR instead of the bug report would be worse than useless.
|
||||
assert self._find(monkeypatch, [
|
||||
{"number": 3, "title": self.TITLE, "state": "open", # Gitea PR
|
||||
"pull_request": {"merged": False}},
|
||||
{"number": 4, "title": self.TITLE, "state": "open", # GitHub PR
|
||||
"pull_request": {"url": "https://api.github.com/..."}},
|
||||
]) is None
|
||||
|
||||
def test_an_unrelated_issue_is_not_mistaken_for_the_report(self, monkeypatch):
|
||||
assert self._find(monkeypatch, [
|
||||
{"number": 1, "title": "TypeError when create B2 backup on Electric Eel",
|
||||
"state": "closed", "pull_request": None},
|
||||
]) is None
|
||||
|
||||
|
||||
class TestTheNextMaintenanceReleaseIsChecked:
|
||||
"""`release/25.10.5` fell through every sieve, and it is the one that reaches users.
|
||||
|
||||
Shipped versions come from `TS-*` TAGS; unreleased ones come from `release/*`
|
||||
BRANCHES that carry `-BETA`/`-RC`. A branched-but-untagged MAINTENANCE release is
|
||||
neither: no tag, and its line (25.10) has already shipped, so the "prereleases of
|
||||
a shipped line are history" filter threw it out. It was invisible.
|
||||
|
||||
That is backwards. `release/24.10-RC.2` is history -- nobody can install it. But
|
||||
`release/25.10.5` is the FUTURE of a shipped line: it is what a 25.10.4 box gets
|
||||
on its next update. A break there ships to real users before the daily check has
|
||||
ever looked at it.
|
||||
"""
|
||||
|
||||
TAGS = ["TS-24.10.2.4", "TS-25.04.2.6", "TS-25.10.4"]
|
||||
HEADS = [
|
||||
"release/25.10.4.1",
|
||||
"release/25.10.5", # branched, untagged -- the next maintenance release
|
||||
"release/24.10-RC.2", # history: its line shipped long ago
|
||||
"release/25.20.2.2", # iX's typo branch: 25.20 is not a TrueNAS version
|
||||
"release/26.0.0-BETA.3",
|
||||
"master",
|
||||
]
|
||||
|
||||
def _refs(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
compat, "_ls_remote",
|
||||
lambda remote, what: self.TAGS if what == "--tags" else self.HEADS)
|
||||
return compat.discover_refs("origin")
|
||||
|
||||
def test_the_next_maintenance_release_is_checked(self, monkeypatch):
|
||||
assert "release/25.10.5" in self._refs(monkeypatch), (
|
||||
"the next thing a 25.10.4 box updates to is not checked, so a break in it "
|
||||
"reaches users before the bot ever sees it"
|
||||
)
|
||||
|
||||
def test_a_superseded_maintenance_branch_is_not(self, monkeypatch):
|
||||
# 25.10.4.1 sorts OLDER than the newest tag TS-25.10.4? No -- it is NEWER, and
|
||||
# both are on the 25.10 line, so only the newest branch on the line is taken.
|
||||
refs = self._refs(monkeypatch)
|
||||
assert "release/25.10.4.1" not in refs, "only the newest branch per line"
|
||||
|
||||
def test_the_typo_branch_stays_out(self, monkeypatch):
|
||||
# 25.20 has no TS tag, so it is not a release line at all. A typo branch in the
|
||||
# matrix reads as a real supported release we are silently broken on.
|
||||
assert "release/25.20.2.2" not in self._refs(monkeypatch)
|
||||
|
||||
def test_a_prerelease_of_an_already_shipped_line_stays_out(self, monkeypatch):
|
||||
assert "release/24.10-RC.2" not in self._refs(monkeypatch)
|
||||
|
||||
def test_an_untagged_branch_counts_as_UNRELEASED(self, monkeypatch):
|
||||
# The exit code keys off this. Calling 25.10.5 "shipped" would fail the build
|
||||
# as a live outage on a version nobody is running yet.
|
||||
assert compat.is_unreleased("release/25.10.5")
|
||||
assert compat.is_unreleased("master")
|
||||
assert not compat.is_unreleased("TS-25.10.4")
|
||||
|
||||
|
||||
class TestMasterIsNotTheNextRelease:
|
||||
"""A red `master` row used to read as "the version you are about to install".
|
||||
|
||||
On 2026-07-14 master was 27-dev -- every recent commit targeted 27.0.0-BETA.1 --
|
||||
while 26 was still in beta on its own branches. So `master BROKEN` meant "iX will
|
||||
break us a major release from now", but the matrix said "master _(unreleased)_",
|
||||
which any reader takes as the next thing out the door. For a table whose whole job
|
||||
is helping somebody decide whether to trust this with their backups, that is a
|
||||
false alarm in the worst possible place.
|
||||
"""
|
||||
|
||||
def _rows(self, refs):
|
||||
return [{"ref": r, "unreleased": compat.is_unreleased(r), "modules": {}}
|
||||
for r in refs]
|
||||
|
||||
def test_master_is_labelled_with_the_major_AFTER_the_newest_known_one(self):
|
||||
rows = self._rows(["TS-25.10.4", "release/26.0.0-BETA.3", "master"])
|
||||
assert compat.dev_label(rows) == "27-dev"
|
||||
|
||||
def test_it_rolls_over_on_its_own_when_the_next_beta_branches(self):
|
||||
# Derived, not hardcoded: when release/27.0.0-BETA.1 appears, master is 28-dev.
|
||||
rows = self._rows(["TS-26.0.0", "release/27.0.0-BETA.1", "master"])
|
||||
assert compat.dev_label(rows) == "28-dev"
|
||||
|
||||
def test_the_rendered_matrix_says_dev_not_unreleased(self):
|
||||
healthy = check_files(with_())
|
||||
rows = [
|
||||
{"ref": r, "unreleased": compat.is_unreleased(r), "modules": healthy}
|
||||
for r in ("TS-25.10.4", "release/26.0.0-BETA.3", "master")
|
||||
]
|
||||
md = compat.render_markdown(rows)
|
||||
assert "master _(27-dev)_" in md
|
||||
assert "master _(unreleased)_" not in md
|
||||
# ...and the ordinary rows are untouched.
|
||||
assert "| 25.10.4 |" in md
|
||||
assert "| 26.0.0-BETA.3 _(unreleased)_ |" in md
|
||||
|
||||
|
||||
class TestTheBodyIsTruthAndCommentsAreTheChangelog:
|
||||
"""An unchanged fingerprint used to freeze the BODY, not just silence the comments.
|
||||
|
||||
Two different questions were sharing one answer. "Have the findings changed?" gates
|
||||
COMMENTS -- they notify, and a daily "still broken, same as yesterday" is what
|
||||
teaches people to ignore the one that finally matters. But "is the body still
|
||||
true?" gates the BODY, and editing a body notifies nobody, so keeping it honest
|
||||
costs nothing.
|
||||
|
||||
Conflated, an unchanged fingerprint meant the report could never be corrected --
|
||||
and the fingerprint deliberately ignores everything that moves on its own, which
|
||||
includes how a row is LABELLED. Relabelling master `27-dev` would have reached the
|
||||
README and never the issue anybody opens.
|
||||
"""
|
||||
|
||||
ROWS = [{"ref": "master", "unreleased": True,
|
||||
"modules": check_files(with_(**{
|
||||
"plugins/cloud_backup/restic.py":
|
||||
"class ResticConfig:\n cmd: list\n\n"
|
||||
"def get_restic_config(entry, credentials):\n pass\n",
|
||||
}))}]
|
||||
|
||||
def _run(self, monkeypatch, tmp_path, existing_body):
|
||||
calls = []
|
||||
|
||||
def fake(url, token, method="GET", data=None):
|
||||
calls.append((method, url, data))
|
||||
if url.endswith("/issues?state=all&per_page=100&limit=100"):
|
||||
return [{"number": 1, "title": compat_publish.TITLE,
|
||||
"state": "open", "body": existing_body,
|
||||
"pull_request": None}]
|
||||
return {"number": 1}
|
||||
|
||||
monkeypatch.setattr(compat_publish, "_call", fake)
|
||||
matrix = tmp_path / "m.json"
|
||||
matrix.write_text(json.dumps(self.ROWS))
|
||||
compat_publish.main([
|
||||
"prog", "--api", "https://forge/api", "--token", "t",
|
||||
"--matrix", str(matrix)])
|
||||
return calls
|
||||
|
||||
def _writes(self, calls):
|
||||
patched = [c for c in calls if c[0] == "PATCH"]
|
||||
commented = [c for c in calls if c[0] == "POST" and c[1].endswith("/comments")]
|
||||
return patched, commented
|
||||
|
||||
def test_identical_body_and_findings_touches_nothing(self, monkeypatch, tmp_path):
|
||||
body = compat.render_issue(self.ROWS)
|
||||
patched, commented = self._writes(self._run(monkeypatch, tmp_path, body))
|
||||
assert not patched and not commented, "a quiet run must be completely silent"
|
||||
|
||||
def test_a_relabel_refreshes_the_body_but_says_NOTHING(self, monkeypatch, tmp_path):
|
||||
# Same findings (same fingerprint), different rendering -- the exact shape of
|
||||
# the master -> 27-dev relabel.
|
||||
stale = compat.render_issue(self.ROWS).replace("27-dev", "unreleased")
|
||||
assert compat.extract_fingerprint(stale) == compat.fingerprint(self.ROWS)
|
||||
|
||||
patched, commented = self._writes(self._run(monkeypatch, tmp_path, stale))
|
||||
assert patched, "the body was left stale, so the issue keeps telling lies"
|
||||
assert "27-dev" in patched[0][2]["body"]
|
||||
assert not commented, (
|
||||
"a rendering change is not news -- commenting on it is how the bot gets "
|
||||
"muted before the next real finding"
|
||||
)
|
||||
|
||||
def test_a_REAL_findings_change_still_comments(self, monkeypatch, tmp_path):
|
||||
# ...and the fix must not have made it mute.
|
||||
stale = compat.render_issue(self.ROWS).replace(
|
||||
compat.fingerprint(self.ROWS), "0" * 16)
|
||||
patched, commented = self._writes(self._run(monkeypatch, tmp_path, stale))
|
||||
assert patched and commented, "a genuine change must still notify"
|
||||
|
||||
def test_a_body_differing_only_by_CRLF_is_not_rewritten(self, monkeypatch, tmp_path):
|
||||
# Forges round-trip line endings. Without normalising, every run would rewrite
|
||||
# the body -- silent, but it churns updated_at and looks freshly touched daily.
|
||||
body = compat.render_issue(self.ROWS).replace("\n", "\r\n")
|
||||
patched, _ = self._writes(self._run(monkeypatch, tmp_path, body))
|
||||
assert not patched
|
||||
|
||||
@@ -94,3 +94,40 @@ class TestTheReadmeStaysAReadme:
|
||||
assert "24.10" in text[:text.index("## Install")], (
|
||||
"the minimum TrueNAS version must be visible above the install steps"
|
||||
)
|
||||
|
||||
|
||||
class TestInstallDoesNotDirtyTheCheckout:
|
||||
"""install.sh chmod +x's scripts. If git records them as 100644, that chmod is a
|
||||
TRACKED MODIFICATION -- and update.sh refuses to run over a dirty tree.
|
||||
|
||||
So installing once permanently blocked updating, for every user, with a message
|
||||
telling them to `git checkout -- .` (which would just undo the exec bit and let
|
||||
the next install re-dirty it). Found on a real box that had been stuck on an old
|
||||
version for exactly this reason.
|
||||
|
||||
Every script install.sh makes executable must already be executable in git.
|
||||
"""
|
||||
|
||||
def test_every_chmodded_script_is_already_executable_in_git(self):
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
with open(os.path.join(ROOT, "install.sh"), encoding="utf-8") as fh:
|
||||
m = re.search(r"^for _exe in (.+?); do", fh.read(), re.M)
|
||||
assert m, "could not find install.sh's chmod loop"
|
||||
scripts = m.group(1).split()
|
||||
|
||||
out = subprocess.run(
|
||||
["git", "ls-files", "-s", *scripts],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True,
|
||||
).stdout
|
||||
|
||||
not_exec = [
|
||||
line.split("\t")[-1] for line in out.strip().splitlines()
|
||||
if not line.startswith("100755")
|
||||
]
|
||||
assert not not_exec, (
|
||||
"install.sh chmod +x's these, but git records them as non-executable — "
|
||||
"so installing dirties the checkout and update.sh then refuses to run:\n "
|
||||
+ "\n ".join(not_exec)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Behavioural tests for the "installed but NOT loaded" alert.
|
||||
|
||||
apply.log can only report what was written to disk. Whether the middlewared that
|
||||
restarted afterwards actually imported those files is a different fact, and when
|
||||
the two disagree nothing else notices: on 2026-08-19 every B2 backup failed for
|
||||
nineteen hours while the log said OK. This alert is the only thing that closes
|
||||
that gap, so it is tested against real objects rather than by reading source.
|
||||
|
||||
The middlewared package does not exist off-box, so the modules the alert source
|
||||
imports are stubbed here.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
ALERT_SRC = os.path.join(os.path.dirname(__file__), "..", "patch", "alert_source.py")
|
||||
|
||||
|
||||
class _StubAlertClass:
|
||||
pass
|
||||
|
||||
|
||||
class _StubThreadedAlertSource:
|
||||
pass
|
||||
|
||||
|
||||
class _StubAlert:
|
||||
def __init__(self, klass, args=None, key=None):
|
||||
self.klass = klass
|
||||
self.args = args
|
||||
self.key = key
|
||||
|
||||
|
||||
def _module(name):
|
||||
mod = types.ModuleType(name)
|
||||
sys.modules[name] = mod
|
||||
return mod
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alert_source(monkeypatch, tmp_path):
|
||||
"""Load patch/alert_source.py against stubbed middlewared modules."""
|
||||
for name in list(sys.modules):
|
||||
if name == "middlewared" or name.startswith("middlewared."):
|
||||
monkeypatch.delitem(sys.modules, name, raising=False)
|
||||
|
||||
_module("middlewared")
|
||||
_module("middlewared.alert")
|
||||
base = _module("middlewared.alert.base")
|
||||
base.Alert = _StubAlert
|
||||
base.AlertClass = _StubAlertClass
|
||||
base.ThreadedAlertSource = _StubThreadedAlertSource
|
||||
base.AlertCategory = types.SimpleNamespace(SYSTEM="SYSTEM")
|
||||
base.AlertLevel = types.SimpleNamespace(
|
||||
INFO="INFO", WARNING="WARNING", CRITICAL="CRITICAL"
|
||||
)
|
||||
schedule = _module("middlewared.alert.schedule")
|
||||
schedule.IntervalSchedule = lambda delta: ("interval", delta)
|
||||
|
||||
spec = importlib.util.spec_from_file_location("_tc_alert_source", ALERT_SRC)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
mod.PATCH_DIR = str(tmp_path)
|
||||
return mod
|
||||
|
||||
|
||||
def _write_status(tmp_path, providers_active=True):
|
||||
payload = {
|
||||
"patched_at": "2026-08-26T00:00:00Z",
|
||||
"patches": {
|
||||
"providers": {"ok": True, "active": providers_active, "detail": "x"},
|
||||
"nested_snapshots": {"ok": True, "active": True, "detail": "x"},
|
||||
},
|
||||
}
|
||||
(tmp_path / "hook_status.json").write_text(json.dumps(payload))
|
||||
|
||||
|
||||
def _install_provider_modules(monkeypatch, *, restic_patched, b2_patched):
|
||||
"""Stub the two modules the alert inspects, in the requested state."""
|
||||
plugins = _module("middlewared.plugins")
|
||||
_module("middlewared.plugins.cloud_backup")
|
||||
restic = _module("middlewared.plugins.cloud_backup.restic")
|
||||
|
||||
def get_restic_config(task):
|
||||
return None
|
||||
|
||||
if restic_patched:
|
||||
get_restic_config._truecloud_patched = True
|
||||
restic.get_restic_config = get_restic_config
|
||||
|
||||
rclone_base = _module("middlewared.rclone.base")
|
||||
_module("middlewared.rclone")
|
||||
_module("middlewared.rclone.remote")
|
||||
b2_mod = _module("middlewared.rclone.remote.b2")
|
||||
|
||||
class BaseRcloneRemote:
|
||||
def get_restic_config(self, task):
|
||||
raise NotImplementedError
|
||||
|
||||
class B2RcloneRemote(BaseRcloneRemote):
|
||||
pass
|
||||
|
||||
if b2_patched:
|
||||
B2RcloneRemote.get_restic_config = staticmethod(lambda task: ("url", {}))
|
||||
|
||||
rclone_base.BaseRcloneRemote = BaseRcloneRemote
|
||||
b2_mod.B2RcloneRemote = B2RcloneRemote
|
||||
b2_mod.BaseRcloneRemote = BaseRcloneRemote
|
||||
plugins.__path__ = []
|
||||
|
||||
for name in (
|
||||
"middlewared.plugins",
|
||||
"middlewared.plugins.cloud_backup",
|
||||
"middlewared.plugins.cloud_backup.restic",
|
||||
"middlewared.rclone",
|
||||
"middlewared.rclone.base",
|
||||
"middlewared.rclone.remote",
|
||||
"middlewared.rclone.remote.b2",
|
||||
):
|
||||
monkeypatch.setitem(sys.modules, name, sys.modules[name])
|
||||
|
||||
|
||||
def _source(alert_source):
|
||||
cls = alert_source.TrueCloudPatchNotLoadedAlertSource
|
||||
return cls.__new__(cls)
|
||||
|
||||
|
||||
def test_no_alert_when_patch_is_loaded(alert_source, monkeypatch, tmp_path):
|
||||
_write_status(tmp_path)
|
||||
_install_provider_modules(monkeypatch, restic_patched=True, b2_patched=True)
|
||||
assert _source(alert_source)._check() is None
|
||||
|
||||
|
||||
def test_alert_when_middlewared_loaded_stock_modules(alert_source, monkeypatch, tmp_path):
|
||||
"""The exact 2026-08-19 state: patched on disk, stock in the process."""
|
||||
_write_status(tmp_path)
|
||||
_install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
|
||||
alert = _source(alert_source)._check()
|
||||
assert alert is not None
|
||||
assert alert.klass is alert_source.TrueCloudPatchNotLoadedAlertClass
|
||||
|
||||
|
||||
def test_alert_when_only_b2_half_is_missing(alert_source, monkeypatch, tmp_path):
|
||||
# b2.py is the half that supplies B2's get_restic_config. restic.py alone
|
||||
# being patched still means every B2 task raises NotImplementedError.
|
||||
_write_status(tmp_path)
|
||||
_install_provider_modules(monkeypatch, restic_patched=True, b2_patched=False)
|
||||
assert _source(alert_source)._check() is not None
|
||||
|
||||
|
||||
def test_alert_when_only_restic_half_is_missing(alert_source, monkeypatch, tmp_path):
|
||||
_write_status(tmp_path)
|
||||
_install_provider_modules(monkeypatch, restic_patched=False, b2_patched=True)
|
||||
assert _source(alert_source)._check() is not None
|
||||
|
||||
|
||||
def test_silent_when_the_kill_switch_is_set(alert_source, monkeypatch, tmp_path):
|
||||
# The operator turned the patch off on purpose; stock is the intended state.
|
||||
_write_status(tmp_path)
|
||||
(tmp_path / "disabled").write_text("")
|
||||
_install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
|
||||
assert _source(alert_source)._check() is None
|
||||
|
||||
|
||||
def test_silent_when_providers_module_is_retired(alert_source, monkeypatch, tmp_path):
|
||||
# TrueNAS went native for B2: not loading our providers patch is correct.
|
||||
_write_status(tmp_path, providers_active=False)
|
||||
_install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
|
||||
assert _source(alert_source)._check() is None
|
||||
|
||||
|
||||
def test_silent_when_the_patch_was_never_applied_here(alert_source, monkeypatch, tmp_path):
|
||||
# No hook_status.json at all -- nothing claims a patch, so nothing is broken.
|
||||
_install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
|
||||
assert _source(alert_source)._check() is None
|
||||
|
||||
|
||||
def test_update_alert_silencer_does_not_mute_a_broken_backup_path(
|
||||
alert_source, monkeypatch, tmp_path
|
||||
):
|
||||
# update_alerts_disabled mutes release notifications. It must not hide the
|
||||
# fact that TrueCloud backups are silently running stock.
|
||||
_write_status(tmp_path)
|
||||
(tmp_path / "update_alerts_disabled").write_text("")
|
||||
_install_provider_modules(monkeypatch, restic_patched=False, b2_patched=False)
|
||||
assert _source(alert_source)._check() is not None
|
||||
|
||||
|
||||
def test_check_sync_never_raises(alert_source, monkeypatch, tmp_path):
|
||||
"""An alert source that raises is polled forever inside middlewared."""
|
||||
_write_status(tmp_path)
|
||||
|
||||
def boom(self):
|
||||
raise RuntimeError("provider import exploded")
|
||||
|
||||
monkeypatch.setattr(
|
||||
alert_source.TrueCloudPatchNotLoadedAlertSource, "_check", boom, raising=True
|
||||
)
|
||||
assert _source(alert_source).check_sync() is None
|
||||
|
||||
|
||||
def test_alert_is_critical_and_names_the_recovery_command(alert_source):
|
||||
klass = alert_source.TrueCloudPatchNotLoadedAlertClass
|
||||
assert klass.level == "CRITICAL"
|
||||
assert "install.sh" in klass.text
|
||||
@@ -9,6 +9,7 @@ across install.sh / uninstall.sh / recover.sh / apply.sh and nothing noticed.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
@@ -205,3 +206,88 @@ class TestSignificance:
|
||||
assert version_tuple("0.4.2") > version_tuple("0.4.1")
|
||||
# Pre-release suffixes are dropped, not ranked above the release.
|
||||
assert version_tuple("v0.5.0-rc1") == version_tuple("v0.5.0")
|
||||
|
||||
|
||||
class TestCandidateNotesResolveToTheBaseVersion:
|
||||
"""`notes v0.6.0-rc1` must return v0.6.0's section.
|
||||
|
||||
A candidate ships the same code as the release it is a candidate for, and the
|
||||
CHANGELOG only ever has the one section. Without this, the release workflow cut
|
||||
v0.6.0-rc1, passed every gate, and then died extracting the body -- so the tag
|
||||
existed but nothing was ever published. Caught in an rc, which is the entire
|
||||
point of having them.
|
||||
"""
|
||||
|
||||
CHANGELOG = "# C\n\n## v0.6.0 — 2026-07-13\n\n### Added\n- the thing\n\n## v0.5.1 — 2026-07-13\n\n- older\n"
|
||||
|
||||
def test_an_rc_resolves_to_its_base_version(self):
|
||||
body = extract_notes(self.CHANGELOG, "v0.6.0-rc1")
|
||||
assert "the thing" in body
|
||||
assert "older" not in body
|
||||
|
||||
def test_rc10_too(self):
|
||||
assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0-rc10")
|
||||
|
||||
def test_the_plain_version_still_works(self):
|
||||
assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0")
|
||||
|
||||
def test_a_genuinely_missing_section_still_raises(self):
|
||||
with pytest.raises(KeyError):
|
||||
extract_notes(self.CHANGELOG, "v9.9.9-rc1")
|
||||
|
||||
|
||||
class TestTheChangelogIsStructurallySound:
|
||||
"""The release body IS this file, so a mangled section ships to every user.
|
||||
|
||||
It has been mangled once: an edit matched the literal `## Unreleased` inside a
|
||||
backticked phrase in a prose bullet and spliced a whole new section into the middle
|
||||
of it, splitting the sentence in half.
|
||||
"""
|
||||
|
||||
def changelog(self):
|
||||
with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
def test_no_version_section_is_empty(self):
|
||||
text = self.changelog()
|
||||
for v in changelog_versions(text):
|
||||
assert extract_notes(text, v).strip(), f"v{v} has an empty section"
|
||||
|
||||
def test_versions_are_in_descending_order(self):
|
||||
from release_notes import version_tuple
|
||||
versions = changelog_versions(self.changelog())
|
||||
assert versions == sorted(versions, key=version_tuple, reverse=True), (
|
||||
"CHANGELOG versions are out of order — a section was spliced in wrong"
|
||||
)
|
||||
|
||||
def test_headings_are_at_the_start_of_a_line_and_not_inside_prose(self):
|
||||
# A `### Fixed` that ends up indented under a bullet is a section nobody sees.
|
||||
for i, line in enumerate(self.changelog().splitlines(), 1):
|
||||
if line.lstrip().startswith(("## ", "### ")) and line != line.lstrip():
|
||||
raise AssertionError(
|
||||
f"line {i}: heading is indented, so it is inside a list item "
|
||||
f"rather than being a section: {line!r}"
|
||||
)
|
||||
|
||||
def test_every_bullet_that_opens_a_bold_phrase_closes_it(self):
|
||||
# The splice cut `- **A stable release ... under \`## Unreleased` in half,
|
||||
# leaving an unterminated ** and a dangling sentence.
|
||||
#
|
||||
# A bullet is the `- ` line plus everything up to the next top-level bullet or
|
||||
# heading -- bold phrases routinely wrap across lines, so a per-line check
|
||||
# would flag every long bullet in the file.
|
||||
text = self.changelog()
|
||||
bullets = re.split(r"^(?=- |#{2,3} )", text, flags=re.M)
|
||||
bad = []
|
||||
for b in bullets:
|
||||
if not b.startswith("- "):
|
||||
continue
|
||||
# Code spans are not markup: `*args, **kwargs` is a literal, not a bold
|
||||
# phrase, and counting its ** would flag a perfectly well-formed bullet.
|
||||
prose = re.sub(r"`[^`]*`", "", b)
|
||||
if prose.count("**") % 2:
|
||||
bad.append(b.splitlines()[0][:70])
|
||||
assert not bad, (
|
||||
"unbalanced ** in a bullet — a section was probably spliced into the "
|
||||
"middle of it:\n " + "\n ".join(bad)
|
||||
)
|
||||
|
||||
+1480
-31
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,199 @@
|
||||
"""The deferred restart must re-apply the patch before it restarts middlewared.
|
||||
|
||||
Patching at PREINIT and restarting minutes later is only sound while the patched
|
||||
files are still on the live path when middlewared re-imports them. They may not
|
||||
be: the patch lives in an overlay mounted inside /usr, and anything that
|
||||
remounts that hierarchy detaches it. On 2026-08-19 a systemd-sysext refresh over
|
||||
/usr ran four seconds after apply.sh mounted its overlay; the deferred restart
|
||||
then loaded stock modules and every B2 cloud_backup job failed for nineteen
|
||||
hours while apply.log reported "OK".
|
||||
|
||||
These tests pin the ordering that makes that non-recoverable failure impossible:
|
||||
re-apply, verify, restart, verify again.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
HERE = os.path.dirname(__file__)
|
||||
WAIT_RESTART = os.path.join(HERE, "..", "patch", "wait_restart.sh")
|
||||
APPLY_SH = os.path.join(HERE, "..", "patch", "apply.sh")
|
||||
|
||||
|
||||
def wait_restart_source():
|
||||
with open(WAIT_RESTART, encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
def apply_source():
|
||||
with open(APPLY_SH, encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
def test_wait_restart_is_executable():
|
||||
# apply.sh schedules it as `/bin/bash <script>`, but install.sh ships exec
|
||||
# bits and a mode-only diff once blocked update.sh outright (v0.6.0).
|
||||
assert os.access(WAIT_RESTART, os.X_OK)
|
||||
|
||||
|
||||
def test_wait_restart_is_syntactically_valid():
|
||||
subprocess.run(["bash", "-n", WAIT_RESTART], check=True)
|
||||
|
||||
|
||||
def test_reapply_runs_before_the_restart():
|
||||
src = wait_restart_source()
|
||||
reapply = src.index("TRUECLOUD_REAPPLY=1")
|
||||
restart = src.index("systemctl try-restart middlewared")
|
||||
assert reapply < restart, "the re-apply pass must precede the restart"
|
||||
|
||||
|
||||
def test_restart_is_not_exec_so_verification_can_follow():
|
||||
# Up to v0.7.0 the script ended in `exec systemctl try-restart middlewared`,
|
||||
# which replaces the shell -- nothing could run afterwards. The post-restart
|
||||
# verification only exists if the restart is a plain call.
|
||||
src = wait_restart_source()
|
||||
assert not re.search(r"^\s*exec\s+systemctl", src, re.M)
|
||||
|
||||
|
||||
def test_middlewared_is_restarted_exactly_once():
|
||||
"""No restart loop.
|
||||
|
||||
`try-restart` returns at READY; middlewared then brings docker up, and
|
||||
docker.configure_nvidia merges the nvidia sysext over /usr right about then
|
||||
-- detaching the overlay AFTER the patched modules are already imported. A
|
||||
disk check after the restart therefore false-negatives on a healthy system,
|
||||
and restarting on that signal would restart a correctly-patched middlewared
|
||||
straight back into the same race.
|
||||
"""
|
||||
src = wait_restart_source()
|
||||
assert src.count("systemctl try-restart middlewared") == 1
|
||||
|
||||
|
||||
def test_patch_is_verified_after_the_restart():
|
||||
src = wait_restart_source()
|
||||
restart = src.index("systemctl try-restart middlewared")
|
||||
assert "_patch_visible" in src[restart:], (
|
||||
"the script must check what the restart actually loaded"
|
||||
)
|
||||
|
||||
|
||||
def test_verification_reads_the_marker_apply_sh_writes():
|
||||
# _patch_visible greps restic.py for TRUECLOUD_PATCH; apply.sh must still be
|
||||
# the thing that puts it there, or the check silently always fails.
|
||||
assert "TRUECLOUD_PATCH" in wait_restart_source()
|
||||
assert "TRUECLOUD_PATCH" in apply_source()
|
||||
|
||||
|
||||
def test_verification_uses_the_recorded_middlewared_dir():
|
||||
# wait_restart.sh must not re-derive site-packages; apply.sh records it.
|
||||
assert ".mw_dir" in wait_restart_source()
|
||||
assert ".mw_dir" in apply_source()
|
||||
|
||||
|
||||
def test_apply_sh_records_the_middlewared_dir():
|
||||
src = apply_source()
|
||||
assert re.search(r'>\s*"\$PATCH_DIR/\.mw_dir"', src), (
|
||||
"apply.sh must write the resolved middlewared dir for wait_restart.sh"
|
||||
)
|
||||
|
||||
|
||||
def test_reapply_pass_does_not_schedule_another_restart():
|
||||
# wait_restart.sh owns the restart. If the re-apply pass scheduled its own
|
||||
# transient unit, each boot would spawn restarts recursively.
|
||||
src = apply_source()
|
||||
guard = src.index('if [ "${TRUECLOUD_REAPPLY:-0}" = "1" ]; then')
|
||||
systemd_run = src.index("systemd-run --no-block")
|
||||
assert guard < systemd_run, (
|
||||
"the TRUECLOUD_REAPPLY branch must short-circuit before systemd-run"
|
||||
)
|
||||
|
||||
|
||||
def test_shadowed_overlay_is_remounted_not_accepted():
|
||||
"""A buried overlay must never pass for a healthy one.
|
||||
|
||||
_ensure_writable reaches its mount-table check only when the directory is
|
||||
NOT writable -- and a live overlay of ours is always writable. So a
|
||||
truecloud mount listed at that point is shadowed, and returning 0 there is
|
||||
exactly how a detached overlay used to masquerade as applied.
|
||||
"""
|
||||
src = apply_source()
|
||||
start = src.index("_ensure_writable()")
|
||||
end = src.index("\n}", start)
|
||||
body = src[start:end]
|
||||
|
||||
check = body.index('mount | grep -qF "truecloud-${tag} on ${dir} "')
|
||||
following = body[check:]
|
||||
# The old code did `return 0` immediately inside this branch.
|
||||
branch_end = following.index("fi")
|
||||
assert "return 0" not in following[:branch_end]
|
||||
assert "umount -l" in following[:branch_end]
|
||||
|
||||
|
||||
def test_workdir_is_recreated_before_mounting():
|
||||
# overlayfs refuses a workdir left behind by a detached mount, so a stale
|
||||
# one would turn every re-mount attempt into "overlay mount failed".
|
||||
src = apply_source()
|
||||
start = src.index("_ensure_writable()")
|
||||
end = src.index("\n}", start)
|
||||
body = src[start:end]
|
||||
assert re.search(r'rm -rf "\$work"', body)
|
||||
|
||||
|
||||
def test_upperdir_is_preserved_across_remounts():
|
||||
# The upperdir holds everything patched earlier this boot; reusing it is
|
||||
# what lets a re-mount restore those files instead of re-deriving them.
|
||||
src = apply_source()
|
||||
start = src.index("_ensure_writable()")
|
||||
end = src.index("\n}", start)
|
||||
body = src[start:end]
|
||||
assert 'rm -rf "$upper"' not in body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["0", "1", "2"])
|
||||
def test_patch_visible_returns_three_distinct_states(state):
|
||||
# patched / stock / cannot-tell must stay distinguishable: "cannot tell"
|
||||
# has to re-apply rather than assume the patch is fine.
|
||||
src = wait_restart_source()
|
||||
assert f"return {state}" in src or f") return {state}" in src
|
||||
|
||||
|
||||
def test_mount_retries_on_a_private_workdir():
|
||||
"""A lazily-detached overlay can still pin the shared workdir.
|
||||
|
||||
overlayfs refuses a workdir that is in use, so without a retry the re-mount
|
||||
this whole fix depends on would fail exactly when it is most needed.
|
||||
"""
|
||||
src = apply_source()
|
||||
start = src.index("_ensure_writable()")
|
||||
end = src.index("\n}", start)
|
||||
body = src[start:end]
|
||||
assert body.count("mount -t overlay") == 2, "expected a retry mount"
|
||||
assert 'work="/run/truecloud-${tag}-work.$$"' in body
|
||||
|
||||
|
||||
def test_post_restart_remount_preserves_the_patched_at_stamp():
|
||||
"""create_task.py verify compares middlewared's start time to patched_at.
|
||||
|
||||
The post-restart re-mount restores the same patch the boot pass applied, so
|
||||
letting apply.sh re-stamp would make patched_at newer than the process that
|
||||
correctly imported it -- verify would then report FAIL forever on every boot
|
||||
where docker's sysext merge detaches the overlay.
|
||||
"""
|
||||
src = wait_restart_source()
|
||||
tail = src[src.index("systemctl try-restart middlewared"):]
|
||||
assert "hook_status.json" in tail
|
||||
save = tail.index("/run/truecloud-hook_status.pre")
|
||||
reapply = tail.index("TRUECLOUD_REAPPLY=1")
|
||||
restore = tail.rindex("hook_status.json")
|
||||
assert save < reapply < restore, "snapshot must bracket the re-apply"
|
||||
|
||||
|
||||
def test_status_snapshot_is_not_written_into_the_repo():
|
||||
"""A leftover file in the repo dir leaves the tree dirty, and update.sh
|
||||
refuses to run over a dirty tree -- that once made the patch un-updatable."""
|
||||
src = wait_restart_source()
|
||||
assert "/run/truecloud-hook_status.pre" in src
|
||||
assert '"$PATCH_DIR/hook_status.json.pre' not in src
|
||||
+149
-4
@@ -10,7 +10,8 @@ import re
|
||||
|
||||
import pytest
|
||||
|
||||
WORKFLOWS = os.path.join(os.path.dirname(__file__), "..", ".github", "workflows")
|
||||
ROOT = os.path.join(os.path.dirname(__file__), "..")
|
||||
WORKFLOWS = os.path.join(ROOT, ".github", "workflows")
|
||||
|
||||
|
||||
def workflow_files():
|
||||
@@ -84,11 +85,77 @@ class TestBothForges:
|
||||
assert "if: ${{ contains(github.server_url, 'github.com') }}" in src
|
||||
assert "if: ${{ !contains(github.server_url, 'github.com') }}" in src
|
||||
|
||||
def test_compat_files_an_issue_on_each_forge(self):
|
||||
def test_compat_files_its_report_through_ONE_implementation(self):
|
||||
# It used to be two near-identical shell steps, one per forge. Two copies of
|
||||
# "find the issue, decide whether to comment, post it" is two chances to drift,
|
||||
# and the Gitea one duplicated an issue for real.
|
||||
with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
|
||||
src = fh.read()
|
||||
assert "file a bug report (GitHub)" in src
|
||||
assert "file a bug report (Gitea)" in src
|
||||
assert "tools/compat_publish.py" in src
|
||||
assert "file a bug report (GitHub)" not in src
|
||||
assert "file a bug report (Gitea)" not in src
|
||||
|
||||
|
||||
class TestTheBotDoesNotSpam:
|
||||
"""It left 11 identical 3,000-character comments on one issue in a single day.
|
||||
|
||||
A bot that repeats itself daily gets muted — and then the next REAL finding is
|
||||
scrolled past, which defeats the entire reason for building it.
|
||||
"""
|
||||
|
||||
def publisher(self):
|
||||
with open(os.path.join(ROOT, "tools", "compat_publish.py"), encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
def test_it_compares_a_fingerprint_before_saying_anything(self):
|
||||
src = self.publisher()
|
||||
assert "extract_fingerprint" in src
|
||||
assert "staying quiet" in src
|
||||
|
||||
def test_the_body_is_edited_in_place_not_appended_to(self):
|
||||
src = self.publisher()
|
||||
assert '"PATCH"' in src, "the issue body must be updated, not commented onto"
|
||||
|
||||
def test_it_closes_the_issue_when_everything_is_fixed(self):
|
||||
src = self.publisher()
|
||||
assert '"state": "closed"' in src
|
||||
|
||||
def test_the_matrix_refresh_opens_a_PR_rather_than_pushing_to_main(self):
|
||||
# An unattended push to main from CI is exactly what the release barrier exists
|
||||
# to prevent: a bot that can move main can move it somewhere nobody looked.
|
||||
#
|
||||
# Checked against CODE, not comments — the step's own commentary explains what
|
||||
# it replaced, and that mention must not read as the thing itself.
|
||||
with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
|
||||
src = fh.read()
|
||||
code = "\n".join(
|
||||
ln for ln in src.splitlines() if not ln.lstrip().startswith("#")
|
||||
)
|
||||
assert "/pulls" in code, "the matrix refresh must open a PR"
|
||||
assert "HEAD:main" not in code, "CI still pushes straight to main"
|
||||
|
||||
def test_the_matrix_PR_targets_the_CANONICAL_forge_not_the_mirror(self):
|
||||
# GitHub is a one-way mirror: a PR merged there would be silently clobbered by
|
||||
# the next `fleet-repos mirror` push from Gitea. A bot opening PRs against a
|
||||
# mirror is a bot doing nothing, slowly.
|
||||
with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
|
||||
src = fh.read()
|
||||
i = src.index("refresh the README matrix")
|
||||
step = src[i:i + 400]
|
||||
assert "!contains(github.server_url, 'github.com')" in step, (
|
||||
"the matrix PR must be opened on Gitea (canonical), not GitHub (mirror)"
|
||||
)
|
||||
|
||||
def test_the_workflow_has_the_permissions_its_steps_actually_need(self):
|
||||
# It shipped with `contents: read` while the step pushed a branch and opened a
|
||||
# PR — it would have died with a 403 on the first scheduled run, and I would
|
||||
# have had a bot that silently never worked.
|
||||
with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
|
||||
src = fh.read()
|
||||
perms = src[src.index("permissions:"):src.index("jobs:")]
|
||||
assert "contents: write" in perms, "pushing a branch needs contents: write"
|
||||
assert "pull-requests: write" in perms, "opening a PR needs pull-requests: write"
|
||||
assert "issues: write" in perms
|
||||
|
||||
|
||||
class TestCompatCannotSilentlyPass:
|
||||
@@ -105,3 +172,81 @@ class TestCompatCannotSilentlyPass:
|
||||
with open(os.path.join(WORKFLOWS, "compat.yml"), encoding="utf-8") as fh:
|
||||
src = fh.read()
|
||||
assert "steps.check.outputs.shipped_broken != '0'" in src
|
||||
|
||||
|
||||
class TestActionCacheRace:
|
||||
"""CI must not run concurrent jobs on the self-hosted runner.
|
||||
|
||||
`act` caches each ACTION as one shared clone under /root/.cache/act/<hash>
|
||||
and re-pulls it per job, so jobs starting together fight over that directory
|
||||
and the loser dies with `lstat .../<file>: no such file or directory` before
|
||||
any test runs -- a red `main` with zero suite output and a different victim
|
||||
each push. The runner also force-pulls its base image per job, so job count
|
||||
is also Docker Hub pull count, and four-per-push exhausted the anonymous
|
||||
limit in an afternoon. Both problems have the same cure: one job.
|
||||
"""
|
||||
|
||||
def _ci(self):
|
||||
with open(os.path.join(WORKFLOWS, "ci.yml"), encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
def test_ci_runs_as_exactly_one_job(self):
|
||||
"""The fix is the absence of concurrency, not the absence of one action.
|
||||
|
||||
Dropping astral-sh/setup-uv only shrank the surface -- every job still
|
||||
used actions/checkout. A single job cannot race itself whatever actions
|
||||
it uses, which is why this, and not the action count, is the invariant.
|
||||
"""
|
||||
ci = self._ci()
|
||||
# Scope to the jobs: block -- `on:` has two-space keys of its own
|
||||
# (push/pull_request/workflow_dispatch) that look identical otherwise.
|
||||
body = ci[ci.index("\njobs:"):]
|
||||
jobs = re.findall(r"^ (\w[\w-]*):$", body, re.M)
|
||||
assert len(jobs) == 1, (
|
||||
f"ci.yml defines {len(jobs)} jobs ({jobs}); concurrent jobs on the "
|
||||
"self-hosted runner race on act's shared action cache and multiply "
|
||||
"Docker Hub pulls. Keep CI to one job."
|
||||
)
|
||||
|
||||
def test_no_matrix_reintroduces_parallel_jobs(self):
|
||||
ci = self._ci()
|
||||
assert "strategy:" not in ci and "matrix:" not in ci, (
|
||||
"a matrix fans out into concurrent jobs again -- sweep versions "
|
||||
"inside one job instead"
|
||||
)
|
||||
|
||||
def test_every_python_version_still_runs_after_one_fails(self):
|
||||
"""`fail-fast: false` is what the loop has to preserve.
|
||||
|
||||
A 3.11 break must not hide whether 3.12 and 3.13 are fine; that is
|
||||
precisely the information you want at that moment.
|
||||
"""
|
||||
ci = self._ci()
|
||||
assert 'PYTHONS: "3.11 3.12 3.13"' in ci
|
||||
assert ci.count("fail=1") >= 2, "the sweeps must collect failures, not exit early"
|
||||
|
||||
def test_uv_is_installed_without_an_action(self):
|
||||
"""Checks `uses:` directives, not prose.
|
||||
|
||||
The comment in ci.yml names the action it deliberately avoids, and that
|
||||
explanation is the most useful thing in the file -- a test that greps the
|
||||
raw text would forbid documenting the very lesson it enforces. Parsed
|
||||
with a regex rather than PyYAML on purpose: CI runs `uvx pytest`, whose
|
||||
environment holds pytest and nothing else, so a third-party import here
|
||||
fails on the runner while passing locally.
|
||||
"""
|
||||
ci = self._ci()
|
||||
used = re.findall(r"^\s*-?\s*uses:\s*(\S+)", ci, re.M)
|
||||
assert not [u for u in used if "setup-uv" in u], (
|
||||
"the action was only fetching a binary; a run: step does the same "
|
||||
"with one less moving part"
|
||||
)
|
||||
assert "astral.sh/uv/" in ci
|
||||
|
||||
def test_the_uv_version_is_pinned(self):
|
||||
ci = self._ci()
|
||||
assert re.search(r'UV_VERSION:\s*"\d+\.\d+\.\d+"', ci), (
|
||||
"an unpinned uv lets any upstream release turn main red with no "
|
||||
"code change here -- the same rule ruff is pinned under"
|
||||
)
|
||||
assert "https://astral.sh/uv/${UV_VERSION}/install.sh" in ci
|
||||
|
||||
+396
-80
@@ -43,6 +43,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
@@ -132,9 +133,43 @@ ASSUMPTIONS = [
|
||||
params=["middleware", "job", "cloud_backup"],
|
||||
why="SYNC_BLOCK wraps it to tear down bind mounts in a finally",
|
||||
),
|
||||
Assumption(
|
||||
# Not a plugin method -- a method on the middleware OBJECT itself, which the
|
||||
# manifest had no way to express and therefore never checked.
|
||||
#
|
||||
# The nested module calls `middleware.get_service(<ns>)` to decide whether to
|
||||
# sweep snapshots through `pool.snapshot` or `zfs.snapshot` (see
|
||||
# SNAPSHOT_SERVICES). If it ever disappears, `_can_delete()` catches the
|
||||
# AttributeError, reports BOTH namespaces unusable, and every nested backup
|
||||
# fails -- loudly, but only at RUN time, on a box the preflight had already
|
||||
# declared healthy. Checking it costs one file read.
|
||||
"get-service", NESTED, "utils/plugins.py",
|
||||
"LoadPluginsMixin.get_service", kind="method",
|
||||
params=["self", "name"],
|
||||
why="snapshot_service() resolves the snapshot namespace through it; without "
|
||||
"it the module cannot sweep the snapshot it just took",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def accepted_spellings(name):
|
||||
"""The method names that satisfy a call to `<namespace>.<name>`.
|
||||
|
||||
A CRUDService exposes `create`/`update`/`delete` from methods NAMED
|
||||
`do_create`/`do_update`/`do_delete`. Both are live across the matrix: 24.10 and
|
||||
25.04 declare `do_delete`, 25.10 renamed it to `delete`, and all of them answer
|
||||
to `<ns>.delete`. Accepting only the literal name reported working releases as
|
||||
BROKEN and would have switched nested snapshots off on boxes where they work.
|
||||
"""
|
||||
return (name, f"do_{name}")
|
||||
|
||||
|
||||
#: The spellings that satisfy `<ns>.delete`. A test binds this to the runtime's
|
||||
#: `truecloud_nested.DELETE_METHODS`, so the checker and the patch cannot come to
|
||||
#: disagree about what "can delete" means on the same box.
|
||||
DELETE_NAMES = accepted_spellings("delete")
|
||||
|
||||
|
||||
class MiddlewareCall:
|
||||
"""A middlewared METHOD the injected code calls at runtime.
|
||||
|
||||
@@ -161,86 +196,166 @@ class MiddlewareCall:
|
||||
the whole design: declining is always the cheaper mistake.
|
||||
"""
|
||||
|
||||
def __init__(self, ident, module, method, path, why=""):
|
||||
def __init__(self, ident, module, method, path, why="", also=()):
|
||||
self.id = ident
|
||||
self.module = module
|
||||
self.method = method # "zfs.snapshot.delete"
|
||||
self.method = method # "pool.snapshot.delete"
|
||||
self.path = path # plugin file that declares it
|
||||
self.why = why
|
||||
#: Equally acceptable spellings of the SAME call, as (method, path) pairs.
|
||||
#:
|
||||
#: No single snapshot namespace spans every supported release. 24.10 and
|
||||
#: 25.04 expose the CRUD service as the public `zfs.snapshot`; 25.10
|
||||
#: promoted it to `pool.snapshot` and demoted `zfs.snapshot` to private;
|
||||
#: 26 deleted `plugins/zfs_/` entirely. Pinning either one alone marks
|
||||
#: half the matrix BROKEN and declines to apply on versions that work
|
||||
#: perfectly well.
|
||||
#:
|
||||
#: The call is satisfied if ANY option is present. The runtime picks the
|
||||
#: same way -- see `pick_snapshot_service()` in the nested module -- so
|
||||
#: what this checks and what the patch does cannot drift apart.
|
||||
self.also = tuple(also)
|
||||
|
||||
@property
|
||||
def namespace(self):
|
||||
return self.method.rsplit(".", 1)[0]
|
||||
def options(self):
|
||||
"""Every (method, path) that would satisfy this call, best first."""
|
||||
return ((self.method, self.path), *self.also)
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.method.rsplit(".", 1)[1]
|
||||
@staticmethod
|
||||
def namespace_of(method):
|
||||
return method.rsplit(".", 1)[0]
|
||||
|
||||
@staticmethod
|
||||
def name_of(method):
|
||||
return method.rsplit(".", 1)[1]
|
||||
|
||||
|
||||
#: Every middlewared method the nested module calls at runtime.
|
||||
#: The middleware methods the nested module CALLS.
|
||||
#:
|
||||
#: These used to be the PRIVATE `zfs.*` service (`zfs.dataset.query`,
|
||||
#: `zfs.snapshot.delete`, `zfs.snapshot.query`). TrueNAS 26 deleted
|
||||
#: `plugins/zfs_/` outright and every one of them vanished -- silently, because a
|
||||
#: private service carries no stability contract and nothing warned us. The patch
|
||||
#: would have applied cleanly and then failed on the first backup.
|
||||
#:
|
||||
#: The replacements are the PUBLIC `pool.*` API, and switching to it is not merely
|
||||
#: a TrueNAS 26 fix -- it is the correct call on every version:
|
||||
#:
|
||||
#: * It is public, documented, and covered by iX's deprecation policy, so it
|
||||
#: cannot be deleted from under us the way `zfs.*` just was.
|
||||
#: * The same methods, in the same files, exist on 24.10 through 26. One code
|
||||
#: path, no version conditionals.
|
||||
#: * Both spellings take `recursive`, so ONE call sweeps the whole tree instead
|
||||
#: of ~250 individual deletes, any of which could be missed.
|
||||
MIDDLEWARE_CALLS = [
|
||||
MiddlewareCall(
|
||||
"call-zfs-dataset-query", NESTED, "zfs.dataset.query",
|
||||
"plugins/zfs_/dataset.py",
|
||||
why="SNAPSHOT_BLOCK enumerates FILESYSTEM datasets to build the staging plan",
|
||||
),
|
||||
MiddlewareCall(
|
||||
"call-zfs-snapshot-delete", NESTED, "zfs.snapshot.delete",
|
||||
"plugins/zfs_/snapshot.py",
|
||||
"call-snapshot-delete", NESTED, "pool.snapshot.delete",
|
||||
"plugins/pool_/snapshot.py",
|
||||
also=[("zfs.snapshot.delete", "plugins/zfs_/snapshot.py")],
|
||||
why="delete_snapshot_tree() sweeps the recursive snapshot. Without it every "
|
||||
"run orphans one snapshot per descendant dataset (250 on a real pool)",
|
||||
),
|
||||
MiddlewareCall(
|
||||
"call-zfs-snapshot-query", NESTED, "zfs.snapshot.query",
|
||||
"plugins/zfs_/snapshot.py",
|
||||
why="delete_snapshot_tree()'s fallback sweep enumerates the tree by name",
|
||||
),
|
||||
]
|
||||
|
||||
# There is deliberately NO entry here for a dataset or snapshot QUERY.
|
||||
#
|
||||
# The patch used to call `zfs.dataset.query` / `zfs.snapshot.query` (private, and
|
||||
# deleted in TrueNAS 26). The obvious port was to the public `pool.dataset.query` /
|
||||
# `pool.snapshot.query` -- and that port was WRONG in a way no source check could
|
||||
# ever have caught, because the methods are all present and correctly shaped.
|
||||
#
|
||||
# They are simply filtered. On a real box they return 205 of 274 datasets and 205
|
||||
# of 274 snapshots, hiding `ix-apps/*`, `.system/*` and `.ix-virt/*` -- 84 of 270
|
||||
# on the production pool, including live application data. Staging from that view
|
||||
# silently omits them; sweeping from it orphans one snapshot per hidden dataset,
|
||||
# forever.
|
||||
#
|
||||
# So the module enumerates from ZFS itself and there is no middleware assumption
|
||||
# left to check. That is the point: the fewer things we assume about middleware,
|
||||
# the less there is for iX to break. Only the MUTATION is still a middleware call,
|
||||
# and that is the one entry above.
|
||||
|
||||
|
||||
def check_call(c: MiddlewareCall, src: str | None,
|
||||
method: str | None = None, path: str | None = None,
|
||||
) -> tuple[str, str | None]:
|
||||
"""Is `method` still registered by middlewared?
|
||||
|
||||
`method`/`path` name WHICH spelling of the call is being tried -- a call may
|
||||
have several equally acceptable ones (see MiddlewareCall.also). They default
|
||||
to the preferred spelling.
|
||||
"""
|
||||
method = method or c.method
|
||||
path = path or c.path
|
||||
namespace = MiddlewareCall.namespace_of(method)
|
||||
name = MiddlewareCall.name_of(method)
|
||||
|
||||
def check_call(c: MiddlewareCall, src: str | None) -> tuple[str, str | None]:
|
||||
"""Is `c.method` still registered by middlewared?"""
|
||||
if src is None:
|
||||
return "broken", (
|
||||
f"{c.path} no longer exists, so `{c.method}` is gone"
|
||||
f"{path} no longer exists, so `{method}` is gone"
|
||||
)
|
||||
|
||||
try:
|
||||
tree = ast.parse(_stock(src))
|
||||
except SyntaxError as e:
|
||||
return "unknown", f"{c.path} does not parse: {e}"
|
||||
return "unknown", f"{path} does not parse: {e}"
|
||||
|
||||
# namespace = 'zfs.snapshot' on some Service class in this file...
|
||||
namespaces = {
|
||||
n.value.value
|
||||
for n in ast.walk(tree)
|
||||
if isinstance(n, ast.Assign)
|
||||
and isinstance(n.value, ast.Constant)
|
||||
and isinstance(n.value.value, str)
|
||||
and any(isinstance(t, ast.Name) and t.id == "namespace" for t in n.targets)
|
||||
}
|
||||
if c.namespace not in namespaces:
|
||||
return "broken", (
|
||||
f"{c.path} no longer declares namespace {c.namespace!r} "
|
||||
f"(found: {sorted(namespaces) or 'none'}), so `{c.method}` is gone"
|
||||
)
|
||||
|
||||
# ...and it defines the method.
|
||||
# Find the CLASS that declares this namespace, and look for the method THERE.
|
||||
#
|
||||
# Not anywhere in the file. `ast.walk` over the whole module made *any* function
|
||||
# called `delete` satisfy the check -- one on an unrelated class, or even a nested
|
||||
# local function inside `do_query`. That is a FALSE OK, and it breaks the one
|
||||
# invariant this checker and the runtime share: `_defines_delete()` looks in
|
||||
# `vars(klass)` for a PLUGIN class on the service's MRO. If iX gutted
|
||||
# `PoolSnapshotService.do_delete` while some other class in the same file still had
|
||||
# a `delete`, compat would say ok, apply.sh would patch, and the runtime would then
|
||||
# correctly refuse `pool.snapshot`, fall through to a `zfs.snapshot` that does not
|
||||
# exist on 26, and fail every nested backup on a box the preflight called healthy.
|
||||
#
|
||||
# Same question on both sides: does the class that OWNS this namespace define the
|
||||
# method?
|
||||
#
|
||||
# A CRUDService exposes `create`/`update`/`delete` from methods NAMED
|
||||
# `do_create`/`do_update`/`do_delete`. Both spellings are live right now:
|
||||
# 24.10 and 25.04 declare `do_delete`, 25.10 renamed it to `delete`, and all
|
||||
# three answer to `zfs.snapshot.delete`. Accepting only the literal name reported
|
||||
# the two older releases as broken -- a false BROKEN that would have switched off
|
||||
# nested snapshots on boxes where they work.
|
||||
defined = {
|
||||
n.name for n in ast.walk(tree)
|
||||
if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef)
|
||||
}
|
||||
if c.name not in defined and f"do_{c.name}" not in defined:
|
||||
return "broken", f"{c.path} no longer defines `{c.method}`"
|
||||
# `do_create`/`do_update`/`do_delete`. Both spellings are live: 24.10 and 25.04
|
||||
# declare `do_delete`, 25.10 renamed it to `delete`, and all answer to
|
||||
# `<ns>.delete`. Accepting only the literal name reported working releases as
|
||||
# broken.
|
||||
owners = []
|
||||
all_namespaces = set()
|
||||
for cls in (n for n in ast.walk(tree) if isinstance(n, ast.ClassDef)):
|
||||
declared = {
|
||||
n.value.value
|
||||
for n in ast.walk(cls)
|
||||
if isinstance(n, ast.Assign)
|
||||
and isinstance(n.value, ast.Constant)
|
||||
and isinstance(n.value.value, str)
|
||||
and any(isinstance(t, ast.Name) and t.id == "namespace" for t in n.targets)
|
||||
}
|
||||
all_namespaces |= declared
|
||||
if namespace in declared:
|
||||
owners.append(cls)
|
||||
|
||||
return "ok", None
|
||||
if not owners:
|
||||
return "broken", (
|
||||
f"{path} no longer declares namespace {namespace!r} "
|
||||
f"(found: {sorted(all_namespaces) or 'none'}), so `{method}` is gone"
|
||||
)
|
||||
|
||||
wanted = accepted_spellings(name)
|
||||
for cls in owners:
|
||||
# Direct members of the class, not its nested scopes: a `def delete` inside
|
||||
# another method is a local function, not a service method.
|
||||
if any(
|
||||
isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef) and n.name in wanted
|
||||
for n in cls.body
|
||||
):
|
||||
return "ok", None
|
||||
|
||||
return "broken", (
|
||||
f"{path} still declares namespace {namespace!r}, but its class no longer "
|
||||
f"defines `{'` or `'.join(wanted)}` -- so `{method}` is gone"
|
||||
)
|
||||
|
||||
|
||||
#: Things that mean iX has done the job themselves and the module should RETIRE,
|
||||
@@ -403,7 +518,10 @@ def check_source(a: Assumption, src: str | None) -> tuple[str, str | None]:
|
||||
breaks a box that was working.
|
||||
"""
|
||||
if src is None:
|
||||
return "broken", f"{a.path} does not exist"
|
||||
# Name the SYMBOL, not just the file. Whoever reads the bug report needs to
|
||||
# know what the patch can no longer reach, and "utils/plugins.py does not
|
||||
# exist" does not tell them that `get_service` is gone.
|
||||
return "broken", f"{a.path} does not exist, so `{a.symbol}` is gone"
|
||||
|
||||
try:
|
||||
tree = ast.parse(src)
|
||||
@@ -538,6 +656,7 @@ def check(loader, modules=None) -> dict:
|
||||
out[a.module]["unknown"] = True
|
||||
out[a.module]["problems"].append({
|
||||
"id": a.id, "detail": f"could not read {a.path}: {e}", "why": a.why,
|
||||
"state": "unknown",
|
||||
})
|
||||
continue
|
||||
|
||||
@@ -545,37 +664,57 @@ def check(loader, modules=None) -> dict:
|
||||
if status == "broken":
|
||||
out[a.module]["ok"] = False
|
||||
out[a.module]["problems"].append({
|
||||
"id": a.id, "detail": detail, "why": a.why,
|
||||
"id": a.id, "detail": detail, "why": a.why, "state": "broken",
|
||||
})
|
||||
elif status == "unknown":
|
||||
out[a.module]["unknown"] = True
|
||||
out[a.module]["problems"].append({
|
||||
"id": a.id, "detail": detail, "why": a.why,
|
||||
"id": a.id, "detail": detail, "why": a.why, "state": "unknown",
|
||||
})
|
||||
|
||||
# The methods the injected code CALLS, not just the symbols it wraps.
|
||||
#
|
||||
# A call may have several equally acceptable spellings, because no single
|
||||
# snapshot namespace spans every supported release (24.10 has `zfs.snapshot`,
|
||||
# 26 has only `pool.snapshot`). It is satisfied if ANY of them is present --
|
||||
# exactly as the runtime resolves it -- and BROKEN only when they all vanish.
|
||||
for c in MIDDLEWARE_CALLS:
|
||||
if c.module not in out:
|
||||
continue
|
||||
try:
|
||||
text = src(c.path)
|
||||
except Unreadable as e:
|
||||
out[c.module]["unknown"] = True
|
||||
out[c.module]["problems"].append({
|
||||
"id": c.id, "detail": f"could not read {c.path}: {e}", "why": c.why,
|
||||
})
|
||||
|
||||
satisfied, unknown, details = False, False, []
|
||||
for method, path in c.options:
|
||||
try:
|
||||
text = src(path)
|
||||
except Unreadable as e:
|
||||
unknown = True
|
||||
details.append(f"could not read {path}: {e}")
|
||||
continue
|
||||
|
||||
status, detail = check_call(c, text, method, path)
|
||||
if status == "ok":
|
||||
satisfied = True
|
||||
break
|
||||
if status == "unknown":
|
||||
unknown = True
|
||||
details.append(detail)
|
||||
|
||||
if satisfied:
|
||||
continue
|
||||
|
||||
status, detail = check_call(c, text)
|
||||
if status == "broken":
|
||||
out[c.module]["ok"] = False
|
||||
out[c.module]["problems"].append({
|
||||
"id": c.id, "detail": detail, "why": c.why,
|
||||
})
|
||||
elif status == "unknown":
|
||||
# Every spelling failed. If we could not READ one of them we do not know
|
||||
# that it is broken -- a rate-limited fetch is not a regression.
|
||||
if unknown:
|
||||
out[c.module]["unknown"] = True
|
||||
out[c.module]["problems"].append({
|
||||
"id": c.id, "detail": detail, "why": c.why,
|
||||
"id": c.id, "detail": "; ".join(details), "why": c.why,
|
||||
"state": "unknown",
|
||||
})
|
||||
else:
|
||||
out[c.module]["ok"] = False
|
||||
out[c.module]["problems"].append({
|
||||
"id": c.id, "detail": "; ".join(details), "why": c.why,
|
||||
"state": "broken",
|
||||
})
|
||||
|
||||
for module, (path, phrase, native_when_present) in NATIVE_PROBES.items():
|
||||
@@ -732,6 +871,24 @@ def discover_refs(remote: str = REPO) -> list[str]:
|
||||
* UNRELEASED comes from the BRANCHES, because that is where a beta appears
|
||||
first: `release/26.0.0-BETA.3` had no tag yet while it was the newest beta.
|
||||
Catching breakage here, before it ships, is the whole point of this file.
|
||||
|
||||
THE NEXT MAINTENANCE RELEASE IS ALSO A BRANCH, and it used to fall through both
|
||||
sieves. `release/25.10.5` is branched but not yet tagged, and 25.10 has already
|
||||
shipped -- so it is not in `shipped` (no tag) and it was excluded from `upcoming`
|
||||
(its line is in `shipped_lines`). It was invisible. That is the one ref a 25.10.4
|
||||
user is actually about to be upgraded onto, so a break there reaches people
|
||||
BEFORE the daily check ever looks at it -- the exact hole this file exists to
|
||||
close, on the only line anybody is running.
|
||||
|
||||
The rule that separates it from the two things we must NOT report:
|
||||
|
||||
* `release/24.10-RC.2` -- a prerelease of an already-shipped line. It is
|
||||
history, not a warning; it sorts OLDER than TS-24.10.2.4, so it is dropped.
|
||||
* `release/25.20.2.2` -- iX's typo branch. 25.20 never shipped, so it has no
|
||||
TS tag, so it is not a line at all and is dropped.
|
||||
|
||||
...which is: a plain `release/X.Y.Z` branch counts only if its line HAS shipped
|
||||
and it sorts NEWER than that line's newest tag. Both exclusions fall out of it.
|
||||
"""
|
||||
tags = _ls_remote(remote, "--tags")
|
||||
heads = _ls_remote(remote, "--heads")
|
||||
@@ -739,26 +896,45 @@ def discover_refs(remote: str = REPO) -> list[str]:
|
||||
shipped = _newest_per_line([
|
||||
t for t in tags if t.startswith("TS-") and "-BETA" not in t and "-RC" not in t
|
||||
])
|
||||
newest_shipped = {_version_of(t)[0][:2]: _version_of(t) for t in shipped}
|
||||
|
||||
release_heads = [h for h in heads if h.startswith("release/")]
|
||||
|
||||
# A prerelease of a line that has ALREADY shipped is history, not a warning:
|
||||
# release/24.10-RC.2 still exists, and the nested module does not apply to it,
|
||||
# but 24.10 shipped long ago and TS-24.10.2.4 is fine. Reporting it would be a
|
||||
# standing red row in the matrix for a version nobody can install.
|
||||
shipped_lines = {_version_of(t)[0][:2] for t in shipped}
|
||||
upcoming = [
|
||||
h for h in _newest_per_line([
|
||||
h for h in heads
|
||||
if h.startswith("release/") and ("-BETA" in h or "-RC" in h)
|
||||
h for h in release_heads if "-BETA" in h or "-RC" in h
|
||||
])
|
||||
if _version_of(h)[0][:2] not in shipped_lines
|
||||
if _version_of(h)[0][:2] not in newest_shipped
|
||||
]
|
||||
|
||||
return [*shipped, *upcoming, "master"]
|
||||
# The next maintenance release of a line that HAS shipped: branched, untagged,
|
||||
# and the very next thing those users get. See the docstring.
|
||||
pending = []
|
||||
for h in _newest_per_line([
|
||||
h for h in release_heads if "-BETA" not in h and "-RC" not in h
|
||||
]):
|
||||
v = _version_of(h)
|
||||
tagged = newest_shipped.get(v[0][:2])
|
||||
if tagged and v > tagged:
|
||||
pending.append(h)
|
||||
|
||||
return [*shipped, *pending, *upcoming, "master"]
|
||||
|
||||
|
||||
def is_unreleased(ref: str) -> bool:
|
||||
"""master and any BETA/RC. Breakage here is early warning, not an outage."""
|
||||
return ref == "master" or "-BETA" in ref or "-RC" in ref
|
||||
"""Anything iX has not TAGGED. Breakage here is early warning, not an outage.
|
||||
|
||||
Keyed on where the ref came from, not on its name: `discover_refs` takes shipped
|
||||
releases from `TS-*` TAGS and everything else from BRANCHES, so a `release/*` ref
|
||||
is by construction something iX has not released yet. Testing for `-BETA`/`-RC`
|
||||
instead would call `release/25.10.5` SHIPPED, and a break there would fail the
|
||||
build as a live outage -- on a version nobody is running yet.
|
||||
"""
|
||||
return ref == "master" or ref.startswith("release/")
|
||||
|
||||
|
||||
def matrix(refs=None, remote: str = REPO) -> list[dict]:
|
||||
@@ -802,7 +978,16 @@ def is_broken(r: dict) -> bool:
|
||||
#: is static analysis of iX's source, which proves the patch's assumptions hold --
|
||||
#: a strictly weaker claim than "a restore worked". Add a row only after doing it.
|
||||
HARDWARE_VERIFIED = {
|
||||
"25.10.4": "nested + providers; 252-snapshot recursive backup of /mnt/Tap, 18m",
|
||||
"25.10.4": (
|
||||
"v0.7.0: 3 live tasks — 191-dataset nested backup of /mnt/Tap, a "
|
||||
"215-filesystem/2-zvol backup of /mnt/Tank/backups, and a non-nested one; "
|
||||
"0 orphans, 0 leaked mounts, byte-identical restore; the collector also "
|
||||
"reclaimed a real orphan the pool had been carrying"
|
||||
),
|
||||
"26.0.0-BETA.1": (
|
||||
"v0.7.0: 274-snapshot recursive backup of a 292-dataset pool; restored a "
|
||||
"4-deep child dataset byte-identical; zvol-orphan case reproduced then closed"
|
||||
),
|
||||
}
|
||||
|
||||
_LEGEND = """
|
||||
@@ -815,6 +1000,16 @@ _LEGEND = """
|
||||
"ok" means *the patch's assumptions hold*, checked automatically against iX's
|
||||
source. It does not mean a human ran a backup on it — that is the
|
||||
**Hardware-verified** column, which is filled in by hand and only by doing it.
|
||||
|
||||
**`master` is not the next release.** iX branches each major off to its own
|
||||
`release/` line and master rolls straight on to the one after — so master is
|
||||
`27-dev` while 26 is still in beta. A **BROKEN** master means iX has changed
|
||||
something that will reach users *a major release from now*, not in the version you
|
||||
are about to install. Read the numbered rows for that.
|
||||
|
||||
A row like `25.10.5 _(unreleased)_` is the next maintenance release: branched by iX,
|
||||
not tagged yet, and the very next thing a 25.10.4 box gets. It is checked precisely
|
||||
because it is the one unshipped ref that reaches real users without warning.
|
||||
"""
|
||||
|
||||
|
||||
@@ -862,17 +1057,42 @@ def update_readme(rows: list[dict], path: str = README) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def dev_label(rows: list[dict]) -> str:
|
||||
"""What `master` is a development line FOR -- e.g. "27-dev".
|
||||
|
||||
`master` is NOT the next release, and labelling it "master _(unreleased)_" said
|
||||
it was. On 2026-07-14 every recent commit on master targeted 27.0.0-BETA.1 while
|
||||
26 was still in beta on its own `release/26.0.0-BETA.*` branches: master had
|
||||
already rolled over to the major AFTER the one that has not shipped yet. So a red
|
||||
`master` row read as "the version you are about to install is broken" when the
|
||||
breakage was a year out, on a line nobody can even download. That is a false alarm
|
||||
aimed squarely at the person deciding whether to trust this with their backups.
|
||||
|
||||
Derived, not hardcoded: the newest major we can see anywhere, plus one. iX branches
|
||||
`release/N.0.0-BETA.1` off master and master immediately becomes N+1 -- so when 27
|
||||
betas appear, this says 28-dev on its own.
|
||||
"""
|
||||
majors = [
|
||||
v[0][0] for v in (_version_of(r["ref"]) for r in rows if r["ref"] != "master")
|
||||
if v
|
||||
]
|
||||
return f"{max(majors) + 1}-dev" if majors else "unreleased"
|
||||
|
||||
|
||||
def render_markdown(rows: list[dict]) -> str:
|
||||
"""The matrix, for the README."""
|
||||
out = [
|
||||
"| TrueNAS | B2/S3 providers | Nested snapshots | Hardware-verified |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
dev = dev_label(rows)
|
||||
for row in rows:
|
||||
m = row["modules"]
|
||||
ref = row["ref"]
|
||||
label = ref.removeprefix("TS-").removeprefix("release/")
|
||||
if row["unreleased"]:
|
||||
if ref == "master":
|
||||
label = f"master _({dev})_"
|
||||
elif row["unreleased"]:
|
||||
label = f"{label} _(unreleased)_"
|
||||
|
||||
cells = []
|
||||
@@ -892,6 +1112,96 @@ def render_markdown(rows: list[dict]) -> str:
|
||||
return "\n".join(out) + "\n" + _LEGEND
|
||||
|
||||
|
||||
FINGERPRINT = "<!-- compat-fingerprint:"
|
||||
|
||||
|
||||
def fingerprint(rows: list[dict]) -> str:
|
||||
"""A stable digest of WHAT IS BROKEN, and nothing else.
|
||||
|
||||
The bug report must be updated when the findings change and stay silent when they
|
||||
do not. Without this the workflow commented on every run -- it left **11 identical
|
||||
3,000-character comments** on one issue in a single day, which is not a warning
|
||||
system, it is a mute button with extra steps.
|
||||
|
||||
Deliberately excludes anything that moves on its own: the matrix's `ok` rows, the
|
||||
hardware-verified column, and the exact TrueNAS point-release (`TS-25.10.4` ->
|
||||
`TS-25.10.5` is not news). Only the broken (ref, module, problem-id) triples count.
|
||||
"""
|
||||
findings = sorted(
|
||||
(r["ref"], mod, p["id"])
|
||||
for r in rows
|
||||
for mod, m in r["modules"].items()
|
||||
if is_broken(m)
|
||||
for p in m["problems"]
|
||||
# `unknown` problems are things we could not READ (a 429, an EACCES), not
|
||||
# things iX changed. On a ref that is broken for some other reason they would
|
||||
# otherwise join the digest, so one transient network blip rewrites the issue
|
||||
# body and the next clean run rewrites it back. That is the daily-noise
|
||||
# failure this fingerprint exists to prevent, wearing a different hat.
|
||||
if p.get("state", "broken") == "broken"
|
||||
)
|
||||
return hashlib.sha256(repr(findings).encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def extract_fingerprint(body: str) -> str | None:
|
||||
"""The fingerprint a previous run left in the issue body, if any."""
|
||||
if not body:
|
||||
return None
|
||||
i = body.find(FINGERPRINT)
|
||||
if i == -1:
|
||||
return None
|
||||
return body[i + len(FINGERPRINT):].split("-->", 1)[0].strip() or None
|
||||
|
||||
|
||||
def render_issue(rows: list[dict]) -> str:
|
||||
"""The bug report body: what is broken, why it matters, and nothing else up front.
|
||||
|
||||
Short by design. The full matrix and the healthy versions go in a fold -- somebody
|
||||
opening this wants to know what broke and whether it can hurt them, not to re-read
|
||||
a table they can see in the README.
|
||||
"""
|
||||
broken = [r for r in rows if any(is_broken(m) for m in r["modules"].values())]
|
||||
|
||||
out = [
|
||||
"`tools/compat.py` checks what this patch assumes about middlewared against "
|
||||
"iXsystems' actual source, every day. Those assumptions no longer hold on the "
|
||||
"versions below.",
|
||||
"",
|
||||
"**This does not break anyone today.** `apply.sh` re-checks on every boot and "
|
||||
"**declines to apply** a module whose assumptions fail, so TrueNAS is left "
|
||||
"stock rather than half-patched. The cost is the module's feature, not a "
|
||||
"broken backup.",
|
||||
"",
|
||||
]
|
||||
|
||||
for r in broken:
|
||||
out.append(f"### `{r['ref']}`")
|
||||
out.append("")
|
||||
for mod, m in sorted(r["modules"].items()):
|
||||
if not is_broken(m):
|
||||
continue
|
||||
out.append(f"**{mod}**")
|
||||
out.append("")
|
||||
for p in m["problems"]:
|
||||
out.append(f"- {p['detail']}")
|
||||
out.append(f" <br><sub>{p['why']}</sub>")
|
||||
out.append("")
|
||||
|
||||
out += [
|
||||
"<details><summary>Full support matrix</summary>",
|
||||
"",
|
||||
render_markdown(rows),
|
||||
"</details>",
|
||||
"",
|
||||
"_Filed and kept up to date by "
|
||||
"[`compat.yml`](.github/workflows/compat.yml). It edits this body when the "
|
||||
"findings change, and stays quiet when they do not._",
|
||||
"",
|
||||
f"{FINGERPRINT} {fingerprint(rows)} -->",
|
||||
]
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def render_matrix(rows: list[dict]) -> str:
|
||||
"""A support table.
|
||||
|
||||
@@ -900,7 +1210,13 @@ def render_matrix(rows: list[dict]) -> str:
|
||||
hardware-verified column lives in COMPATIBILITY.md and is maintained by hand,
|
||||
because nothing else can honestly fill it in.
|
||||
"""
|
||||
w = max((len(r["ref"]) for r in rows), default=10)
|
||||
dev = dev_label(rows)
|
||||
# Same relabel as the README: a red `master` is a warning about the major AFTER
|
||||
# next, and "master" alone reads as "the release you are about to install".
|
||||
names = {r["ref"]: (f"master ({dev})" if r["ref"] == "master" else r["ref"])
|
||||
for r in rows}
|
||||
|
||||
w = max((len(n) for n in names.values()), default=10)
|
||||
lines = [
|
||||
f"{'TrueNAS'.ljust(w)} {'providers':<10} {'nested':<10}",
|
||||
f"{'-' * w} {'-' * 10} {'-' * 10}",
|
||||
@@ -908,7 +1224,7 @@ def render_matrix(rows: list[dict]) -> str:
|
||||
for row in rows:
|
||||
m = row["modules"]
|
||||
lines.append(
|
||||
f"{row['ref'].ljust(w)} "
|
||||
f"{names[row['ref']].ljust(w)} "
|
||||
f"{_verdict(m[PROVIDERS]):<10} {_verdict(m[NESTED]):<10}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Keep ONE bug report in sync with what compat.py currently finds.
|
||||
|
||||
WHY THIS IS NOT JUST "POST A COMMENT"
|
||||
-------------------------------------
|
||||
The first version commented on every run that found a break. In one day it left
|
||||
**11 identical 3,000-character comments** on the same issue. That is not a warning
|
||||
system; it is a mute button with extra steps. The next real finding would have been
|
||||
scrolled past, which defeats the entire point of building it.
|
||||
|
||||
So:
|
||||
|
||||
* **The issue body is the current truth.** It is edited in place, never appended to.
|
||||
* **Comments are a changelog of CHANGES.** A run whose findings are identical to the
|
||||
last one says nothing at all -- no comment, no edit, no notification.
|
||||
* A fingerprint of the findings (broken ref/module/problem triples only) is embedded
|
||||
in the body. It deliberately ignores things that move on their own -- healthy rows,
|
||||
the hardware-verified column, TrueNAS point releases -- so `TS-25.10.4` becoming
|
||||
`TS-25.10.5` is not news, and does not wake anybody up.
|
||||
|
||||
* When everything is fixed, the issue is **closed** with a comment saying so.
|
||||
|
||||
Works against GitHub and Gitea, which differ only in the auth header and the issue
|
||||
list URL. One implementation, so the two cannot drift.
|
||||
|
||||
python3 tools/compat_publish.py --api <url> --token <tok> --matrix /tmp/matrix.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
sys.path.insert(0, __file__.rsplit("/", 1)[0])
|
||||
|
||||
from compat import ( # noqa: E402
|
||||
extract_fingerprint,
|
||||
fingerprint,
|
||||
is_broken,
|
||||
render_issue,
|
||||
)
|
||||
|
||||
TITLE = "TrueNAS compatibility: the patch's assumptions no longer hold"
|
||||
|
||||
|
||||
def _call(url, token, method="GET", data=None):
|
||||
req = urllib.request.Request(
|
||||
url, method=method,
|
||||
headers={
|
||||
# Gitea wants `token <t>`; GitHub accepts `Bearer <t>`. GitHub also
|
||||
# accepts `token <t>`, so one header serves both.
|
||||
"Authorization": f"token {token}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/vnd.github+json",
|
||||
},
|
||||
data=json.dumps(data).encode() if data else None,
|
||||
)
|
||||
with urllib.request.urlopen(req) as r: # noqa: S310
|
||||
return json.load(r) if r.length != 0 else {}
|
||||
|
||||
|
||||
def _same(a, b):
|
||||
"""Is the issue body already what we would write?
|
||||
|
||||
Compared after normalising line endings and trailing space: forges are free to
|
||||
round-trip `\r\n`, and a body that only "differs" by that would be rewritten on
|
||||
every single run -- a silent edit, but a pointless one that churns `updated_at`
|
||||
and makes the issue look freshly touched every morning.
|
||||
"""
|
||||
def norm(s):
|
||||
return "\n".join(line.rstrip() for line in (s or "").replace("\r\n", "\n").split("\n")).strip()
|
||||
return norm(a) == norm(b)
|
||||
|
||||
|
||||
def find_issue(api, token, title):
|
||||
"""The LOWEST-numbered issue with this title, open or closed.
|
||||
|
||||
Lowest, not "whichever the API returns first": two issues with the same title
|
||||
existed once (an earlier version put the ref list in the title, so the identity
|
||||
changed whenever that set changed), and an order-dependent pick would alternate
|
||||
between them -- reopening one while commenting on the other.
|
||||
|
||||
Both forges list PRs alongside issues, but they SAY SO DIFFERENTLY: GitHub omits
|
||||
the `pull_request` key on a plain issue, Gitea sends it as `null`. Testing for the
|
||||
KEY therefore discards every Gitea issue as if it were a PR -- so this returned
|
||||
None on every Gitea run, and the bot filed a brand-new duplicate report each time
|
||||
instead of editing the one it already had. Test the VALUE; it is the only form
|
||||
that is true on both.
|
||||
"""
|
||||
issues = _call(f"{api}/issues?state=all&per_page=100&limit=100", token)
|
||||
mine = [
|
||||
i for i in issues
|
||||
if i.get("title") == title and not i.get("pull_request")
|
||||
]
|
||||
return min(mine, key=lambda i: i["number"]) if mine else None
|
||||
|
||||
|
||||
def main(argv):
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--api", required=True, help="…/repos/<owner>/<repo>")
|
||||
ap.add_argument("--token", required=True)
|
||||
ap.add_argument("--matrix", required=True, help="compat.py --matrix --json output")
|
||||
args = ap.parse_args(argv[1:])
|
||||
|
||||
with open(args.matrix, encoding="utf-8") as fh:
|
||||
rows = json.load(fh)
|
||||
|
||||
broken = [r for r in rows if any(is_broken(m) for m in r["modules"].values())]
|
||||
issue = find_issue(args.api, args.token, TITLE)
|
||||
|
||||
# ── everything is healthy ────────────────────────────────────────────────
|
||||
if not broken:
|
||||
if issue and issue["state"] == "open":
|
||||
_call(f"{args.api}/issues/{issue['number']}/comments", args.token, "POST",
|
||||
{"body": "All of the patch's assumptions hold again on every "
|
||||
"checked TrueNAS version. Closing."})
|
||||
_call(f"{args.api}/issues/{issue['number']}", args.token, "PATCH",
|
||||
{"state": "closed"})
|
||||
print(f"closed #{issue['number']} — nothing is broken any more")
|
||||
else:
|
||||
print("nothing broken; no open report to close")
|
||||
return 0
|
||||
|
||||
body = render_issue(rows)
|
||||
want = fingerprint(rows)
|
||||
|
||||
# ── nothing to file yet ──────────────────────────────────────────────────
|
||||
if issue is None:
|
||||
made = _call(f"{args.api}/issues", args.token, "POST",
|
||||
{"title": TITLE, "body": body})
|
||||
print(f"filed #{made['number']}")
|
||||
return 0
|
||||
|
||||
have = extract_fingerprint(issue.get("body") or "")
|
||||
n = issue["number"]
|
||||
|
||||
# ── two different questions, and they were being answered with one answer ────
|
||||
#
|
||||
# * IS THE BODY STILL TRUE? -> if not, rewrite it. Editing an issue body
|
||||
# notifies NOBODY on either forge, so keeping it honest is free.
|
||||
# * HAVE THE FINDINGS CHANGED? -> only then comment. Comments DO notify, and a
|
||||
# daily "still broken, same as yesterday" is what teaches everyone to ignore
|
||||
# the one that finally matters.
|
||||
#
|
||||
# Conflating them meant an unchanged FINGERPRINT froze the BODY. The fingerprint
|
||||
# deliberately ignores everything that moves on its own -- healthy rows, the
|
||||
# hardware-verified column, point releases, how a row is LABELLED -- so none of
|
||||
# that could ever reach the report. Relabelling master `27-dev` (it is not the
|
||||
# next release; a red row there was reading as "the version you are about to
|
||||
# install is broken") would have shipped to the README and never to the issue
|
||||
# anybody actually opens.
|
||||
body_is_current = _same(issue.get("body"), body)
|
||||
|
||||
if have == want and issue["state"] == "open" and body_is_current:
|
||||
print(f"#{n} is already current ({want}) — staying quiet")
|
||||
return 0
|
||||
|
||||
_call(f"{args.api}/issues/{n}", args.token, "PATCH", {"body": body, "state": "open"})
|
||||
|
||||
if have != want:
|
||||
refs = ", ".join(f"`{r['ref']}`" for r in broken)
|
||||
note = (
|
||||
"The findings changed — the report above has been updated.\n\n"
|
||||
f"Currently broken on: {refs}."
|
||||
if have else
|
||||
"This report is now kept up to date automatically: the body above always "
|
||||
"reflects the current findings, and a comment is only added when they "
|
||||
"change."
|
||||
)
|
||||
_call(f"{args.api}/issues/{n}/comments", args.token, "POST", {"body": note})
|
||||
print(f"updated #{n}: {have} -> {want}")
|
||||
elif issue["state"] != "open":
|
||||
print(f"reopened #{n}")
|
||||
else:
|
||||
# Same findings, new rendering. Silent by design: nothing has changed that
|
||||
# anybody needs waking up for, but the report should not be telling lies.
|
||||
print(f"#{n}: findings unchanged ({want}); body refreshed silently")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -107,10 +107,15 @@ def changelog_versions(text: str) -> list[str]:
|
||||
def extract_notes(text: str, version: str) -> str:
|
||||
"""The body of one version's section, without its heading.
|
||||
|
||||
A release candidate resolves to its BASE version: v0.6.0-rc2 ships the same code
|
||||
as v0.6.0 and therefore the same notes, and the CHANGELOG only ever has the one
|
||||
section. Without this, the release workflow cut the tag, passed every gate, and
|
||||
then died extracting the body -- so the candidate existed but was never published.
|
||||
|
||||
Raises KeyError if the version has no section -- a release with an empty or
|
||||
wrong body is worse than a failed release.
|
||||
"""
|
||||
want = normalise(version)
|
||||
want = base_version(version)
|
||||
lines = text.splitlines()
|
||||
|
||||
start = None
|
||||
|
||||
+6
-1
@@ -3,7 +3,7 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="0.6.0"
|
||||
VERSION="0.8.0"
|
||||
|
||||
PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
_HOOK_COMMENT='TrueCloud provider patch (S3/B2)'
|
||||
@@ -124,6 +124,11 @@ if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]; then
|
||||
rm -f "$PATCH_DIR/nested_snapshots_enabled"
|
||||
echo " Removed nested-snapshot opt-in marker."
|
||||
fi
|
||||
|
||||
# Runtime breadcrumb recorded by apply.sh for wait_restart.sh. Harmless, but a
|
||||
# stale path left in an uninstalled tree is exactly the sort of thing that reads
|
||||
# as state later.
|
||||
rm -f "$PATCH_DIR/.mw_dir"
|
||||
echo ""
|
||||
|
||||
if [ "$_restore_failed" -eq 1 ]; then
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="0.6.0"
|
||||
VERSION="0.8.0"
|
||||
|
||||
PATCH_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
_PREV_FILE="$PATCH_DIR/.update_previous"
|
||||
@@ -123,7 +123,7 @@ cd "$PATCH_DIR"
|
||||
|
||||
if ! git rev-parse --git-dir >/dev/null 2>&1; then
|
||||
echo "ERROR: $PATCH_DIR is not a git clone — nothing to update." >&2
|
||||
echo " Re-clone from https://git.onetick.ninja/flan/truenas-truecloud-patch" >&2
|
||||
echo " Re-clone from https://github.com/sudolulo/truenas-truecloud-patch" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -136,9 +136,21 @@ fi
|
||||
|
||||
# A dirty tree means someone edited or scp'd files in place; merging over that
|
||||
# silently loses their changes, or conflicts halfway through.
|
||||
if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
|
||||
#
|
||||
# CONTENT changes only. A mode-only change (100644 -> 100755) is not somebody's work
|
||||
# and must not block an update -- and it is not hypothetical: install.sh chmod +x's
|
||||
# these very scripts, so on any version where git recorded one as 100644, INSTALLING
|
||||
# dirtied the checkout and update.sh then refused to run. Install once, and updating
|
||||
# was blocked forever, with an error telling the user to `git checkout -- .` (which
|
||||
# merely undoes the exec bit so the next install can re-dirty it). A real box sat on
|
||||
# an old version for exactly this reason.
|
||||
#
|
||||
# `git diff --numstat` reports "0 0 file" for a mode-only change, so anything with a
|
||||
# nonzero insert or delete count is a genuine edit.
|
||||
_dirty=$(git diff --numstat HEAD -- . | awk '$1 != 0 || $2 != 0 { print $3 }')
|
||||
if [ -n "$_dirty" ]; then
|
||||
echo "ERROR: the working tree has uncommitted changes:" >&2
|
||||
git status --short --untracked-files=no >&2
|
||||
printf ' M %s\n' $_dirty >&2
|
||||
echo "" >&2
|
||||
echo " Refusing to update over them. Commit, stash, or discard them first:" >&2
|
||||
echo " git -C $PATCH_DIR checkout -- ." >&2
|
||||
|
||||
Reference in New Issue
Block a user