Fix post-merge audit findings; unify staging teardown
Audit ----- - create_task.py verify failed on a DEFAULT install. hook_status.json emitted a per-file entry for the nested module with ok:false whenever the feature was switched off -- the default -- so verify printed [FAIL] and exited 1, right after the README tells users to run it. Status is now per MODULE with an `active` flag, and verify renders an inactive module as [SKIP]. - A partial apply suppressed the middlewared restart. The exit code conflated "nothing applied" with "one module applied, one failed", so a failing providers patch would prevent the restart that a freshly-applied nested patch needs, leaving it on disk and never loaded. Exit 2 now means partial and the restart still fires. - The native-nested probe could never fire. It scanned crud.py for the guard message, but our own injected block quotes that message, so once applied the probe would always conclude the guard was still present. It now reads only the stock portion of the file. - recover.sh did not unmount staging trees, so an emergency recovery left bind mounts pinning ZFS snapshots that could then never be destroyed. - uninstall.sh deleted sidecar files without reading them. A sidecar is the only record that an interrupted run's snapshot tree is still on disk; both scripts now name the snapshot before clearing it. - Removed a dead branch in the restart gate (unreachable: the kill switch exits). Refactor -------- - Staging teardown had been copy-pasted into uninstall.sh and recover.sh -- two untested shell copies of the fiddly depth-ordering and lazy-umount logic. Both now call `python3 patch/truecloud_nested.py cleanup`, so there is exactly one implementation and it is the one under test. - Dropped the in-memory ACTIVE dict. The sidecar file was already the source of truth; a second in-process record could only desync -- and it is precisely the middlewared-restart case (which empties it) that must not orphan a snapshot tree. One record, on disk, or none. Not done: the overlay-unmount loop is duplicated across apply.sh/uninstall.sh/ recover.sh. It is pre-existing, and apply.sh runs at PREINIT under a tight timeout -- giving it a source dependency would trade 10 lines of duplication for a boot-time failure mode. 74 tests, ruff and shellcheck clean.
This commit is contained in:
flan
+51
-34
@@ -153,15 +153,22 @@ except Exception:
|
||||
pass
|
||||
|
||||
# nested: stock gates nested datasets with a validation in plugins/cloud/crud.py.
|
||||
# We only ever append to that file, so the stock text survives our patch -- if the
|
||||
# guard is gone, iX removed it, which means they implemented the traversal.
|
||||
# If that guard is gone, iX removed it, which means they implemented the traversal.
|
||||
#
|
||||
# Only look at the STOCK part of the file. Our own CRUD_BLOCK quotes the guard
|
||||
# message (it filters on it), so scanning the whole file would find the string in
|
||||
# our own patch and conclude the guard is still there. That happens to be
|
||||
# harmless today because detection runs before patching, but it makes the probe
|
||||
# silently order-dependent -- so cut our block off explicitly.
|
||||
#
|
||||
# If the file cannot be read we assume 'no' and keep patching: worst case the
|
||||
# patch declines to apply and the option simply stays unavailable.
|
||||
try:
|
||||
crud = os.path.join(result['mw_dir'], 'plugins', 'cloud', 'crud.py')
|
||||
with open(crud, encoding='utf-8', errors='replace') as fh:
|
||||
if 'no further nesting' not in fh.read():
|
||||
result['native_nested'] = 'yes'
|
||||
stock_src = fh.read().split('\n# TRUECLOUD_PATCH', 1)[0]
|
||||
if 'no further nesting' not in stock_src:
|
||||
result['native_nested'] = 'yes'
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -532,29 +539,21 @@ else:
|
||||
print('WARNING: Stock nesting guard remains; snapshot option stays unavailable')
|
||||
print('WARNING: for nested datasets. Existing backups are unaffected.')
|
||||
|
||||
# One entry per MODULE, not per file. `ok` means "nothing is wrong", so a module
|
||||
# that is inactive (superseded, or opt-in and off) is ok -- reporting a disabled
|
||||
# opt-in feature as FAIL would make `create_task.py verify` fail on a default
|
||||
# install. `active` says whether the module is doing anything.
|
||||
patches = {
|
||||
'module.providers': {
|
||||
'ok': bool(b2_ok and restic_ok) or not providers_needed,
|
||||
'providers': {
|
||||
'ok': (not providers_needed) or bool(b2_ok and restic_ok),
|
||||
'active': providers_needed,
|
||||
'detail': providers_detail,
|
||||
},
|
||||
'module.nested_snapshots': {
|
||||
'ok': nested_ok or not nested_needed,
|
||||
'nested_snapshots': {
|
||||
'ok': (not nested_needed) or nested_ok,
|
||||
'active': nested_needed,
|
||||
'detail': nested_detail,
|
||||
},
|
||||
'middlewared.rclone.remote.b2': {
|
||||
'ok': b2_ok,
|
||||
'detail': 'patched on disk in overlay at boot' if b2_ok else providers_detail,
|
||||
},
|
||||
'middlewared.plugins.cloud_backup.restic': {
|
||||
'ok': restic_ok,
|
||||
'detail': 'patched on disk in overlay at boot' if restic_ok else providers_detail,
|
||||
},
|
||||
'middlewared.plugins.cloud.nested_snapshot': {
|
||||
'ok': nested_ok,
|
||||
'detail': nested_detail,
|
||||
},
|
||||
}
|
||||
payload = {'patched_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'patches': patches}
|
||||
tmp = status_path + '.tmp'
|
||||
@@ -566,18 +565,34 @@ try:
|
||||
except OSError as e:
|
||||
print(f'WARNING: Could not write hook_status.json: {e}')
|
||||
|
||||
# Exit 0 only if every module that is still NEEDED applied cleanly. A module that
|
||||
# was skipped (superseded or opt-out) is not a failure.
|
||||
_providers_done = (not providers_needed) or (b2_ok and restic_ok)
|
||||
# Exit code tells the caller whether a middlewared restart is still worth doing:
|
||||
#
|
||||
# 0 every module that was needed applied cleanly
|
||||
# 2 PARTIAL -- one module failed but another landed, so there IS something new
|
||||
# on disk waiting to be loaded
|
||||
# 1 nothing landed; a restart would accomplish nothing
|
||||
#
|
||||
# Collapsing 2 into 1 would mean a failing providers patch suppresses the restart
|
||||
# that a freshly-applied nested patch needs, leaving it on disk and never loaded.
|
||||
_providers_done = (not providers_needed) or bool(b2_ok and restic_ok)
|
||||
_nested_done = (not nested_needed) or nested_ok
|
||||
sys.exit(0 if (_providers_done and _nested_done) else 1)
|
||||
_landed = (providers_needed and b2_ok and restic_ok) or (nested_needed and nested_ok)
|
||||
|
||||
if _providers_done and _nested_done:
|
||||
sys.exit(0)
|
||||
sys.exit(2 if _landed else 1)
|
||||
PYEOF
|
||||
then
|
||||
_backend_ok=1
|
||||
else
|
||||
# Individual results already printed above; exit code 1 means at least
|
||||
# one module that was still needed failed to apply.
|
||||
_backend_ok=0
|
||||
_rc=$?
|
||||
if [ "$_rc" = "2" ]; then
|
||||
# One module failed, but another was applied and still needs loading.
|
||||
_backend_ok=1
|
||||
echo "WARNING: a module failed to apply; the other landed and will be loaded."
|
||||
else
|
||||
_backend_ok=0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -625,15 +640,17 @@ fi
|
||||
|
||||
echo "--- deferred restart ---"
|
||||
|
||||
# Restart when ANY still-needed backend module landed. Keying this off the
|
||||
# providers module alone would skip the restart on a box where B2 has gone native
|
||||
# but the nested module was freshly patched — leaving it on disk and never loaded.
|
||||
# Restart when ANY still-needed backend module landed (_backend_ok, incl. the
|
||||
# partial case). Keying this off the providers module alone would skip the restart
|
||||
# on a box where B2 has gone native but the nested module was freshly patched —
|
||||
# leaving it on disk and never loaded.
|
||||
#
|
||||
# "No module active at all" cannot reach here: that is the kill-switch branch
|
||||
# above, which exits.
|
||||
if ! grep -aq middlewared "/proc/$PPID/cmdline" 2>/dev/null; then
|
||||
echo "Manual run (parent is not middlewared) — no restart scheduled."
|
||||
elif [ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]; then
|
||||
echo "No backend module active — no restart scheduled (nothing new to load)."
|
||||
elif [ "${_backend_ok:-0}" != "1" ]; then
|
||||
echo "Backend patch incomplete — no restart scheduled (nothing new to load)."
|
||||
elif [ "$_backend_ok" != "1" ]; then
|
||||
echo "Nothing landed on disk — no restart scheduled (nothing new to load)."
|
||||
else
|
||||
# A failed unit from an earlier attempt this boot would block systemd-run.
|
||||
systemctl reset-failed truecloud-mw-restart.service 2>/dev/null
|
||||
|
||||
+13
-1
@@ -127,13 +127,21 @@ def cmd_verify():
|
||||
print(f"Hook status (recorded at {status.get('patched_at', 'unknown')})")
|
||||
print()
|
||||
all_ok = True
|
||||
any_active = False
|
||||
for module, info in status.get("patches", {}).items():
|
||||
ok = info.get("ok", False)
|
||||
# A module can be inactive because TrueNAS now does it natively, or
|
||||
# because it is opt-in and switched off. Neither is a failure.
|
||||
active = info.get("active", True)
|
||||
label = "OK " if ok else "FAIL"
|
||||
if ok and not active:
|
||||
label = "SKIP"
|
||||
detail = f" — {info['detail']}" if info.get("detail") else ""
|
||||
print(f" [{label}] {module}{detail}")
|
||||
if not ok:
|
||||
all_ok = False
|
||||
if active:
|
||||
any_active = True
|
||||
|
||||
# The disk status alone can false-positive: at boot the files are patched
|
||||
# while middlewared is already running with the stock modules imported.
|
||||
@@ -146,7 +154,11 @@ def cmd_verify():
|
||||
mw_start = _middlewared_start_epoch()
|
||||
|
||||
proc_stale = False
|
||||
if patched_epoch is None or mw_start is None:
|
||||
if not any_active:
|
||||
# Nothing is patched into middlewared, so whether it restarted since is
|
||||
# irrelevant -- there is nothing for it to have loaded.
|
||||
print(" [-- ] running middlewared process — no active module; nothing to load")
|
||||
elif patched_epoch is None or mw_start is None:
|
||||
print(" [?? ] running middlewared process — could not compare start time;")
|
||||
print(" the results above reflect the on-disk state only")
|
||||
elif mw_start + 2 < patched_epoch:
|
||||
|
||||
+68
-14
@@ -60,10 +60,10 @@ import stat
|
||||
import subprocess
|
||||
|
||||
__all__ = [
|
||||
"ACTIVE",
|
||||
"STAGING_BASE",
|
||||
"StagingError",
|
||||
"apply_plan",
|
||||
"cleanup_all",
|
||||
"cleanup_task",
|
||||
"current_mounts_under",
|
||||
"delete_snapshot_tree",
|
||||
@@ -79,9 +79,10 @@ __all__ = [
|
||||
#: Where staging trees are assembled. tmpfs; bind mounts consume no space.
|
||||
STAGING_BASE = "/run/truecloud-nested"
|
||||
|
||||
#: staging_root -> zfs snapshot name. A cache; the sidecar file is the source of
|
||||
#: truth, so that a middlewared restart cannot orphan a snapshot.
|
||||
ACTIVE: dict[str, str] = {}
|
||||
# Which snapshot a staging tree pins is recorded ONLY in the sidecar file, never
|
||||
# also in memory. An in-process dict would be a second source of truth that a
|
||||
# middlewared restart silently empties -- and it is exactly the restart case that
|
||||
# must not orphan a 250-snapshot tree. One record, on disk, or none.
|
||||
|
||||
|
||||
class StagingError(Exception):
|
||||
@@ -459,8 +460,6 @@ async def stage_nested(middleware, path, snapshot, base_dataset, base_mountpoint
|
||||
await middleware.run_in_thread(_remove_sidecar, staging_root)
|
||||
raise
|
||||
|
||||
ACTIVE[staging_root] = snapshot
|
||||
|
||||
if logger:
|
||||
logger.info(
|
||||
"truecloud-patch: staged %d dataset(s) from %s at %s",
|
||||
@@ -475,12 +474,7 @@ async def cleanup_task(middleware, task_name, logger=None):
|
||||
Safe to call unconditionally: a no-op when the task was never staged.
|
||||
"""
|
||||
staging_root = staging_root_for(task_name)
|
||||
sidecar = sidecar_for(staging_root)
|
||||
|
||||
snapshot = ACTIVE.pop(staging_root, None)
|
||||
if snapshot is None:
|
||||
# Sidecar survives a middlewared restart; ACTIVE does not.
|
||||
snapshot = _read_sidecar(staging_root)
|
||||
snapshot = _read_sidecar(staging_root)
|
||||
|
||||
if snapshot is None and not os.path.isdir(staging_root):
|
||||
return # never staged; nothing to do
|
||||
@@ -493,5 +487,65 @@ async def cleanup_task(middleware, task_name, logger=None):
|
||||
if snapshot is not None:
|
||||
await delete_snapshot_tree(middleware, snapshot, logger=logger)
|
||||
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(sidecar)
|
||||
_remove_sidecar(staging_root)
|
||||
|
||||
|
||||
# ── offline cleanup (uninstall.sh / recover.sh) ───────────────────────────────
|
||||
|
||||
|
||||
def cleanup_all(base=None, runner=_run, mounts_file="/proc/self/mounts",
|
||||
glob_fn=None, read_sidecar=_read_sidecar):
|
||||
"""Tear down every staging tree. Used by uninstall.sh and recover.sh.
|
||||
|
||||
Those scripts must work when middlewared is dead, so they cannot go through
|
||||
the async path -- but they must not reimplement the teardown either: the
|
||||
depth-ordering and lazy-umount fallback are fiddly, and a second copy in
|
||||
shell would be the untested one. This is the same tested code.
|
||||
|
||||
Returns ``(lines, errors)``: report lines to print, and unmount errors.
|
||||
"""
|
||||
import glob as _glob
|
||||
|
||||
base = base or STAGING_BASE
|
||||
glob_fn = glob_fn or _glob.glob
|
||||
lines = []
|
||||
|
||||
# Report orphaned snapshots BEFORE removing the sidecars that name them --
|
||||
# a sidecar is the only record that an interrupted run's snapshot tree (one
|
||||
# snapshot per descendant dataset) is still on disk.
|
||||
for sc in sorted(glob_fn(os.path.join(base, "*.snapshot"))):
|
||||
snap = read_sidecar(sc[: -len(".snapshot")])
|
||||
if snap:
|
||||
lines.append(f" NOTE: an interrupted backup left snapshot '{snap}' behind.")
|
||||
lines.append(f" Remove it and its children: zfs destroy -r '{snap}'")
|
||||
|
||||
mounts = current_mounts_under(base, mounts_file=mounts_file)
|
||||
if not mounts:
|
||||
lines.append(" None active.")
|
||||
for mp in mounts:
|
||||
lines.append(f" Unmounting: {mp}")
|
||||
|
||||
errors = teardown(base, runner=runner, mounts_file=mounts_file)
|
||||
for err in errors:
|
||||
lines.append(f" WARNING: could not unmount {err}")
|
||||
|
||||
if not errors:
|
||||
for sc in glob_fn(os.path.join(base, "*.snapshot")):
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(sc)
|
||||
with contextlib.suppress(OSError):
|
||||
os.rmdir(base)
|
||||
|
||||
return lines, errors
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "cleanup":
|
||||
_lines, _errors = cleanup_all()
|
||||
for _line in _lines:
|
||||
print(_line)
|
||||
sys.exit(1 if _errors else 0)
|
||||
print("usage: truecloud_nested.py cleanup", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
Reference in New Issue
Block a user