Fix post-merge audit findings; unify staging teardown

Audit
-----
- create_task.py verify failed on a DEFAULT install. hook_status.json emitted a
  per-file entry for the nested module with ok:false whenever the feature was
  switched off -- the default -- so verify printed [FAIL] and exited 1, right
  after the README tells users to run it. Status is now per MODULE with an
  `active` flag, and verify renders an inactive module as [SKIP].

- A partial apply suppressed the middlewared restart. The exit code conflated
  "nothing applied" with "one module applied, one failed", so a failing providers
  patch would prevent the restart that a freshly-applied nested patch needs,
  leaving it on disk and never loaded. Exit 2 now means partial and the restart
  still fires.

- The native-nested probe could never fire. It scanned crud.py for the guard
  message, but our own injected block quotes that message, so once applied the
  probe would always conclude the guard was still present. It now reads only the
  stock portion of the file.

- recover.sh did not unmount staging trees, so an emergency recovery left bind
  mounts pinning ZFS snapshots that could then never be destroyed.

- uninstall.sh deleted sidecar files without reading them. A sidecar is the only
  record that an interrupted run's snapshot tree is still on disk; both scripts
  now name the snapshot before clearing it.

- Removed a dead branch in the restart gate (unreachable: the kill switch exits).

Refactor
--------
- Staging teardown had been copy-pasted into uninstall.sh and recover.sh -- two
  untested shell copies of the fiddly depth-ordering and lazy-umount logic. Both
  now call `python3 patch/truecloud_nested.py cleanup`, so there is exactly one
  implementation and it is the one under test.

- Dropped the in-memory ACTIVE dict. The sidecar file was already the source of
  truth; a second in-process record could only desync -- and it is precisely the
  middlewared-restart case (which empties it) that must not orphan a snapshot
  tree. One record, on disk, or none.

Not done: the overlay-unmount loop is duplicated across apply.sh/uninstall.sh/
recover.sh. It is pre-existing, and apply.sh runs at PREINIT under a tight
timeout -- giving it a source dependency would trade 10 lines of duplication for
a boot-time failure mode.

74 tests, ruff and shellcheck clean.
This commit is contained in:
flan
2026-07-12 22:29:31 +00:00
parent c2e1976659
commit 24f1f2c648
9 changed files with 308 additions and 95 deletions
+83 -10
View File
@@ -21,9 +21,9 @@ import pytest
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "patch"))
from truecloud_nested import ( # noqa: E402
ACTIVE,
StagingError,
apply_plan,
cleanup_all,
cleanup_task,
current_mounts_under,
delete_snapshot_tree,
@@ -255,9 +255,6 @@ class TestDeleteSnapshotTree:
class TestStageNestedOrdering:
def setup_method(self):
ACTIVE.clear()
def test_sidecar_is_written_before_anything_is_mounted(self, tmp_path, monkeypatch):
# middlewared can die at any moment. If the snapshot were recorded only
# after apply_plan, a crash in that window would orphan a 160-snapshot
@@ -344,13 +341,10 @@ class TestStageNestedOrdering:
class TestCleanupTask:
def setup_method(self):
ACTIVE.clear()
def test_recovers_snapshot_from_sidecar_after_middlewared_restart(self, tmp_path,
monkeypatch):
# ACTIVE is in-process; a restart wipes it. The sidecar is the source of
# truth, otherwise the snapshot tree is orphaned forever.
# The sidecar is the ONLY record of the pinned snapshot, precisely so a
# middlewared restart cannot orphan the tree.
import truecloud_nested as tn
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
@@ -359,7 +353,6 @@ class TestCleanupTask:
with open(sidecar_for(root), "w", encoding="utf-8") as fh:
fh.write("Tap@snap")
ACTIVE.clear() # simulate the restart
mw = FakeMiddleware(["Tap@snap", "Tap/apps@snap"])
monkeypatch.setattr(tn, "teardown", lambda *_a, **_k: [])
@@ -489,6 +482,86 @@ class TestTeardown:
assert ["umount", "-l", ROOT] in runner.calls
class TestCleanupAll:
"""uninstall.sh and recover.sh call this instead of reimplementing teardown."""
def test_reports_orphan_snapshots_before_deleting_their_sidecars(self, tmp_path):
# The sidecar is the only record that an interrupted run's snapshot tree
# is still on disk. Deleting it without naming the snapshot orphans the
# whole tree silently.
base = tmp_path / "stage"
base.mkdir()
(base / "cloud_backup-5.snapshot").write_text("Tap@interrupted")
mounts_file = tmp_path / "mounts"
mounts_file.write_text("")
lines, errors = cleanup_all(
base=str(base), runner=FakeRunner(), mounts_file=str(mounts_file)
)
assert errors == []
assert any("Tap@interrupted" in ln for ln in lines)
assert any("zfs destroy -r" in ln for ln in lines)
# Sidecar cleared only after being reported.
assert not (base / "cloud_backup-5.snapshot").exists()
def test_unmounts_everything_under_the_base_deepest_first(self, tmp_path):
base = tmp_path / "stage"
base.mkdir()
mounts_file = tmp_path / "mounts"
mounts_file.write_text(
f"tmpfs {base} tmpfs rw 0 0\n"
f"tmpfs {base}/cloud_backup-5 tmpfs rw 0 0\n"
f"tmpfs {base}/cloud_backup-5/apps tmpfs rw 0 0\n"
)
runner = FakeRunner()
_lines, errors = cleanup_all(
base=str(base), runner=runner, mounts_file=str(mounts_file)
)
assert errors == []
order = [c[-1] for c in runner.calls if c[0] == "umount"]
assert order == [
f"{base}/cloud_backup-5/apps",
f"{base}/cloud_backup-5",
str(base),
]
def test_keeps_sidecars_when_an_unmount_failed(self, tmp_path):
# If a mount is stuck, the snapshot is still pinned — so the record of it
# must survive for the next run (or the operator) to act on.
base = tmp_path / "stage"
base.mkdir()
(base / "cloud_backup-5.snapshot").write_text("Tap@stuck")
mounts_file = tmp_path / "mounts"
mounts_file.write_text(f"tmpfs {base}/cloud_backup-5 tmpfs rw 0 0\n")
class Stuck(FakeRunner):
def __call__(self, cmd):
self.calls.append(cmd)
class R:
returncode = 32
stderr = "target is busy"
return R
_lines, errors = cleanup_all(
base=str(base), runner=Stuck(), mounts_file=str(mounts_file)
)
assert errors, "a stuck unmount must be reported"
assert (base / "cloud_backup-5.snapshot").exists()
def test_is_a_noop_on_a_clean_system(self, tmp_path):
mounts_file = tmp_path / "mounts"
mounts_file.write_text("")
lines, errors = cleanup_all(
base=str(tmp_path / "absent"), runner=FakeRunner(),
mounts_file=str(mounts_file),
)
assert errors == []
assert lines == [" None active."]
class TestCurrentMountsUnder:
def test_matches_only_the_staging_subtree(self, tmp_path):
mounts_file = tmp_path / "mounts"