flan
0d04c2cd1c
A reboot mid-backup orphaned the entire tree, permanently. The sidecar is the record of which snapshots a run pinned -- and /run is tmpfs. A reboot or crash between the recursive snapshot and its cleanup destroyed that record, leaving one snapshot per descendant dataset (250+ on a real pool) with nothing pointing at them. Nothing would ever have found them. gc_stale_snapshots() identifies leftovers by NAME, so it works when the record is gone. It runs after the sidecar reclaim -- the recorded path stays authoritative and the collector only mops up what the record lost. It deletes data on a name match, which is a weaker claim than a recorded fact, so the selection is a pure function with the harshest tests here. A snapshot is collected only if the name is exactly <dataset>@<task>-<YYYYMMDDHHMMSS>, it is not the current run's, NOTHING IS MOUNTED FROM IT (this, not the age guard, is what protects a concurrent backup), and it is over an hour old. Checked against the real pool: of 4728 snapshots including 2341 periodic ones, it selects exactly the orphans of the task being run and nothing else.
1052 lines
42 KiB
Python
1052 lines
42 KiB
Python
"""Tests for nested-dataset snapshot staging.
|
|
|
|
Two rules are under test above all else:
|
|
|
|
1. A tree that cannot be staged completely must fail LOUDLY. A silently
|
|
incomplete backup is the exact failure that stock TrueNAS's "no further
|
|
nesting" guard exists to prevent.
|
|
|
|
2. Every snapshot we cause to exist must be cleaned up. ``zfs.snapshot.delete``
|
|
is non-recursive by default and stock calls it with no options, so a
|
|
recursive snapshot would otherwise orphan one snapshot per descendant dataset
|
|
on EVERY run.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "patch"))
|
|
|
|
from truecloud_nested import ( # noqa: E402
|
|
StagingError,
|
|
apply_plan,
|
|
cleanup_all,
|
|
cleanup_task,
|
|
current_mounts_under,
|
|
delete_snapshot_tree,
|
|
plan_staging,
|
|
sidecar_for,
|
|
snapshot_tree_names,
|
|
staging_root_for,
|
|
teardown,
|
|
verify_staged,
|
|
)
|
|
|
|
SNAP = "cloud_backup-5-20260712030000"
|
|
ROOT = "/run/truecloud-nested/cloud_backup-5"
|
|
|
|
|
|
def ds(name, mountpoint, mounted="yes"):
|
|
return {
|
|
"name": name,
|
|
"properties": {
|
|
"mountpoint": {"value": mountpoint},
|
|
"mounted": {"value": mounted},
|
|
},
|
|
}
|
|
|
|
|
|
# Mirrors the real layout: apps are datasets, several with their own children.
|
|
DATASETS = [
|
|
ds("Tap", "/mnt/Tap"),
|
|
ds("Tap/apps", "/mnt/Tap/apps"),
|
|
ds("Tap/apps/lidarr", "/mnt/Tap/apps/lidarr"),
|
|
ds("Tap/apps/lidarr/config", "/mnt/Tap/apps/lidarr/config"),
|
|
ds("Tap/apps/immich", "/mnt/Tap/apps/immich"),
|
|
ds("Tap/apps/immich/pgdata", "/mnt/Tap/apps/immich/pgdata"),
|
|
]
|
|
|
|
|
|
def yes(_path):
|
|
return "ok"
|
|
|
|
|
|
def plan(datasets=DATASETS, base_dataset="Tap", base_mp="/mnt/Tap",
|
|
path="/mnt/Tap", probe=yes):
|
|
return plan_staging(base_dataset, base_mp, path, SNAP, datasets, ROOT, probe=probe)
|
|
|
|
|
|
class TestPlanStaging:
|
|
def test_stages_every_descendant_dataset(self):
|
|
mounts, skipped = plan()
|
|
assert skipped == []
|
|
assert len(mounts) == 6 # root + 5 descendants
|
|
assert mounts[0] == (f"/mnt/Tap/.zfs/snapshot/{SNAP}", ROOT)
|
|
|
|
by_target = {t: s for s, t in mounts}
|
|
assert by_target[f"{ROOT}/apps"] == f"/mnt/Tap/apps/.zfs/snapshot/{SNAP}"
|
|
assert by_target[f"{ROOT}/apps/immich/pgdata"] == (
|
|
f"/mnt/Tap/apps/immich/pgdata/.zfs/snapshot/{SNAP}"
|
|
)
|
|
|
|
def test_parents_are_mounted_before_children(self):
|
|
# A child's mountpoint dir only exists inside its parent's snapshot, so
|
|
# mounting a child first would fail.
|
|
mounts, _ = plan()
|
|
seen = set()
|
|
for _src, target in mounts:
|
|
if target != ROOT:
|
|
assert os.path.dirname(target) in seen
|
|
seen.add(target)
|
|
|
|
def test_backup_path_below_dataset_root(self):
|
|
mounts, _ = plan(base_dataset="Tap/apps", base_mp="/mnt/Tap/apps",
|
|
path="/mnt/Tap/apps")
|
|
assert mounts[0] == (f"/mnt/Tap/apps/.zfs/snapshot/{SNAP}", ROOT)
|
|
targets = [t for _s, t in mounts]
|
|
assert f"{ROOT}/lidarr" in targets
|
|
assert f"{ROOT}/apps/lidarr" not in targets
|
|
|
|
def test_base_dataset_is_not_a_descendant_of_itself(self):
|
|
mounts, _ = plan(datasets=[ds("Tap", "/mnt/Tap")])
|
|
assert len(mounts) == 1
|
|
|
|
|
|
class TestScoping:
|
|
def test_unrelated_datasets_are_ignored_silently(self):
|
|
# Regression: scoping by mountpoint first dragged in every
|
|
# mountpoint-less dataset on the box (all of Tank/.system/*), burying the
|
|
# warnings that actually matter.
|
|
noisy = DATASETS + [
|
|
ds("Tank/.system", "none"),
|
|
ds("Tank/.system/cores", "legacy"),
|
|
ds("Tank/backups", "/mnt/Tank/backups"),
|
|
]
|
|
mounts, skipped = plan(datasets=noisy)
|
|
assert len(mounts) == 6
|
|
assert skipped == [], "datasets outside the base dataset must not be reported"
|
|
|
|
def test_in_scope_dataset_without_mountpoint_is_reported(self):
|
|
datasets = DATASETS + [ds("Tap/apps/weird", "none")]
|
|
_mounts, skipped = plan(datasets=datasets)
|
|
assert ("Tap/apps/weird", "mountpoint is none") in skipped
|
|
|
|
def test_unmounted_dataset_is_skipped_but_never_silently(self):
|
|
datasets = DATASETS + [ds("Tap/apps/vault", "/mnt/Tap/apps/vault", mounted="no")]
|
|
mounts, skipped = plan(datasets=datasets)
|
|
assert f"{ROOT}/apps/vault" not in [t for _s, t in mounts]
|
|
assert ("Tap/apps/vault", "dataset is not mounted (locked/encrypted?)") in skipped
|
|
|
|
def test_descendant_mounted_outside_the_path_is_not_an_omission(self):
|
|
datasets = DATASETS + [ds("Tap/elsewhere", "/mnt/other")]
|
|
mounts, skipped = plan(datasets=datasets)
|
|
assert len(mounts) == 6
|
|
assert skipped == []
|
|
|
|
|
|
class TestSilentOmissionGuard:
|
|
"""The whole point of the feature. These are the tests that matter."""
|
|
|
|
@staticmethod
|
|
def _missing_pgdata(path):
|
|
return "missing" if "/mnt/Tap/apps/immich/pgdata/" in path else "ok"
|
|
|
|
@staticmethod
|
|
def _denied_pgdata(path):
|
|
if "/mnt/Tap/apps/immich/pgdata/" in path:
|
|
return "cannot be read (Permission denied)"
|
|
return "ok"
|
|
|
|
def test_missing_snapshot_on_descendant_raises(self):
|
|
with pytest.raises(StagingError, match="incomplete tree"):
|
|
plan(probe=self._missing_pgdata)
|
|
|
|
def test_error_names_the_offending_dataset(self):
|
|
with pytest.raises(StagingError, match="Tap/apps/immich/pgdata"):
|
|
plan(probe=self._missing_pgdata)
|
|
|
|
def test_missing_and_unreadable_are_reported_differently(self):
|
|
# os.path.isdir() collapses both into False, which would report a
|
|
# permission problem as "has no snapshot" and send you hunting for a
|
|
# snapshot that is sitting right there. Both abort -- but say which.
|
|
with pytest.raises(StagingError, match="has no snapshot"):
|
|
plan(probe=self._missing_pgdata)
|
|
with pytest.raises(StagingError, match="Permission denied"):
|
|
plan(probe=self._denied_pgdata)
|
|
|
|
|
|
class TestSnapshotTreeNames:
|
|
"""zfs.snapshot.delete is non-recursive; we must sweep children ourselves."""
|
|
|
|
ALL = [
|
|
"Tap@cloud_backup-5-20260712030000",
|
|
"Tap/apps@cloud_backup-5-20260712030000",
|
|
"Tap/apps/lidarr/config@cloud_backup-5-20260712030000",
|
|
"Tap@auto-2026-07-12_03-00", # unrelated periodic snapshot
|
|
"Tap/apps@cloud_backup-9-20260712030000", # another task
|
|
"Tank/backups@cloud_backup-5-20260712030000", # different pool
|
|
]
|
|
|
|
def test_returns_parent_and_all_children(self):
|
|
got = snapshot_tree_names("Tap@cloud_backup-5-20260712030000", self.ALL)
|
|
assert set(got) == {
|
|
"Tap@cloud_backup-5-20260712030000",
|
|
"Tap/apps@cloud_backup-5-20260712030000",
|
|
"Tap/apps/lidarr/config@cloud_backup-5-20260712030000",
|
|
}
|
|
|
|
def test_never_touches_periodic_or_other_tasks_or_other_pools(self):
|
|
got = snapshot_tree_names("Tap@cloud_backup-5-20260712030000", self.ALL)
|
|
assert "Tap@auto-2026-07-12_03-00" not in got
|
|
assert "Tap/apps@cloud_backup-9-20260712030000" not in got
|
|
assert "Tank/backups@cloud_backup-5-20260712030000" not in got
|
|
|
|
def test_malformed_snapshot_name_yields_nothing(self):
|
|
assert snapshot_tree_names("Tap", self.ALL) == []
|
|
|
|
|
|
class FakeMiddleware:
|
|
"""middlewared as this module actually uses it: `call_sync`, from a thread.
|
|
|
|
The module is synchronous on purpose -- see the orchestration note in
|
|
truecloud_nested.py. TrueNAS <= 25.10 reaches it through
|
|
`await middleware.run_in_thread(...)` and TrueNAS 26 calls it directly, but the
|
|
logic below the boundary is the same code either way, so it is tested once.
|
|
"""
|
|
|
|
def __init__(self, snapshots=None):
|
|
self.snapshots = list(snapshots or [])
|
|
self.calls = []
|
|
self.logger = None
|
|
|
|
def call_sync(self, method, *args):
|
|
self.calls.append((method, args))
|
|
if method == "zfs.snapshot.query":
|
|
return [{"name": n} for n in self.snapshots]
|
|
if method == "zfs.snapshot.delete":
|
|
name = args[0]
|
|
opts = args[1] if len(args) > 1 else {}
|
|
if name not in self.snapshots:
|
|
raise RuntimeError("does not exist")
|
|
if opts.get("recursive"):
|
|
# Real `zfs destroy -r` takes the parent and every child snapshot.
|
|
for n in snapshot_tree_names(name, list(self.snapshots)):
|
|
self.snapshots.remove(n)
|
|
else:
|
|
self.snapshots.remove(name)
|
|
return True
|
|
raise AssertionError(f"unexpected call {method}")
|
|
|
|
|
|
def stub_core(monkeypatch, tn, *, plan=None, order=None, plan_raises=None):
|
|
"""Replace the blocking core (plan/apply/verify/teardown) with recorders.
|
|
|
|
stage_nested calls these directly now, so they are patched by NAME rather than
|
|
intercepted at a `run_in_thread` boundary that no longer exists.
|
|
"""
|
|
def record(name, result):
|
|
def fn(*args, **kwargs):
|
|
if order is not None:
|
|
order.append(name)
|
|
if name == "plan_staging" and plan_raises is not None:
|
|
raise plan_raises
|
|
return result() if callable(result) else result
|
|
fn.__name__ = name
|
|
return fn
|
|
|
|
real_write = tn._write_sidecar
|
|
|
|
def write_sidecar(*args, **kwargs):
|
|
if order is not None:
|
|
order.append("_write_sidecar")
|
|
return real_write(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(tn, "_write_sidecar", write_sidecar)
|
|
monkeypatch.setattr(tn, "plan_staging", record("plan_staging", plan or ([], [])))
|
|
monkeypatch.setattr(tn, "apply_plan", record("apply_plan", True))
|
|
monkeypatch.setattr(tn, "verify_staged", record("verify_staged", True))
|
|
monkeypatch.setattr(tn, "teardown", record("teardown", []))
|
|
|
|
|
|
class TestDeleteSnapshotTree:
|
|
def test_deletes_parent_and_every_child(self):
|
|
mw = FakeMiddleware([
|
|
"Tap@snap", "Tap/apps@snap", "Tap/apps/lidarr@snap", "Tap@keepme",
|
|
])
|
|
delete_snapshot_tree(mw, "Tap@snap")
|
|
assert mw.snapshots == ["Tap@keepme"]
|
|
|
|
def test_is_idempotent_when_stock_already_removed_the_parent(self):
|
|
# Stock's finally can win the race once our mounts are released.
|
|
mw = FakeMiddleware(["Tap/apps@snap", "Tap/apps/lidarr@snap"])
|
|
delete_snapshot_tree(mw, "Tap@snap")
|
|
assert mw.snapshots == []
|
|
|
|
def test_uses_a_single_recursive_delete_not_252_individual_ones(self):
|
|
# 252 sequential deletes are slow AND not atomic: a run killed part-way
|
|
# through leaves exactly the orphans this function exists to prevent.
|
|
mw = FakeMiddleware(["Tap@snap", "Tap/apps@snap", "Tap/apps/lidarr@snap"])
|
|
delete_snapshot_tree(mw, "Tap@snap")
|
|
assert mw.snapshots == []
|
|
deletes = [a for m, a in mw.calls if m == "zfs.snapshot.delete"]
|
|
assert len(deletes) == 1, "should be ONE recursive call, not one per snapshot"
|
|
assert deletes[0][1] == {"recursive": True}
|
|
assert not [m for m, _a in mw.calls if m == "zfs.snapshot.query"], (
|
|
"no enumeration needed on the fast path"
|
|
)
|
|
|
|
def test_survives_recursive_and_query_failure_by_deleting_the_parent(self):
|
|
class Broken(FakeMiddleware):
|
|
def call_sync(self, method, *args):
|
|
if method == "zfs.snapshot.query":
|
|
raise RuntimeError("boom")
|
|
if method == "zfs.snapshot.delete" and len(args) > 1:
|
|
raise RuntimeError("recursive delete unavailable")
|
|
return super().call_sync(method, *args)
|
|
|
|
mw = Broken(["Tap@snap"])
|
|
delete_snapshot_tree(mw, "Tap@snap")
|
|
assert mw.snapshots == []
|
|
|
|
def test_leaves_unrelated_snapshots_alone_when_the_tree_is_gone(self):
|
|
mw = FakeMiddleware(["Tap@unrelated"])
|
|
delete_snapshot_tree(mw, "Tap@snap")
|
|
assert mw.snapshots == ["Tap@unrelated"]
|
|
|
|
|
|
class TestStageNestedOrdering:
|
|
def test_sidecar_is_written_before_anything_is_mounted(self, tmp_path, monkeypatch):
|
|
# middlewared can die at any moment. If the snapshot were recorded only
|
|
# after apply_plan, a crash in that window would orphan a 160-snapshot
|
|
# tree -- the precise failure the sidecar exists to prevent.
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
|
|
order = []
|
|
stub_core(monkeypatch, tn, order=order,
|
|
plan=([("/src", str(tmp_path / "cloud_backup-5"))], []))
|
|
|
|
tn.stage_nested(
|
|
FakeMiddleware(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap",
|
|
"cloud_backup-5", DATASETS,
|
|
)
|
|
|
|
assert order.index("_write_sidecar") < order.index("apply_plan")
|
|
|
|
def test_reclaims_the_snapshot_tree_left_by_a_crashed_run(self, tmp_path,
|
|
monkeypatch):
|
|
# teardown() reclaims the crashed run's MOUNTS, but nothing else would
|
|
# ever reclaim its SNAPSHOTS -- and we are about to overwrite the only
|
|
# record of them. One crash would orphan 160+ snapshots permanently.
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
|
|
root = tn.staging_root_for("cloud_backup-5")
|
|
os.makedirs(os.path.dirname(root), exist_ok=True)
|
|
with open(sidecar_for(root), "w", encoding="utf-8") as fh:
|
|
fh.write("Tap@old-crashed-run")
|
|
|
|
mw = FakeMiddleware(["Tap@old-crashed-run", "Tap/apps@old-crashed-run"])
|
|
stub_core(monkeypatch, tn, plan=([("/src", root)], []))
|
|
|
|
tn.stage_nested(
|
|
mw, "/mnt/Tap", "Tap@new", "Tap", "/mnt/Tap",
|
|
"cloud_backup-5", DATASETS,
|
|
)
|
|
|
|
assert mw.snapshots == [], "the crashed run's snapshot tree must be reclaimed"
|
|
|
|
def test_sidecar_is_KEPT_when_staging_fails(self, tmp_path, monkeypatch):
|
|
# The caller sweeps the snapshot tree on the way out, and anything still busy
|
|
# SURVIVES that sweep -- with the sidecar as its only record. Removing the
|
|
# sidecar here would orphan those snapshots permanently.
|
|
#
|
|
# The asymmetry is the point: a sidecar left behind when the tree is already
|
|
# gone costs one no-op delete on the next run; a sidecar removed while the tree
|
|
# still exists is unrecoverable.
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
|
|
root = tn.staging_root_for("cloud_backup-5")
|
|
stub_core(monkeypatch, tn, plan_raises=StagingError("boom"))
|
|
|
|
with pytest.raises(StagingError):
|
|
tn.stage_nested(
|
|
FakeMiddleware(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap",
|
|
"cloud_backup-5", DATASETS,
|
|
)
|
|
|
|
assert os.path.exists(sidecar_for(root)), (
|
|
"sidecar removed on staging failure — any snapshot the caller's sweep "
|
|
"cannot delete is now orphaned forever"
|
|
)
|
|
with open(sidecar_for(root), encoding="utf-8") as fh:
|
|
assert fh.read().strip() == "Tap@snap"
|
|
|
|
|
|
class TestCleanupTask:
|
|
def test_recovers_snapshot_from_sidecar_after_middlewared_restart(self, tmp_path,
|
|
monkeypatch):
|
|
# The sidecar is the ONLY record of the pinned snapshot, precisely so a
|
|
# middlewared restart cannot orphan the tree.
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
|
|
root = tn.staging_root_for("cloud_backup-5", base=str(tmp_path))
|
|
os.makedirs(root, exist_ok=True)
|
|
with open(sidecar_for(root), "w", encoding="utf-8") as fh:
|
|
fh.write("Tap@snap")
|
|
|
|
mw = FakeMiddleware(["Tap@snap", "Tap/apps@snap"])
|
|
monkeypatch.setattr(tn, "teardown", lambda *_a, **_k: [])
|
|
|
|
cleanup_task(mw, "cloud_backup-5")
|
|
|
|
assert mw.snapshots == []
|
|
assert not os.path.exists(sidecar_for(root))
|
|
|
|
def test_is_a_noop_when_never_staged(self, tmp_path, monkeypatch):
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path / "nope"))
|
|
mw = FakeMiddleware(["Tap@snap"])
|
|
cleanup_task(mw, "cloud_backup-5")
|
|
assert mw.calls == []
|
|
assert mw.snapshots == ["Tap@snap"]
|
|
|
|
|
|
class TestVerifyStaged:
|
|
"""Anti-regression guard: proves the staged tree is real before we back it up."""
|
|
|
|
def test_passes_when_every_target_is_mounted_and_root_non_empty(self):
|
|
mounts = [("/src", ROOT), ("/src/a", f"{ROOT}/a")]
|
|
assert verify_staged(mounts, ismount=lambda p: True, listdir=lambda p: ["apps"])
|
|
|
|
def test_raises_when_a_target_is_not_actually_mounted(self):
|
|
# This is the case that would produce a silently-empty backup.
|
|
mounts = [("/src", ROOT), ("/src/a", f"{ROOT}/a")]
|
|
with pytest.raises(StagingError, match="not a mountpoint"):
|
|
verify_staged(mounts, ismount=lambda p: p == ROOT, listdir=lambda p: ["apps"])
|
|
|
|
def test_raises_when_staging_root_is_empty(self):
|
|
with pytest.raises(StagingError, match="empty"):
|
|
verify_staged([("/src", ROOT)], ismount=lambda p: True, listdir=lambda p: [])
|
|
|
|
def test_raises_on_empty_plan(self):
|
|
with pytest.raises(StagingError):
|
|
verify_staged([])
|
|
|
|
|
|
class FakeRunner:
|
|
def __init__(self, fail_on=None):
|
|
self.fail_on = fail_on
|
|
self.calls = []
|
|
|
|
def __call__(self, cmd):
|
|
self.calls.append(cmd)
|
|
|
|
class R:
|
|
returncode = 0
|
|
stderr = ""
|
|
|
|
if self.fail_on and cmd[0] == "mount" and cmd[2] == self.fail_on:
|
|
R.returncode = 32
|
|
R.stderr = "mount failed"
|
|
return R
|
|
|
|
|
|
class TestApplyPlanRollback:
|
|
def test_rolls_back_mounts_when_one_fails(self, tmp_path):
|
|
# A half-built tree must never reach the backup tool.
|
|
root = str(tmp_path / "root")
|
|
mounts = [("/src", root), ("/src/a", root + "/a"), ("/src/b", root + "/b")]
|
|
|
|
runner = FakeRunner(fail_on="/src/b")
|
|
with pytest.raises(StagingError, match="bind-mount"):
|
|
apply_plan(mounts, runner=runner, isdir=lambda _p: True)
|
|
|
|
umounts = [c[-1] for c in runner.calls if c[0] == "umount"]
|
|
assert umounts == [root + "/a", root]
|
|
|
|
def test_raises_when_target_missing(self, tmp_path):
|
|
root = str(tmp_path / "root")
|
|
with pytest.raises(StagingError, match="does not exist"):
|
|
apply_plan(
|
|
[("/src", root), ("/src/a", root + "/a")],
|
|
runner=FakeRunner(),
|
|
isdir=lambda p: p == root,
|
|
)
|
|
|
|
|
|
class TestTeardown:
|
|
def test_unmounts_deepest_first(self, tmp_path):
|
|
mounts_file = tmp_path / "mounts"
|
|
mounts_file.write_text(
|
|
f"tmpfs {ROOT} tmpfs rw 0 0\n"
|
|
f"tmpfs {ROOT}/apps tmpfs rw 0 0\n"
|
|
f"tmpfs {ROOT}/apps/lidarr/config tmpfs rw 0 0\n"
|
|
f"tmpfs {ROOT}/apps/lidarr tmpfs rw 0 0\n"
|
|
"tmpfs /somewhere/else tmpfs rw 0 0\n"
|
|
)
|
|
runner = FakeRunner()
|
|
teardown(ROOT, runner=runner, mounts_file=str(mounts_file))
|
|
|
|
order = [c[-1] for c in runner.calls if c[0] == "umount"]
|
|
assert order == [
|
|
f"{ROOT}/apps/lidarr/config",
|
|
f"{ROOT}/apps/lidarr",
|
|
f"{ROOT}/apps",
|
|
ROOT,
|
|
]
|
|
assert "/somewhere/else" not in order
|
|
|
|
def test_is_idempotent_when_nothing_mounted(self, tmp_path):
|
|
mounts_file = tmp_path / "mounts"
|
|
mounts_file.write_text("tmpfs /somewhere/else tmpfs rw 0 0\n")
|
|
runner = FakeRunner()
|
|
assert teardown(ROOT, runner=runner, mounts_file=str(mounts_file)) == []
|
|
assert runner.calls == []
|
|
|
|
def test_falls_back_to_lazy_umount(self, tmp_path):
|
|
mounts_file = tmp_path / "mounts"
|
|
mounts_file.write_text(f"tmpfs {ROOT} tmpfs rw 0 0\n")
|
|
|
|
class Busy(FakeRunner):
|
|
def __call__(self, cmd):
|
|
self.calls.append(cmd)
|
|
|
|
class R:
|
|
returncode = 0 if "-l" in cmd else 32
|
|
stderr = "target is busy"
|
|
|
|
return R
|
|
|
|
runner = Busy()
|
|
assert teardown(ROOT, runner=runner, mounts_file=str(mounts_file)) == []
|
|
assert ["umount", "-l", ROOT] in runner.calls
|
|
|
|
|
|
class TestCleanupAll:
|
|
"""uninstall.sh and recover.sh call this instead of reimplementing teardown."""
|
|
|
|
def test_reports_orphan_snapshots_before_deleting_their_sidecars(self, tmp_path):
|
|
# The sidecar is the only record that an interrupted run's snapshot tree
|
|
# is still on disk. Deleting it without naming the snapshot orphans the
|
|
# whole tree silently.
|
|
base = tmp_path / "stage"
|
|
base.mkdir()
|
|
(base / "cloud_backup-5.snapshot").write_text("Tap@interrupted")
|
|
|
|
mounts_file = tmp_path / "mounts"
|
|
mounts_file.write_text("")
|
|
|
|
lines, errors = cleanup_all(
|
|
base=str(base), runner=FakeRunner(), mounts_file=str(mounts_file)
|
|
)
|
|
assert errors == []
|
|
assert any("Tap@interrupted" in ln for ln in lines)
|
|
assert any("zfs destroy -r" in ln for ln in lines)
|
|
# Sidecar cleared only after being reported.
|
|
assert not (base / "cloud_backup-5.snapshot").exists()
|
|
|
|
def test_unmounts_everything_under_the_base_deepest_first(self, tmp_path):
|
|
base = tmp_path / "stage"
|
|
base.mkdir()
|
|
mounts_file = tmp_path / "mounts"
|
|
mounts_file.write_text(
|
|
f"tmpfs {base} tmpfs rw 0 0\n"
|
|
f"tmpfs {base}/cloud_backup-5 tmpfs rw 0 0\n"
|
|
f"tmpfs {base}/cloud_backup-5/apps tmpfs rw 0 0\n"
|
|
)
|
|
runner = FakeRunner()
|
|
_lines, errors = cleanup_all(
|
|
base=str(base), runner=runner, mounts_file=str(mounts_file)
|
|
)
|
|
assert errors == []
|
|
order = [c[-1] for c in runner.calls if c[0] == "umount"]
|
|
assert order == [
|
|
f"{base}/cloud_backup-5/apps",
|
|
f"{base}/cloud_backup-5",
|
|
str(base),
|
|
]
|
|
|
|
def test_keeps_sidecars_when_an_unmount_failed(self, tmp_path):
|
|
# If a mount is stuck, the snapshot is still pinned — so the record of it
|
|
# must survive for the next run (or the operator) to act on.
|
|
base = tmp_path / "stage"
|
|
base.mkdir()
|
|
(base / "cloud_backup-5.snapshot").write_text("Tap@stuck")
|
|
mounts_file = tmp_path / "mounts"
|
|
mounts_file.write_text(f"tmpfs {base}/cloud_backup-5 tmpfs rw 0 0\n")
|
|
|
|
class Stuck(FakeRunner):
|
|
def __call__(self, cmd):
|
|
self.calls.append(cmd)
|
|
|
|
class R:
|
|
returncode = 32
|
|
stderr = "target is busy"
|
|
|
|
return R
|
|
|
|
_lines, errors = cleanup_all(
|
|
base=str(base), runner=Stuck(), mounts_file=str(mounts_file)
|
|
)
|
|
assert errors, "a stuck unmount must be reported"
|
|
assert (base / "cloud_backup-5.snapshot").exists()
|
|
|
|
def test_is_a_noop_on_a_clean_system(self, tmp_path):
|
|
mounts_file = tmp_path / "mounts"
|
|
mounts_file.write_text("")
|
|
lines, errors = cleanup_all(
|
|
base=str(tmp_path / "absent"), runner=FakeRunner(),
|
|
mounts_file=str(mounts_file),
|
|
)
|
|
assert errors == []
|
|
assert lines == [" None active."]
|
|
|
|
|
|
class TestCurrentMountsUnder:
|
|
def test_matches_only_the_staging_subtree(self, tmp_path):
|
|
mounts_file = tmp_path / "mounts"
|
|
# "cloud_backup-50" must NOT match "cloud_backup-5".
|
|
mounts_file.write_text(
|
|
f"tmpfs {ROOT} tmpfs rw 0 0\n"
|
|
"tmpfs /run/truecloud-nested/cloud_backup-50 tmpfs rw 0 0\n"
|
|
)
|
|
assert current_mounts_under(ROOT, mounts_file=str(mounts_file)) == [ROOT]
|
|
|
|
|
|
class TestStagingRootFor:
|
|
def test_stable_per_task(self):
|
|
assert staging_root_for("cloud_backup-5") == "/run/truecloud-nested/cloud_backup-5"
|
|
|
|
def test_sanitises_path_separators(self):
|
|
assert "/" not in staging_root_for("evil/name").rsplit("/", 1)[-1]
|
|
|
|
@pytest.mark.parametrize("name", ["..", ".", "...", "/", ""])
|
|
def test_dot_components_cannot_escape_the_staging_base(self, name):
|
|
# os.path.join(BASE, "..") normalises to /run — teardown would rmdir it.
|
|
root = staging_root_for(name)
|
|
assert os.path.normpath(root).startswith("/run/truecloud-nested/")
|
|
|
|
|
|
class TestZfsAutomountKeepsSnapshotsBusy:
|
|
""""dataset is busy" is EXPECTED, TRANSIENT, and used to orphan snapshots forever.
|
|
|
|
Reading `<dataset>/.zfs/snapshot/<snap>/` makes ZFS **automount** that snapshot,
|
|
and it stays mounted for zfs_expire_snapshot seconds (300 by default) after the
|
|
last access. teardown() unmounts OUR bind mounts, but not the automount underneath
|
|
-- so `zfs destroy` refuses with EBUSY for everything restic read recently.
|
|
|
|
Observed on a real 256-snapshot tree: 253 swept cleanly, and the 3 datasets restic
|
|
had touched last failed with "dataset is busy". cleanup_task then removed the
|
|
sidecar anyway, so nothing would ever reclaim them. A few snapshots leaked per run,
|
|
forever.
|
|
"""
|
|
|
|
MOUNTS = (
|
|
"tmpfs /run tmpfs rw 0 0\n"
|
|
"Tap/apps/prometheus /mnt/Tap/apps/prometheus/.zfs/snapshot/snap1 zfs ro 0 0\n"
|
|
"Tap/apps/standing/data /mnt/Tap/apps/standing/data/.zfs/snapshot/snap1 zfs ro 0 0\n"
|
|
"Tap /mnt/Tap/.zfs/snapshot/snap1 zfs ro 0 0\n"
|
|
"Tap/other /mnt/Tap/other/.zfs/snapshot/OTHER zfs ro 0 0\n"
|
|
)
|
|
|
|
def _mounts_file(self, tmp_path):
|
|
p = tmp_path / "mounts"
|
|
p.write_text(self.MOUNTS)
|
|
return str(p)
|
|
|
|
def test_it_finds_the_automounts_for_this_snapshot_only(self, tmp_path):
|
|
import truecloud_nested as tn
|
|
found = tn.snapdir_automounts("snap1", mounts_file=self._mounts_file(tmp_path))
|
|
assert "/mnt/Tap/other/.zfs/snapshot/OTHER" not in found
|
|
assert len(found) == 3
|
|
|
|
def test_deepest_first(self, tmp_path):
|
|
# A child's automount must be released before its parent's.
|
|
import truecloud_nested as tn
|
|
found = tn.snapdir_automounts("snap1", mounts_file=self._mounts_file(tmp_path))
|
|
assert found[-1] == "/mnt/Tap/.zfs/snapshot/snap1"
|
|
|
|
def test_release_snapdirs_unmounts_them(self, tmp_path):
|
|
import truecloud_nested as tn
|
|
called = []
|
|
|
|
class R:
|
|
returncode = 0
|
|
stderr = ""
|
|
|
|
def runner(cmd):
|
|
called.append(cmd)
|
|
return R()
|
|
|
|
errs = tn.release_snapdirs("snap1", runner=runner,
|
|
mounts_file=self._mounts_file(tmp_path))
|
|
assert errs == []
|
|
assert all(c[0] == "umount" for c in called)
|
|
assert len(called) == 3
|
|
|
|
|
|
class BusyMiddleware(FakeMiddleware):
|
|
"""Deletes fail with EBUSY until `busy_until_attempt` passes -- like a ZFS
|
|
automount expiring."""
|
|
|
|
def __init__(self, snapshots, busy, busy_for=2):
|
|
super().__init__(snapshots)
|
|
self.busy = set(busy)
|
|
self.busy_for = busy_for
|
|
self.attempts = 0
|
|
|
|
def call_sync(self, method, *args):
|
|
if method == "zfs.snapshot.delete":
|
|
name = args[0]
|
|
opts = args[1] if len(args) > 1 else {}
|
|
if opts.get("recursive"):
|
|
raise RuntimeError("cannot destroy snapshot: dataset is busy")
|
|
if name in self.busy:
|
|
self.attempts += 1
|
|
if self.attempts <= self.busy_for * len(self.busy):
|
|
raise RuntimeError(f"cannot destroy '{name}': dataset is busy")
|
|
return super().call_sync(method, *args)
|
|
|
|
|
|
class TestDeleteRetriesAndReportsSurvivors:
|
|
def test_a_transient_busy_is_retried_and_wins(self, monkeypatch):
|
|
import truecloud_nested as tn
|
|
monkeypatch.setattr(tn, "release_snapdirs", lambda *a, **k: [])
|
|
|
|
mw = BusyMiddleware(
|
|
["Tap@snap", "Tap/apps@snap", "Tap/apps/prometheus@snap"],
|
|
busy=["Tap/apps/prometheus@snap"], busy_for=1,
|
|
)
|
|
survivors = tn.delete_snapshot_tree(mw, "Tap@snap", sleep=lambda _s: None)
|
|
assert survivors == []
|
|
assert mw.snapshots == []
|
|
|
|
def test_a_permanently_busy_snapshot_is_REPORTED_not_swallowed(self, monkeypatch):
|
|
import truecloud_nested as tn
|
|
monkeypatch.setattr(tn, "release_snapdirs", lambda *a, **k: [])
|
|
|
|
mw = BusyMiddleware(
|
|
["Tap@snap", "Tap/apps/prometheus@snap"],
|
|
busy=["Tap/apps/prometheus@snap"], busy_for=99,
|
|
)
|
|
survivors = tn.delete_snapshot_tree(mw, "Tap@snap", sleep=lambda _s: None)
|
|
assert survivors == ["Tap/apps/prometheus@snap"]
|
|
assert mw.snapshots == ["Tap/apps/prometheus@snap"]
|
|
|
|
def test_the_automounts_are_released_before_deleting(self, monkeypatch):
|
|
import truecloud_nested as tn
|
|
order = []
|
|
monkeypatch.setattr(tn, "release_snapdirs",
|
|
lambda name, **k: order.append(("release", name)) or [])
|
|
mw = FakeMiddleware(["Tap@snap"])
|
|
real = mw.call_sync
|
|
|
|
def spy(method, *args):
|
|
order.append((method, args[0] if args else None))
|
|
return real(method, *args)
|
|
|
|
mw.call_sync = spy
|
|
tn.delete_snapshot_tree(mw, "Tap@snap", sleep=lambda _s: None)
|
|
assert order[0] == ("release", "snap"), order
|
|
|
|
|
|
class TestSidecarSurvivesAnIncompleteSweep:
|
|
def test_the_sidecar_is_KEPT_when_snapshots_could_not_be_deleted(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
# It is the ONLY record those snapshots exist. Removing it orphans them
|
|
# permanently -- which is exactly what happened on the real box.
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
|
|
monkeypatch.setattr(tn, "release_snapdirs", lambda *a, **k: [])
|
|
root = tn.staging_root_for("cloud_backup-5")
|
|
os.makedirs(root, exist_ok=True)
|
|
with open(sidecar_for(root), "w", encoding="utf-8") as fh:
|
|
fh.write("Tap@snap")
|
|
|
|
mw = BusyMiddleware(["Tap@snap", "Tap/apps/prometheus@snap"],
|
|
busy=["Tap/apps/prometheus@snap"], busy_for=99)
|
|
monkeypatch.setattr(tn, "delete_snapshot_tree",
|
|
lambda m, s, logger=None: ["Tap/apps/prometheus@snap"])
|
|
|
|
tn.cleanup_task(mw, "cloud_backup-5")
|
|
assert os.path.exists(sidecar_for(root)), (
|
|
"sidecar removed despite survivors — they are now orphaned forever"
|
|
)
|
|
|
|
def test_the_sidecar_is_removed_on_a_clean_sweep(self, tmp_path, monkeypatch):
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
|
|
root = tn.staging_root_for("cloud_backup-5")
|
|
os.makedirs(root, exist_ok=True)
|
|
with open(sidecar_for(root), "w", encoding="utf-8") as fh:
|
|
fh.write("Tap@snap")
|
|
|
|
monkeypatch.setattr(tn, "delete_snapshot_tree", lambda m, s, logger=None: [])
|
|
tn.cleanup_task(FakeMiddleware(), "cloud_backup-5")
|
|
assert not os.path.exists(sidecar_for(root))
|
|
|
|
|
|
class TestTheSidecarCarriesEveryPendingTree:
|
|
"""The sidecar holds a LIST, and that is a bug fix, not a generalisation.
|
|
|
|
It used to hold ONE snapshot. So a run that reclaimed an older tree, FAILED to
|
|
finish reclaiming it, and then recorded its own snapshot would **overwrite the only
|
|
record of the survivor** — orphaning it permanently, via the exact code written to
|
|
prevent orphans.
|
|
|
|
Observed live: a snapshot survived one run; the next run's reclaim also failed
|
|
(ZFS's 300s automount window had not elapsed, because the two runs were minutes
|
|
apart); the record was overwritten; the snapshot was orphaned for good.
|
|
"""
|
|
|
|
def test_round_trips_a_list(self, tmp_path):
|
|
import truecloud_nested as tn
|
|
root = str(tmp_path / "cloud_backup-5")
|
|
tn._write_sidecar(root, ["Tap@a", "Tap@b"])
|
|
assert tn._read_sidecar(root) == ["Tap@a", "Tap@b"]
|
|
|
|
def test_reads_the_old_single_line_format(self, tmp_path):
|
|
# Boxes upgrading from an older version have a one-line sidecar on disk.
|
|
import truecloud_nested as tn
|
|
root = str(tmp_path / "cloud_backup-5")
|
|
os.makedirs(os.path.dirname(sidecar_for(root)), exist_ok=True)
|
|
with open(sidecar_for(root), "w", encoding="utf-8") as fh:
|
|
fh.write("Tap@legacy")
|
|
assert tn._read_sidecar(root) == ["Tap@legacy"]
|
|
|
|
def test_a_failed_reclaim_is_carried_forward_not_overwritten(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
# THE bug. stage_nested reclaims an old tree, cannot finish, then records its
|
|
# own snapshot -- the survivor must still be in the sidecar afterwards.
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
|
|
root = tn.staging_root_for("cloud_backup-5")
|
|
os.makedirs(os.path.dirname(root), exist_ok=True)
|
|
tn._write_sidecar(root, ["Tap@old"])
|
|
|
|
# The reclaim of Tap@old leaves one snapshot behind (still busy).
|
|
monkeypatch.setattr(
|
|
tn, "delete_snapshot_tree",
|
|
lambda m, s, logger=None: ["Tap/apps/x@old"] if s == "Tap@old" else [],
|
|
)
|
|
stub_core(monkeypatch, tn, plan=([("/src", root)], []))
|
|
|
|
tn.stage_nested(FakeMiddleware(), "/mnt/Tap", "Tap@new", "Tap", "/mnt/Tap",
|
|
"cloud_backup-5", DATASETS)
|
|
|
|
recorded = tn._read_sidecar(root)
|
|
assert "Tap/apps/x@old" in recorded, (
|
|
"the failed reclaim's survivor was dropped — orphaned forever"
|
|
)
|
|
assert "Tap@new" in recorded, "our own snapshot must also be recorded"
|
|
|
|
def test_cleanup_sweeps_every_pending_tree_and_records_only_survivors(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
|
|
root = tn.staging_root_for("cloud_backup-5")
|
|
os.makedirs(root, exist_ok=True)
|
|
tn._write_sidecar(root, ["Tap@old", "Tap@new"])
|
|
|
|
swept = []
|
|
|
|
def fake_delete(m, s, logger=None):
|
|
swept.append(s)
|
|
return ["Tap/apps/x@new"] if s == "Tap@new" else []
|
|
|
|
monkeypatch.setattr(tn, "delete_snapshot_tree", fake_delete)
|
|
tn.cleanup_task(FakeMiddleware(), "cloud_backup-5")
|
|
|
|
assert swept == ["Tap@old", "Tap@new"], "both pending trees must be swept"
|
|
# Only the SURVIVOR is written back -- re-recording Tap@old would make every
|
|
# future run re-sweep a tree that is already gone.
|
|
assert tn._read_sidecar(root) == ["Tap/apps/x@new"]
|
|
|
|
def test_a_fully_clean_sweep_removes_the_sidecar(self, tmp_path, monkeypatch):
|
|
import truecloud_nested as tn
|
|
|
|
monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
|
|
root = tn.staging_root_for("cloud_backup-5")
|
|
os.makedirs(root, exist_ok=True)
|
|
tn._write_sidecar(root, ["Tap@a", "Tap@b"])
|
|
monkeypatch.setattr(tn, "delete_snapshot_tree", lambda m, s, logger=None: [])
|
|
|
|
tn.cleanup_task(FakeMiddleware(), "cloud_backup-5")
|
|
assert not os.path.exists(sidecar_for(root))
|
|
|
|
def test_cleanup_all_reports_each_pending_snapshot_on_its_own_line(self, tmp_path):
|
|
# It formats them for a human during uninstall. A list rendered into an
|
|
# f-string would print "['Tap@a', 'Tap@b']" at them.
|
|
import truecloud_nested as tn
|
|
root = str(tmp_path / "cloud_backup-5")
|
|
tn._write_sidecar(root, ["Tap@a", "Tap@b"])
|
|
|
|
lines, _errors = tn.cleanup_all(
|
|
base=str(tmp_path),
|
|
glob_fn=lambda _p: [sidecar_for(root)],
|
|
mounts_file=os.devnull,
|
|
)
|
|
notes = [ln for ln in lines if "left snapshot" in ln]
|
|
assert len(notes) == 2
|
|
assert "'Tap@a'" in notes[0] and "'Tap@b'" in notes[1]
|
|
assert "[" not in "".join(notes)
|
|
|
|
|
|
class TestGarbageCollectorSelection:
|
|
"""`stale_snapshot_names` DELETES DATA on a name match.
|
|
|
|
A name match is a weaker claim than a recorded fact, so every way it could be wrong
|
|
is a test. It exists because the sidecar — which IS a recorded fact — lives in /run,
|
|
which is tmpfs: a reboot mid-backup destroys it and orphans a 250-snapshot tree with
|
|
nothing left pointing at it. This is the only thing that would ever find those.
|
|
"""
|
|
|
|
import datetime as _dt
|
|
NOW = _dt.datetime(2026, 7, 14, 12, 0, 0, tzinfo=_dt.UTC)
|
|
CURRENT = "Tap@cloud_backup-5-20260714115900" # 1 minute ago
|
|
OLD = "Tap/apps/x@cloud_backup-5-20260713030000" # ~33 hours ago
|
|
|
|
def collect(self, names, **kw):
|
|
import truecloud_nested as tn
|
|
return tn.stale_snapshot_names(
|
|
"cloud_backup-5", self.CURRENT, names, self.NOW, **kw
|
|
)
|
|
|
|
def test_it_collects_our_own_leftovers(self):
|
|
assert self.collect([self.OLD]) == [self.OLD]
|
|
|
|
def test_it_NEVER_touches_the_current_run(self):
|
|
# Both the parent and its children share the current snapname.
|
|
names = [self.CURRENT, "Tap/apps/x@cloud_backup-5-20260714115900"]
|
|
assert self.collect(names) == []
|
|
|
|
def test_it_NEVER_touches_a_periodic_snapshot(self):
|
|
assert self.collect(["Tap/apps/x@auto-2026-07-13_03-00"]) == []
|
|
|
|
def test_it_NEVER_touches_a_human_made_snapshot(self):
|
|
assert self.collect(["Tap@before-i-broke-everything"]) == []
|
|
|
|
def test_it_NEVER_touches_another_TASK(self):
|
|
# cloud_backup-5 must not match cloud_backup-50. This is why the prefix
|
|
# carries the trailing dash.
|
|
assert self.collect(["Tap/apps/x@cloud_backup-50-20260713030000"]) == []
|
|
assert self.collect(["Tap/apps/x@cloud_backup-7-20260713030000"]) == []
|
|
|
|
def test_it_NEVER_touches_a_one_time_backup(self):
|
|
assert self.collect(["Tap@cloud_backup-onetime-20260713030000"]) == []
|
|
|
|
def test_it_NEVER_touches_a_snapshot_that_is_MOUNTED(self):
|
|
# An in-flight run pins its own snapshots. This — not the age heuristic — is
|
|
# what actually protects a concurrent backup.
|
|
assert self.collect([self.OLD], in_use={self.OLD}) == []
|
|
|
|
def test_it_NEVER_touches_a_snapshot_younger_than_the_minimum_age(self):
|
|
# Covers the seconds-long window between `zfs snapshot -r` and the mounts
|
|
# appearing, when a live run's snapshots look exactly like garbage.
|
|
young = "Tap/apps/x@cloud_backup-5-20260714113000" # 30 minutes ago
|
|
assert self.collect([young]) == []
|
|
assert self.collect([young], min_age=60) == [young]
|
|
|
|
def test_a_name_it_cannot_parse_is_left_alone(self):
|
|
assert self.collect(["Tap@cloud_backup-5-not-a-timestamp"]) == []
|
|
assert self.collect(["Tap@cloud_backup-5-"]) == []
|
|
|
|
def test_a_realistic_mixed_pool(self):
|
|
names = [
|
|
self.CURRENT, # ours, running
|
|
"Tap/apps/x@cloud_backup-5-20260714115900", # ours, running (child)
|
|
self.OLD, # ours, orphaned <-
|
|
"Tap/apps/y@cloud_backup-5-20260712030000", # ours, orphaned <-
|
|
"Tap/apps/x@auto-2026-07-13_03-00", # periodic
|
|
"Tap/apps/x@cloud_backup-7-20260713030000", # another task
|
|
"Tap@manual-keepme", # human
|
|
]
|
|
assert sorted(self.collect(names)) == sorted(
|
|
[self.OLD, "Tap/apps/y@cloud_backup-5-20260712030000"]
|
|
)
|
|
|
|
|
|
class TestMountedSnapshots:
|
|
def test_it_reads_snapshot_names_out_of_the_mount_table(self, tmp_path):
|
|
import truecloud_nested as tn
|
|
mounts = tmp_path / "mounts"
|
|
mounts.write_text(
|
|
"tmpfs /run tmpfs rw 0 0\n"
|
|
"Tap/apps/x@snap1 /run/truecloud-nested/t/apps/x zfs ro 0 0\n"
|
|
"Tap/apps/y@snap1 /mnt/Tap/apps/y/.zfs/snapshot/snap1 zfs ro 0 0\n"
|
|
"Tap/live /mnt/Tap/live zfs rw 0 0\n"
|
|
)
|
|
live = tn.mounted_snapshots(str(mounts))
|
|
assert live == {"Tap/apps/x@snap1", "Tap/apps/y@snap1"}
|
|
assert "Tap/live" not in live # a live dataset is not a snapshot
|
|
|
|
|
|
class TestGarbageCollectorExecution:
|
|
def test_it_deletes_the_stale_ones_and_nothing_else(self, monkeypatch, tmp_path):
|
|
import datetime as dt
|
|
import truecloud_nested as tn
|
|
|
|
mounts = tmp_path / "mounts"
|
|
mounts.write_text("")
|
|
now = dt.datetime(2026, 7, 14, 12, 0, 0, tzinfo=dt.UTC)
|
|
|
|
mw = FakeMiddleware([
|
|
"Tap@cloud_backup-5-20260714115900", # current run
|
|
"Tap/apps/x@cloud_backup-5-20260713030000", # orphan <-
|
|
"Tap/apps/x@auto-2026-07-13_03-00", # periodic
|
|
"Tap/apps/x@cloud_backup-7-20260713030000", # other task
|
|
])
|
|
remaining = tn.gc_stale_snapshots(
|
|
mw, "cloud_backup-5", "Tap@cloud_backup-5-20260714115900",
|
|
now=now, mounts_file=str(mounts),
|
|
)
|
|
assert remaining == []
|
|
assert mw.snapshots == [
|
|
"Tap@cloud_backup-5-20260714115900",
|
|
"Tap/apps/x@auto-2026-07-13_03-00",
|
|
"Tap/apps/x@cloud_backup-7-20260713030000",
|
|
]
|
|
|
|
def test_a_busy_orphan_is_reported_not_swallowed(self, monkeypatch, tmp_path):
|
|
import datetime as dt
|
|
import truecloud_nested as tn
|
|
|
|
mounts = tmp_path / "mounts"
|
|
mounts.write_text("")
|
|
now = dt.datetime(2026, 7, 14, 12, 0, 0, tzinfo=dt.UTC)
|
|
orphan = "Tap/apps/x@cloud_backup-5-20260713030000"
|
|
|
|
mw = BusyMiddleware(
|
|
["Tap@cloud_backup-5-20260714115900", orphan],
|
|
busy=[orphan], busy_for=99,
|
|
)
|
|
remaining = tn.gc_stale_snapshots(
|
|
mw, "cloud_backup-5", "Tap@cloud_backup-5-20260714115900",
|
|
now=now, mounts_file=str(mounts),
|
|
)
|
|
assert remaining == [orphan]
|
|
|
|
def test_it_collects_NOTHING_when_the_query_fails(self, tmp_path):
|
|
# Cannot enumerate => cannot know what is ours => delete nothing.
|
|
import datetime as dt
|
|
import truecloud_nested as tn
|
|
|
|
mounts = tmp_path / "mounts"
|
|
mounts.write_text("")
|
|
|
|
class Broken(FakeMiddleware):
|
|
def call_sync(self, method, *args):
|
|
if method == "zfs.snapshot.query":
|
|
raise RuntimeError("middleware is having a day")
|
|
return super().call_sync(method, *args)
|
|
|
|
assert tn.gc_stale_snapshots(
|
|
Broken(["Tap/apps/x@cloud_backup-5-20260713030000"]),
|
|
"cloud_backup-5", "Tap@cloud_backup-5-20260714115900",
|
|
now=dt.datetime(2026, 7, 14, 12, 0, 0, tzinfo=dt.UTC),
|
|
mounts_file=str(mounts),
|
|
) == []
|