flan
a54b4dd7f6
create_snapshot is module-global in plugins/cloud/snapshot.py, and cloud_sync.py imports it as well as cloud_backup/sync.py. So the wrapper sat in the path of every rclone/Storj CloudSync task with snapshot=true, and ran a zfs.dataset.query before concluding it had nothing to do -- a new failure mode for jobs that worked before this patch existed. Worse: a CloudSync task that ever got staged would never be torn down. The teardown is wired into cloud_backup's restic_backup finally, and CRUD_BLOCK deliberately leaves CloudSync's guard intact, so the bind mounts would pin the snapshot forever. The staging path now bails out unless the snapshot is named cloud_backup-*, before any middleware call. Separately: the async wrapper's finally dropped logger=, which the sync one passes. run_in_thread forwards **kwargs, so a cleanup that failed to unmount a bind mount or delete a snapshot tree logged nothing at all -- on the only platform anyone runs.
446 lines
19 KiB
Python
446 lines
19 KiB
Python
"""The *_BLOCK strings in apply.sh are Python source injected into middleware.
|
|
|
|
A syntax error in one of them would be appended to a live middlewared module and
|
|
break the box at boot. They are string literals, so nothing type-checks them --
|
|
these tests do.
|
|
"""
|
|
|
|
import ast
|
|
import os
|
|
import re
|
|
import textwrap
|
|
|
|
import pytest
|
|
|
|
APPLY_SH = os.path.join(os.path.dirname(__file__), "..", "patch", "apply.sh")
|
|
|
|
#: Every block that is actually injected into a middlewared module.
|
|
#:
|
|
#: The three nested blocks come in two flavours. TrueNAS <= 25.10 has an ASYNC
|
|
#: cloud_backup path; TrueNAS 26 rewrote it synchronous. apply.sh reads which one is
|
|
#: installed and injects the matching wrapper -- an `async def` on 26 would hand
|
|
#: sync.py a coroutine where it unpacks a tuple, and a plain `def` on 25.10 would
|
|
#: block the event loop. Both flavours must therefore be valid Python, always.
|
|
EXPECTED_BLOCKS = {
|
|
"B2_BLOCK",
|
|
"RESTIC_BLOCK",
|
|
"SNAPSHOT_ASYNC",
|
|
"SNAPSHOT_SYNC",
|
|
"CRUD_ASYNC",
|
|
"CRUD_SYNC",
|
|
"SYNC_ASYNC",
|
|
"SYNC_SYNC",
|
|
}
|
|
|
|
NESTED_BLOCKS = ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC", "CRUD_ASYNC", "CRUD_SYNC",
|
|
"SYNC_ASYNC", "SYNC_SYNC"]
|
|
|
|
|
|
def heredoc_source():
|
|
with open(APPLY_SH, encoding="utf-8") as fh:
|
|
src = fh.read()
|
|
m = re.search(r"<< 'PYEOF'\n(.*?)\nPYEOF", src, re.S)
|
|
assert m, "could not find the PYEOF heredoc in apply.sh"
|
|
return m.group(1)
|
|
|
|
|
|
def extract_blocks():
|
|
"""The blocks as apply.sh actually builds them.
|
|
|
|
EVALUATED, not read off as string literals: each nested block is a CORE
|
|
concatenated with a flavour-specific wrapper, so reading only `ast.Constant`
|
|
would silently return nothing for them -- a green suite over blocks nobody
|
|
checked. Assignments that need the runtime (argv, imports) simply fail to
|
|
evaluate and are skipped.
|
|
"""
|
|
tree = ast.parse(heredoc_source())
|
|
ns, blocks = {}, {}
|
|
for node in tree.body:
|
|
if not isinstance(node, ast.Assign):
|
|
continue
|
|
try:
|
|
value = eval( # noqa: S307 - our own shipped source, on purpose
|
|
compile(ast.Expression(node.value), "<blocks>", "eval"), {}, ns
|
|
)
|
|
except Exception:
|
|
continue
|
|
for tgt in node.targets:
|
|
if isinstance(tgt, ast.Name) and isinstance(value, str):
|
|
ns[tgt.id] = value
|
|
if tgt.id in EXPECTED_BLOCKS:
|
|
blocks[tgt.id] = value
|
|
return blocks
|
|
|
|
|
|
def _nested_native_detector():
|
|
"""The REAL native-nested probe, lifted out of apply.sh.
|
|
|
|
Extracted rather than reimplemented: a reimplementation would happily pass
|
|
while the shipped probe stayed broken, which is precisely the bug this guards.
|
|
"""
|
|
with open(APPLY_SH, encoding="utf-8") as fh:
|
|
sh = fh.read()
|
|
|
|
m = re.search(
|
|
r"^(\s*)_drop = str\.maketrans\(.*?\n\s*if 'nofurthernesting' not in "
|
|
r"stock_src\.translate\(_drop\):\n\s*result\['native_nested'\] = 'yes'",
|
|
sh, re.S | re.M,
|
|
)
|
|
assert m, "could not find the native-nested probe in apply.sh"
|
|
|
|
# The block lives inside a double-quoted shell string; undo bash's escaping.
|
|
body = m.group(0)
|
|
body = body.replace("\\\\", "\x00").replace('\\"', '"').replace("\x00", "\\")
|
|
body = textwrap.dedent(body)
|
|
|
|
def detect(stock_src):
|
|
ns = {"stock_src": stock_src, "result": {"native_nested": "no"}, "chr": chr}
|
|
exec(body, ns) # noqa: S102 - executing our own shipped code, on purpose
|
|
return ns["result"]["native_nested"]
|
|
|
|
return detect
|
|
|
|
|
|
def test_heredoc_itself_compiles():
|
|
compile(heredoc_source(), "apply.sh:PYEOF", "exec")
|
|
|
|
|
|
def test_all_expected_blocks_present():
|
|
assert set(extract_blocks()) == EXPECTED_BLOCKS
|
|
|
|
|
|
@pytest.mark.parametrize("name", sorted(EXPECTED_BLOCKS))
|
|
def test_injected_block_is_valid_python(name):
|
|
block = extract_blocks()[name]
|
|
compile(block, f"apply.sh:{name}", "exec")
|
|
|
|
|
|
@pytest.mark.parametrize("name", sorted(EXPECTED_BLOCKS))
|
|
def test_injected_block_carries_the_idempotency_marker(name):
|
|
# patch_file() truncates each target file at "\n# TRUECLOUD_PATCH" before
|
|
# re-appending, so every block must start with that marker or repeated runs
|
|
# would stack duplicate copies into the middleware module.
|
|
assert extract_blocks()[name].lstrip("\n").startswith("# TRUECLOUD_PATCH")
|
|
|
|
|
|
@pytest.mark.parametrize("name", NESTED_BLOCKS)
|
|
def test_nested_blocks_degrade_safely_without_the_module(name):
|
|
# If _truecloud_nested failed to install, every nested block must no-op.
|
|
# Critically this includes CRUD_BLOCK: relaxing the guard without the
|
|
# traversal in place would mean silently-empty backups.
|
|
block = extract_blocks()[name]
|
|
assert "_tc_nested = None" in block
|
|
assert "if _tc_nested is not None:" in block
|
|
|
|
|
|
class TestSnapshotLeak:
|
|
"""zfs.snapshot.delete is non-recursive and stock calls it with no options.
|
|
|
|
A recursive snapshot has one child per descendant dataset (160+ here), so
|
|
every path that creates one must also sweep the whole tree.
|
|
"""
|
|
|
|
def test_staging_failure_deletes_the_snapshot_tree(self):
|
|
# On a staging failure, sync.py's `snapshot, local_path = await
|
|
# create_snapshot(...)` never completes, so its local `snapshot` stays
|
|
# None and its finally deletes nothing. We must sweep it ourselves.
|
|
block = extract_blocks()["SNAPSHOT_ASYNC"]
|
|
assert "except Exception:" in block
|
|
assert "delete_snapshot_tree" in block
|
|
assert "raise" in block
|
|
|
|
def test_sync_block_cleans_up_on_every_path(self):
|
|
block = extract_blocks()["SYNC_ASYNC"]
|
|
assert "finally:" in block
|
|
assert "cleanup_task" in block
|
|
|
|
|
|
def test_crud_block_is_scoped_to_cloud_backup():
|
|
# cloudsync has no staging teardown wired in, so its guard must stay.
|
|
for name in ("CRUD_ASYNC", "CRUD_SYNC"):
|
|
assert '!= "cloud_backup"' in extract_blocks()[name]
|
|
|
|
|
|
class TestIndependentModules:
|
|
"""The two modules must retire independently.
|
|
|
|
TrueNAS may ship native B2 support long before (or after) it handles nested
|
|
datasets. A single all-or-nothing kill switch would silently take a
|
|
still-needed module down with the superseded one.
|
|
"""
|
|
|
|
def _sh(self):
|
|
with open(APPLY_SH, encoding="utf-8") as fh:
|
|
return fh.read()
|
|
|
|
def test_native_support_is_detected_per_module(self):
|
|
sh = self._sh()
|
|
assert "native_b2" in sh
|
|
assert "native_nested" in sh
|
|
assert "no further nesting" in sh, "nested native-support probe"
|
|
|
|
def test_kill_switch_only_when_both_modules_are_done(self):
|
|
sh = self._sh()
|
|
assert '[ "$_providers_needed" = "0" ] && [ "$_nested_needed" = "0" ]' in sh
|
|
# ...and that is the only place the kill switch is actually set. (Ignore
|
|
# comment lines, which mention the same path.)
|
|
code = [ln for ln in sh.splitlines() if not ln.lstrip().startswith("#")]
|
|
sets = [ln for ln in code if 'touch "$PATCH_DIR/disabled"' in ln]
|
|
assert len(sets) == 1, f"kill switch set in {len(sets)} places"
|
|
|
|
def test_each_module_is_gated_separately(self):
|
|
src = heredoc_source()
|
|
assert "if not providers_needed:" in src
|
|
assert "elif nested_native:" in src
|
|
|
|
def test_ui_patch_is_tied_to_the_providers_module(self):
|
|
# The UI change widens the credential dropdown; it is meaningless once B2
|
|
# is native, but must NOT be skipped merely because nested is off.
|
|
sh = self._sh()
|
|
i = sh.index("--- UI patch ---")
|
|
assert '[ "$_providers_needed" = "0" ]' in sh[i:i + 400]
|
|
|
|
def test_status_reports_an_inactive_module_as_ok(self):
|
|
# `create_task.py verify` fails if any patches[*].ok is false. An opt-in
|
|
# module that is switched off (the DEFAULT) must not report FAIL, or a
|
|
# stock install fails verification out of the box.
|
|
src = heredoc_source()
|
|
assert "'ok': (not nested_needed) or nested_ok" in src
|
|
assert "'ok': (not providers_needed) or bool(b2_ok and restic_ok)" in src
|
|
assert "'active': nested_needed" in src
|
|
|
|
def test_nested_native_probe_matches_the_real_wrapped_source(self):
|
|
"""Stock splits the guard message across adjacent string literals.
|
|
|
|
Python concatenates them at runtime, so the errmsg is contiguous -- but the
|
|
SOURCE never contains the whole phrase. A raw substring search finds
|
|
nothing, concludes iX removed the guard, and silently skips this module
|
|
forever. This is exactly what happened, and only a run against real
|
|
middlewared caught it.
|
|
"""
|
|
detect = _nested_native_detector()
|
|
|
|
# Verbatim shape from TrueNAS plugins/cloud/crud.py.
|
|
stock_wrapped = (
|
|
' verrors.add(f"{name}.snapshot", '
|
|
'"This option is only available for datasets that have no further "\n'
|
|
' "nesting")\n'
|
|
)
|
|
assert detect(stock_wrapped) == "no", "guard is present; must NOT report native"
|
|
|
|
# Same message on a single line — must also be detected.
|
|
assert detect('verrors.add(x, "... have no further nesting")\n') == "no"
|
|
|
|
# Single-quoted, three-way split — still the guard.
|
|
assert detect(
|
|
"verrors.add(x, 'This option is only available for '\n"
|
|
" 'datasets that have no further '\n"
|
|
" 'nesting')\n"
|
|
) == "no"
|
|
|
|
# Guard genuinely gone -> native support.
|
|
assert detect("def _validate(self):\n pass\n") == "yes"
|
|
|
|
def test_nested_native_probe_ignores_our_own_block(self):
|
|
# CRUD_BLOCK quotes the guard message, so scanning the whole file would
|
|
# find the string in our own patch and never detect native support.
|
|
sh = self._sh()
|
|
assert "split('\\n# TRUECLOUD_PATCH', 1)[0]" in sh
|
|
assert "no further nesting" in extract_blocks()["CRUD_ASYNC"], (
|
|
"if this ever stops being true, the probe comment is stale"
|
|
)
|
|
|
|
def test_restart_fires_when_any_needed_module_landed(self):
|
|
# Keying the restart off providers alone would leave a freshly-patched
|
|
# nested module on disk and never loaded on a native-B2 box.
|
|
sh = self._sh()
|
|
i = sh.index("--- deferred restart ---")
|
|
tail = sh[i:]
|
|
assert '_backend_ok' in tail
|
|
assert '"$_b2_ok"' not in tail
|
|
|
|
def test_partial_failure_still_schedules_the_restart(self):
|
|
# If providers fails but nested landed (or vice versa), something new IS
|
|
# on disk. Collapsing that into "nothing to do" would leave the module
|
|
# that succeeded permanently unloaded.
|
|
src = heredoc_source()
|
|
assert "sys.exit(2 if _landed else 1)" in src
|
|
assert "_landed = (providers_needed and b2_ok and restic_ok) or (nested_needed and nested_ok)" in src
|
|
|
|
sh = self._sh()
|
|
assert '_rc=$?' in sh
|
|
assert '[ "$_rc" = "2" ]' in sh
|
|
|
|
|
|
class TestOptIn:
|
|
"""Nested-snapshot support must be opt-in and must never self-enable."""
|
|
|
|
def test_heredoc_gates_on_the_opt_in_flag(self):
|
|
src = heredoc_source()
|
|
assert re.search(r"nested_enabled = sys\.argv\[\d+\] == \"1\"", src)
|
|
assert "if not nested_enabled:" in src
|
|
|
|
def test_apply_sh_reads_the_marker_file(self):
|
|
with open(APPLY_SH, encoding="utf-8") as fh:
|
|
sh = fh.read()
|
|
assert 'if [ -f "$PATCH_DIR/nested_snapshots_enabled" ]' in sh
|
|
assert '"$_NESTED_ENABLED"' in sh
|
|
|
|
def test_patching_is_skipped_entirely_when_disabled(self):
|
|
# The guard-relaxing crud.py patch must be inside the enabled branch.
|
|
src = heredoc_source()
|
|
gate = src.index("if not nested_needed:")
|
|
crud = src.index("patch_file(crud_py, _crud_block)")
|
|
assert gate < crud, "crud.py patch must sit inside the opt-in branch"
|
|
|
|
def test_disabling_REVERTS_the_patch_rather_than_merely_skipping_it(self):
|
|
"""Skipping is not disabling.
|
|
|
|
The overlay persists for the whole boot, so a patch applied by an earlier
|
|
run this boot is still on disk — and middlewared re-imports it on the
|
|
restart install.sh performs. Without an active revert,
|
|
`--disable-nested-snapshots` reports "disabled" while the feature keeps
|
|
running until the next reboot.
|
|
"""
|
|
src = heredoc_source()
|
|
# The implementation lives in patch/mw_patch.py (see test_mw_patch.py);
|
|
# apply.sh must import and actually call it.
|
|
assert "from mw_patch import patch_file, revert_nested" in src
|
|
gate = src.index("if not nested_needed:")
|
|
revert = src.index("reverted = revert_nested(")
|
|
patch = src.index("patch_file(crud_py, _crud_block)")
|
|
assert gate < revert < patch, "revert belongs in the not-needed branch"
|
|
|
|
def test_import_failure_skips_the_patch_rather_than_crashing(self):
|
|
# apply.sh runs at PREINIT. If mw_patch.py cannot be imported it must
|
|
# degrade to "middlewared starts stock", never take the boot down.
|
|
src = heredoc_source()
|
|
i = src.index("from mw_patch import")
|
|
tail = src[i:i + 400]
|
|
assert "except ImportError" in tail
|
|
assert "skipping backend patch" in tail
|
|
|
|
|
|
def test_guard_is_relaxed_only_after_traversal_is_installed():
|
|
# Ordering in apply.sh is a safety property: copy module -> patch snapshot.py
|
|
# -> patch sync.py -> patch crud.py. crud.py (which unlocks the feature) must
|
|
# come last, so a partial failure never leaves "guard removed, traversal gone".
|
|
src = heredoc_source()
|
|
order = [
|
|
src.index("shutil.copyfile(nested_src, nested_dst)"),
|
|
src.index("patch_file(snapshot_py, _snapshot_block)"),
|
|
src.index("patch_file(sync_path, _sync_block)"),
|
|
src.index("patch_file(crud_py, _crud_block)"),
|
|
]
|
|
assert order == sorted(order), "crud.py must be patched last"
|
|
|
|
|
|
class TestWrappersDoNotHardcodeStockArity:
|
|
"""iX changes the tail of these signatures between releases.
|
|
|
|
SYNC_BLOCK used to spell out `(middleware, job, cloud_backup, dry_run, rate_limit)`
|
|
and forward all five. But 24.10 and 25.04 declare only four -- `rate_limit` arrived
|
|
in 25.10 -- so every nested backup on those two releases raised
|
|
`TypeError: restic_backup() takes 4 positional arguments but 5 were given`.
|
|
It shipped broken and nothing noticed, because the compat check at the time only
|
|
asked whether the parameter NAMES still appeared somewhere in the signature.
|
|
|
|
Forwarding *args/**kwargs makes the wrapper indifferent to a trailing parameter
|
|
being added or dropped, which is the only part iX actually churns.
|
|
"""
|
|
|
|
def test_restic_backup_forwards_rather_than_naming_stock_params(self):
|
|
block = extract_blocks()["SYNC_ASYNC"]
|
|
assert "async def restic_backup(middleware, job, cloud_backup, *args, **kwargs)" in block
|
|
assert "_tc_orig_restic_backup(middleware, job, cloud_backup, *args, **kwargs)" in block
|
|
|
|
# Comments stripped: the block's own commentary explains the rate_limit
|
|
# history, and that must not be mistaken for the code re-declaring it.
|
|
code = "\n".join(
|
|
line for line in block.splitlines()
|
|
if not line.lstrip().startswith("#")
|
|
)
|
|
assert "rate_limit" not in code, (
|
|
"naming a trailing stock parameter re-introduces the arity bug"
|
|
)
|
|
|
|
|
|
class TestTheTwoNativeProbesCannotDrift:
|
|
"""The split-literal trick is implemented TWICE: inline in apply.sh's probe, and
|
|
as compat._squash. It has already caused one silent bug.
|
|
|
|
Stock middleware writes the guard as an implicitly-concatenated literal, so the
|
|
contiguous phrase never appears in the source. A naive search finds nothing,
|
|
concludes iX removed the guard, and reports "native" -- which means "retire the
|
|
module". That would disable nested snapshots on every box that depends on them.
|
|
|
|
apply.sh (runtime, on the box) and compat.py (static, in CI) must therefore agree
|
|
on every input, or one of them is wrong about whether to retire a module.
|
|
"""
|
|
|
|
CASES = [
|
|
# (crud.py source, expected native?)
|
|
("verrors.add('x', 'datasets that have no further nesting')", False),
|
|
# THE case: split across adjacent literals, as stock actually writes it.
|
|
("verrors.add('x', 'datasets that have no further '\n"
|
|
" 'nesting')", False),
|
|
('verrors.add("x", "no further "\n "nesting")', False),
|
|
# Guard genuinely gone -> iX implemented it -> native.
|
|
("verrors.add('x', 'some other validation entirely')", True),
|
|
("", True),
|
|
]
|
|
|
|
@pytest.mark.parametrize("src,expect_native", CASES)
|
|
def test_both_probes_agree(self, src, expect_native):
|
|
import sys as _sys
|
|
_sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
|
|
import compat
|
|
|
|
shipped = _nested_native_detector()(src)
|
|
assert (shipped == "yes") == expect_native, (
|
|
f"apply.sh's probe says native={shipped!r} for {src!r}"
|
|
)
|
|
|
|
path, phrase, native_when_present = compat.NATIVE_PROBES[compat.NESTED]
|
|
present = compat._squash(phrase) in compat._squash(src)
|
|
static_native = (present == native_when_present)
|
|
assert static_native == expect_native, (
|
|
f"compat.py says native={static_native} for {src!r}"
|
|
)
|
|
|
|
|
|
class TestOnlyOurOwnTasksAreTouched:
|
|
"""create_snapshot is module-global, and cloud_sync.py imports it too.
|
|
|
|
plugins/cloud/snapshot.py::create_snapshot is imported by BOTH
|
|
cloud_backup/sync.py and cloud_sync.py, so our wrapper sits in the path of every
|
|
rclone/Storj CloudSync task with snapshot=true -- tasks this patch has no business
|
|
touching. Two consequences, the second much worse than the first:
|
|
|
|
* every middleware call we add is a NEW failure mode for a job that worked
|
|
before we were installed;
|
|
* a staged CloudSync task would NEVER be torn down. The teardown is wired into
|
|
cloud_backup's restic_backup finally, and CRUD_BLOCK deliberately leaves
|
|
CloudSync's nesting guard intact -- so the bind mounts would pin the ZFS
|
|
snapshot forever.
|
|
|
|
cloud_backup names its snapshot "cloud_backup-<id>", cloud_sync "cloud_sync-<id>",
|
|
and stock's default is "cloud_task-onetime".
|
|
"""
|
|
|
|
@pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
|
|
def test_the_staging_path_is_gated_on_cloud_backup(self, name):
|
|
block = extract_blocks()[name]
|
|
assert 'if not name.startswith("cloud_backup"):' in block
|
|
|
|
@pytest.mark.parametrize("name", ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC"])
|
|
def test_the_bail_out_precedes_every_middleware_call(self, name):
|
|
# The point is to add NO new failure mode to a CloudSync task. If any
|
|
# middleware call happened before the bail-out, we would already have broken
|
|
# the thing we are trying not to touch.
|
|
block = extract_blocks()[name]
|
|
gate = block.index('if not name.startswith("cloud_backup"):')
|
|
for call in ("middleware.call_sync(", "_tc_nested.stage_nested(",
|
|
"_tc_nested.delete_snapshot_tree("):
|
|
assert gate < block.index(call), f"{call} runs before the cloud_backup gate"
|