"""Tests for the middlewared compatibility manifest. Two failure directions, and they are NOT symmetric: * a false **BROKEN** makes a module decline to apply on a box where it works. Worse, if both modules go quiet, apply.sh used to set a PERMANENT kill switch that only install.sh clears -- so a network blip or an innocent refactor could take a working box's B2 backups down until someone noticed by hand. * a false **OK** lets the patch inject into middleware it does not fit, which is a broken backup discovered at restore time. Both are tested. The `native` verdict gets its own scrutiny because it is the most dangerous thing this file can say -- it means "TrueNAS does this now, retire the module" -- and it rests on nothing more than a substring match. """ import os import sys import pytest sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools")) import compat # noqa: E402 from compat import ( # noqa: E402 NESTED, PROVIDERS, Unreadable, check, is_broken, ) # A middlewared that the patch fits: TrueNAS 25.10 in miniature. GOOD = { "rclone/remote/b2.py": "class B2RcloneRemote(BaseRcloneRemote):\n pass\n", "plugins/cloud_backup/restic.py": ( "class ResticConfig:\n cmd: list\n\n" "def get_restic_config(cloud_backup):\n return ResticConfig([], {})\n" ), "plugins/cloud/snapshot.py": ( 'async def create_snapshot(middleware, path, name="x"):\n return "s", "p"\n' ), "plugins/cloud/crud.py": ( "class CloudTaskServiceMixin:\n" " async def _validate(self, app, verrors, name, data):\n" " verrors.add('x', 'datasets that have no further '\n" " 'nesting')\n" ), "plugins/cloud_backup/sync.py": ( "async def restic_backup(middleware, job, cloud_backup, dry_run=False, " "rate_limit=None):\n pass\n" ), # The middlewared METHODS the injected code calls. These now go through the # PUBLIC pool.* API: TrueNAS 26 deleted plugins/zfs_/ outright, taking the whole # private zfs.* service with it -- see TestMiddlewareMethodsWeCall. # # This default tree is a MODERN box (25.10/26): it has pool.snapshot and no # zfs.snapshot. The older shape is built explicitly where it is tested. # Not a plugin: a method on the middleware OBJECT. `snapshot_service()` resolves # the snapshot namespace through it, so if it vanishes the module cannot sweep the # snapshot it just took. "utils/plugins.py": ( "class LoadPluginsMixin:\n" " def get_service(self, name):\n pass\n" ), "plugins/pool_/dataset.py": ( "class PoolDatasetService(CRUDService):\n" " class Config:\n" " namespace = 'pool.dataset'\n" " def query(self, filters, options):\n pass\n" ), "plugins/pool_/snapshot.py": ( "class PoolSnapshotService(CRUDService):\n" " class Config:\n" " namespace = 'pool.snapshot'\n" " def query(self, filters, options):\n pass\n" " def delete(self, id_, options={}):\n pass\n" ), } #: A 24.10/25.04 box: `pool.snapshot` does not exist yet and the snapshot CRUD #: service still answers to the (then-public) `zfs.snapshot`. ZFS_ERA_SNAPSHOT = ( "class ZFSSnapshot(CRUDService):\n" " class Config:\n" " namespace = 'zfs.snapshot'\n" " def query(self, filters, options):\n pass\n" " def do_delete(self, id_, options={}):\n pass\n" ) def loader(files): def load(path): if path not in files: return None v = files[path] if isinstance(v, Exception): raise v return v return load def check_files(files, modules=None): return check(loader(files), modules) def with_(**overrides): files = dict(GOOD) files.update(overrides) return files class TestTheBaseline: def test_a_good_tree_is_ok_and_not_native(self): r = check_files(GOOD) for mod in (PROVIDERS, NESTED): assert r[mod]["ok"], r[mod]["problems"] assert not r[mod]["native"] assert not r[mod]["unknown"] class TestFalseOkWouldBreakBackups: """The patch calls the originals POSITIONALLY. A name-subset check passed all of these, and each is a TypeError or -- worse -- silently swapped arguments.""" def test_reordered_parameters_are_broken(self): r = check_files(with_(**{ "plugins/cloud/snapshot.py": 'async def create_snapshot(name, path, middleware):\n return 1, 2\n', })) assert is_broken(r[NESTED]) def test_a_keyword_only_conversion_is_broken(self): r = check_files(with_(**{ "plugins/cloud/snapshot.py": 'async def create_snapshot(middleware, *, path, name="x"):\n return 1, 2\n', })) assert is_broken(r[NESTED]) def test_a_new_required_parameter_is_broken(self): r = check_files(with_(**{ "plugins/cloud/snapshot.py": 'async def create_snapshot(middleware, path, name, dataset):\n return 1, 2\n', })) assert is_broken(r[NESTED]) def test_a_new_optional_parameter_is_fine(self): # The patch simply will not pass it. Refusing here would be false BROKEN. r = check_files(with_(**{ "plugins/cloud/snapshot.py": 'async def create_snapshot(middleware, path, name="x", quiet=False):\n' " return 1, 2\n", })) assert r[NESTED]["ok"], r[NESTED]["problems"] def test_the_master_signature_change_is_caught(self): # iX really did rename this on master: get_restic_config(entry, credentials). # RESTIC_BLOCK rebinds the module-level name to a 1-arg wrapper, so getting # this wrong kills EVERY TrueCloud task -- Storj included. r = check_files(with_(**{ "plugins/cloud_backup/restic.py": "class ResticConfig:\n cmd: list\n\n" "def get_restic_config(entry, credentials):\n pass\n", })) assert is_broken(r[PROVIDERS]) def test_a_vanished_symbol_is_broken(self): r = check_files(with_(**{ "plugins/cloud/snapshot.py": "def something_else():\n pass\n", })) assert is_broken(r[NESTED]) class TestFalseBrokenWouldDisableWorkingBoxes: def test_a_conditionally_defined_symbol_is_not_broken(self): r = check_files(with_(**{ "plugins/cloud/snapshot.py": "try:\n" " from .fast import create_snapshot\n" "except ImportError:\n" ' async def create_snapshot(middleware, path, name="x"):\n' " return 1, 2\n", })) assert not is_broken(r[NESTED]), r[NESTED]["problems"] def test_a_re_exported_symbol_is_unknown_not_broken(self): r = check_files(with_(**{ "plugins/cloud_backup/restic.py": "from ._impl import ResticConfig, get_restic_config\n", })) assert not is_broken(r[PROVIDERS]) assert r[PROVIDERS]["unknown"] def test_an_unreadable_source_is_unknown_not_broken(self): # A rate limit (the matrix makes ~30 unauthenticated requests) must not be # able to say "iX deleted six files, both modules are broken". r = check_files(with_(**{ "plugins/cloud/snapshot.py": Unreadable("HTTP 429"), })) assert not is_broken(r[NESTED]) assert r[NESTED]["unknown"] def test_a_definite_break_still_wins_over_an_unknown(self): r = check_files(with_(**{ "plugins/cloud/snapshot.py": Unreadable("HTTP 429"), "plugins/cloud_backup/sync.py": "async def restic_backup(job, middleware, cloud_backup):\n pass\n", })) assert is_broken(r[NESTED]), "unknown must not launder away a proven break" class TestTheNativeVerdict: """"native" means "retire the module". It is the most destructive thing this file can say, and it is only a substring match — so it must never outrank BROKEN.""" def test_broken_outranks_native(self): # Guard reworded (reads as native) AND the signatures changed (really broken). # This used to render as good news: green CI, no bug report, and a README row # telling users the feature went native while it was in fact broken. r = check_files(with_(**{ "plugins/cloud/crud.py": "class CloudTaskServiceMixin:\n" " async def _validate(self, verrors, name):\n" " verrors.add('x', 'no children allowed')\n", })) assert r[NESTED]["native"] assert is_broken(r[NESTED]) assert compat._verdict(r[NESTED]) == "BROKEN" def test_an_already_patched_tree_does_not_read_as_native(self): # B2_BLOCK writes `B2RcloneRemote.restic = True` into b2.py. Scanning the whole # file finds OUR OWN line and concludes TrueNAS went native — so the command # compat.py's docstring recommends for a live box (`--tree /usr/lib/...`) # reported providers as native on every patched machine. r = check_files(with_(**{ "rclone/remote/b2.py": "class B2RcloneRemote(BaseRcloneRemote):\n pass\n" "\n# TRUECLOUD_PATCH — added by truenas-truecloud-patch/patch/apply.sh\n" "B2RcloneRemote.restic = True\n", })) assert not r[PROVIDERS]["native"], "read its own patch as native support" assert r[PROVIDERS]["ok"] def test_a_genuinely_native_b2_is_native(self): r = check_files(with_(**{ "rclone/remote/b2.py": "class B2RcloneRemote(BaseRcloneRemote):\n restic = True\n", })) assert r[PROVIDERS]["native"] class TestUpdateReadmeCannotPublishAGuess: def test_it_refuses_when_anything_is_unknown(self, tmp_path): readme = tmp_path / "README.md" readme.write_text(f"x\n{compat.BEGIN}\nold\n{compat.END}\ny\n") rows = [{ "ref": "TS-25.10.4", "unreleased": False, "modules": check_files(with_(**{ "plugins/cloud/snapshot.py": Unreadable("HTTP 429"), })), }] with pytest.raises(Unreadable): compat.update_readme(rows, path=str(readme)) assert "old" in readme.read_text(), "a blip must not repaint the matrix" class TestAsyncFlavour: """TrueNAS <= 25.10 is async; 26 is synchronous. Both are supported -- apply.sh injects the wrapper that matches. So asyncness is DETECTED, never assumed.""" def test_async_middleware_is_detected(self): assert compat.async_flavour(loader(GOOD)) is True def test_sync_middleware_is_detected(self): sync = dict(GOOD) sync["plugins/cloud/snapshot.py"] = ( 'def create_snapshot(middleware, path, name="x"):\n return "s", "p"\n' ) sync["plugins/cloud/crud.py"] = ( "class CloudTaskServiceMixin:\n" " def _validate(self, app, verrors, name, data):\n" " verrors.add('x', 'no further nesting')\n" ) sync["plugins/cloud_backup/sync.py"] = ( "def restic_backup(middleware, job, cloud_backup, dry_run=False, " "rate_limit=None):\n pass\n" ) assert compat.async_flavour(loader(sync)) is False def test_a_HALF_converted_middleware_is_refused(self): # The dangerous middle. If iX converts create_snapshot but not restic_backup, # there is no single wrapper flavour that works -- and guessing means either # a coroutine unpacked as a tuple, or the event loop blocked. None means # "do not patch"; apply.sh turns that into a skip, not a guess. half = dict(GOOD) half["plugins/cloud/snapshot.py"] = ( 'def create_snapshot(middleware, path, name="x"):\n return "s", "p"\n' ) assert compat.async_flavour(loader(half)) is None def test_an_unreadable_source_refuses_rather_than_guesses(self): broken = dict(GOOD) broken["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429") assert compat.async_flavour(loader(broken)) is None def test_the_real_truenas_versions(self): # Pinning the actual fact this whole port exists for. assert compat.async_flavour(loader(GOOD)) is True class TestMiddlewareMethodsWeCall: """The assumption class that was MISSING, and that hid a catastrophic break. The manifest recorded the symbols the patch WRAPS. It said nothing about the middlewared methods the patch CALLS -- and TrueNAS 26 deleted plugins/zfs_/dataset.py and plugins/zfs_/snapshot.py outright, taking `zfs.dataset.query`, `zfs.snapshot.query` and `zfs.snapshot.delete` with them. Nothing about the five cloud_backup files reveals that. The patch applied perfectly and every other check went green. The first backup would have failed -- or, far worse, snapshotted fine and then failed to DELETE, orphaning one snapshot per descendant dataset (250 on a real pool) on every run, forever. """ POOL_SNAPSHOT = GOOD["plugins/pool_/snapshot.py"] def _tree(self, **over): files = dict(GOOD) files.update(over) return files #: A 26 box: pool.snapshot only. def _modern(self, **over): return self._tree(**over) #: A 24.10/25.04 box: zfs.snapshot only -- plugins/pool_/snapshot.py does not #: exist yet. def _zfs_era(self, **over): return self._tree(**{ "plugins/pool_/snapshot.py": None, "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT, **over, }) def test_present_methods_are_ok(self): r = check_files(self._modern()) assert r[NESTED]["ok"], r[NESTED]["problems"] def test_the_OLD_zfs_era_snapshot_service_also_satisfies_the_call(self): # 24.10 and 25.04 have no `pool.snapshot` at all -- the CRUD service is the # then-public `zfs.snapshot`. Pinning only the modern spelling marked both of # those releases BROKEN and would have switched nested snapshots OFF on boxes # where they work perfectly. The runtime picks the same way; see # pick_snapshot_service(). r = check_files(self._zfs_era()) assert r[NESTED]["ok"], r[NESTED]["problems"] def test_it_is_broken_only_when_NEITHER_namespace_exists(self): # The real failure: middleware drops the last spelling we know how to call. r = check_files(self._tree(**{ "plugins/pool_/snapshot.py": None, "plugins/zfs_/snapshot.py": None, })) assert is_broken(r[NESTED]) details = " ".join(p["detail"] for p in r[NESTED]["problems"]) assert "pool.snapshot.delete" in details assert "zfs.snapshot.delete" in details, ( "the report must say BOTH spellings were tried, or whoever reads it will " "think we simply never looked for the one their box has" ) def test_we_do_NOT_depend_on_a_middleware_dataset_query_at_all(self): # iX could delete plugins/pool_/dataset.py tomorrow and the patch would not # care, because the staging plan is enumerated from ZFS, not from middleware. # # That is deliberate, and it was expensive to learn. `pool.dataset.query` # exists and is correctly shaped -- and it LIES: it applies a visibility # policy that hides ix-apps/*, .system/* and .ix-virt/* (84 of 270 datasets # on the real pool, including live app data). No source check could ever # have caught that; only running it could. So there is no assumption here # left to break. r = check_files(self._modern(**{"plugins/pool_/dataset.py": None})) assert r[NESTED]["ok"], r[NESTED]["problems"] ids = {c.id for c in compat.MIDDLEWARE_CALLS} assert not any("dataset" in i or "query" in i for i in ids), ( "a dataset/snapshot QUERY assumption crept back into the manifest -- " "middleware's queries are filtered; enumerate from ZFS" ) def test_a_renamed_namespace_is_broken(self): r = check_files(self._tree(**{ "plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace( "'pool.snapshot'", "'zfs.resource.snapshot'"), "plugins/zfs_/snapshot.py": None, })) assert is_broken(r[NESTED]) def test_the_CRUDService_do_prefix_is_accepted(self): # A CRUDService exposes `delete` from a method NAMED `do_delete`. Both # spellings are live across the matrix. Accepting only the literal name # reported working releases as broken. r = check_files(self._modern(**{ "plugins/pool_/snapshot.py": self.POOL_SNAPSHOT.replace( "def delete(", "def do_delete("), })) assert r[NESTED]["ok"], r[NESTED]["problems"] def test_the_snapshot_delete_reason_names_the_orphan_risk(self): # If this ever regresses, whoever reads the bug report must understand that # it is not a cosmetic failure. r = check_files(self._tree(**{ "plugins/pool_/snapshot.py": None, "plugins/zfs_/snapshot.py": None, })) whys = " ".join(p["why"] for p in r[NESTED]["problems"]) assert "orphan" in whys class TestTheMethodCheckIsNotJustANamespaceCheck: """compat must verify the METHOD, not merely that the namespace still exists. Deleting the method check entirely used to leave all 304 tests green -- so the "namespace AND method" claim was unenforced and silently revertible. It is the half of the predicate that catches iX gutting a method while keeping its service, which they have already done to `pool.snapshot.do_update` on master. """ def test_a_namespace_that_no_longer_defines_delete_is_broken(self): gutted = ( "class PoolSnapshotService(CRUDService):\n" " class Config:\n" " namespace = 'pool.snapshot'\n" " def query(self, filters, options):\n pass\n" # do_delete is GONE -- the service is still registered and still a # CRUDService, so it still INHERITS a callable `delete`. ) r = check_files(with_(**{ "plugins/pool_/snapshot.py": gutted, "plugins/zfs_/snapshot.py": None, # no fallback either })) assert is_broken(r[NESTED]), ( "a namespace with no delete must be BROKEN. Checking only that the " "namespace exists would apply the patch to a box that cannot sweep its " "own snapshots." ) def test_the_alternative_still_saves_it_when_only_the_primary_is_gutted(self): gutted = ( "class PoolSnapshotService(CRUDService):\n" " class Config:\n" " namespace = 'pool.snapshot'\n" " def query(self, filters, options):\n pass\n" ) r = check_files(with_(**{ "plugins/pool_/snapshot.py": gutted, "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT, })) assert r[NESTED]["ok"], r[NESTED]["problems"] class TestUnreadableIsNeverOkAndNeverBroken: """A rate limit is not a regression, and it is not a clean bill of health either. compat runs ~30 unauthenticated GitHub requests per matrix; 429 is a real outcome. It also runs at BOOT against the installed tree, where a read can fail with EACCES. * treating unreadable as BROKEN repaints the README, files a bug report, and makes apply.sh refuse the module on a box where it works. * treating it as OK injects a module whose delete may be gone. Both mutations used to pass the whole suite. """ def test_both_spellings_unreadable_is_unknown_not_broken(self): r = check_files(with_(**{ "plugins/pool_/snapshot.py": Unreadable("HTTP 429"), "plugins/zfs_/snapshot.py": Unreadable("HTTP 429"), })) assert not is_broken(r[NESTED]), "a 429 is not iX deleting the snapshot service" assert r[NESTED]["unknown"] def test_an_unreadable_primary_with_a_healthy_alternative_is_ok(self): r = check_files(with_(**{ "plugins/pool_/snapshot.py": Unreadable("HTTP 429"), "plugins/zfs_/snapshot.py": ZFS_ERA_SNAPSHOT, })) assert r[NESTED]["ok"], r[NESTED]["problems"] assert not r[NESTED]["unknown"], ( "one spelling answered the question; the other's 429 is irrelevant" ) def test_a_missing_primary_with_an_unreadable_alternative_is_unknown(self): # We cannot tell whether the box is broken. Saying either would be a guess. r = check_files(with_(**{ "plugins/pool_/snapshot.py": None, "plugins/zfs_/snapshot.py": Unreadable("HTTP 429"), })) assert not is_broken(r[NESTED]) assert r[NESTED]["unknown"] class TestGetServiceIsChecked: """The runtime resolves the snapshot namespace through `middleware.get_service`. It is not a plugin method, so the manifest had no way to express it and never checked it. If it vanishes, `_can_delete` reports BOTH namespaces unusable and every nested backup fails -- on a box the preflight had declared healthy. """ def test_a_middleware_without_get_service_is_broken(self): r = check_files(with_(**{"utils/plugins.py": None})) assert is_broken(r[NESTED]) details = " ".join(p["detail"] for p in r[NESTED]["problems"]) assert "get_service" in details class TestATransientNetworkBlipDoesNotWakeAnybody: """The fingerprint must digest what iX BROKE, not what GitHub failed to serve. `unknown` problems (a 429 on one of ~30 unauthenticated fetches, an EACCES at boot) used to be folded into an already-broken module's problem list, so one blip flipped the fingerprint, `compat_publish` rewrote the issue body, and the next clean run rewrote it back. Daily churn is what teaches people to ignore the bot -- which is the whole thing this fingerprint exists to prevent. """ def _rows(self, files): return [{"ref": "master", "modules": check_files(files)}] def test_an_unreadable_file_does_not_change_the_fingerprint_of_a_broken_ref(self): broken = with_(**{ "plugins/cloud/snapshot.py": "async def create_snapshot(name, path, middleware):\n return 1, 2\n", }) clean = compat.fingerprint(self._rows(broken)) blipped = dict(broken) blipped["rclone/remote/b2.py"] = Unreadable("HTTP 429") assert compat.fingerprint(self._rows(blipped)) == clean, ( "a rate-limited fetch changed the fingerprint, so the bot rewrites the " "issue body and then rewrites it back tomorrow" ) def test_a_REAL_new_finding_still_changes_it(self): # ...and the anti-noise measure must not have made it deaf. broken = with_(**{ "plugins/cloud/snapshot.py": "async def create_snapshot(name, path, middleware):\n return 1, 2\n", }) worse = dict(broken) worse["plugins/cloud_backup/restic.py"] = ( "class ResticConfig:\n cmd: list\n\n" "def get_restic_config(entry, credentials):\n pass\n" ) assert compat.fingerprint(self._rows(worse)) != compat.fingerprint(self._rows(broken))