TrueNAS 26 support: one sync implementation, two wrappers

26 rewrites cloud_backup from async to synchronous AND deletes
get_dataset_recursive(), which SNAPSHOT_BLOCK called out of the host module's
namespace. Either is a broken backup found at restore time.

The nested module is now one synchronous implementation talking to middlewared via
call_sync, behind two thin wrappers. apply.sh reads which flavour the installed
middleware declares and injects the matching one: <= 25.10 reaches it through
'await middleware.run_in_thread(...)', 26 is already in a worker thread and calls
it directly. The snapshot/bind-mount/failure logic exists once -- an async twin
would mean every future fix had to land twice.

A middleware whose three wrapped functions disagree about asyncness is refused,
not guessed at. get_dataset_recursive is vendored, removing the dependency on both
versions rather than asserting it.

master stays BROKEN on purpose: iX are still renaming middleware->context,
cloud_backup->entry and adding a required credentials param there. Chasing a
branch that moves daily is how you ship a patch nobody tested.

tests/test_apply_blocks.py

@@ -14,14 +14,27 @@ import pytest
 
 APPLY_SH = os.path.join(os.path.dirname(__file__), "..", "patch", "apply.sh")
 
+#: Every block that is actually injected into a middlewared module.
+#:
+#: The three nested blocks come in two flavours. TrueNAS <= 25.10 has an ASYNC
+#: cloud_backup path; TrueNAS 26 rewrote it synchronous. apply.sh reads which one is
+#: installed and injects the matching wrapper -- an `async def` on 26 would hand
+#: sync.py a coroutine where it unpacks a tuple, and a plain `def` on 25.10 would
+#: block the event loop. Both flavours must therefore be valid Python, always.
 EXPECTED_BLOCKS = {
     "B2_BLOCK",
     "RESTIC_BLOCK",
-    "SNAPSHOT_BLOCK",
-    "CRUD_BLOCK",
-    "SYNC_BLOCK",
+    "SNAPSHOT_ASYNC",
+    "SNAPSHOT_SYNC",
+    "CRUD_ASYNC",
+    "CRUD_SYNC",
+    "SYNC_ASYNC",
+    "SYNC_SYNC",
 }
 
+NESTED_BLOCKS = ["SNAPSHOT_ASYNC", "SNAPSHOT_SYNC", "CRUD_ASYNC", "CRUD_SYNC",
+                 "SYNC_ASYNC", "SYNC_SYNC"]
+
 
 def heredoc_source():
     with open(APPLY_SH, encoding="utf-8") as fh:
@@ -32,18 +45,30 @@ def heredoc_source():
 
 
 def extract_blocks():
+    """The blocks as apply.sh actually builds them.
+
+    EVALUATED, not read off as string literals: each nested block is a CORE
+    concatenated with a flavour-specific wrapper, so reading only `ast.Constant`
+    would silently return nothing for them -- a green suite over blocks nobody
+    checked. Assignments that need the runtime (argv, imports) simply fail to
+    evaluate and are skipped.
+    """
     tree = ast.parse(heredoc_source())
-    blocks = {}
-    for node in ast.walk(tree):
-        if isinstance(node, ast.Assign):
-            for tgt in node.targets:
-                if (
-                    isinstance(tgt, ast.Name)
-                    and tgt.id.endswith("_BLOCK")
-                    and isinstance(node.value, ast.Constant)
-                    and isinstance(node.value.value, str)
-                ):
-                    blocks[tgt.id] = node.value.value
+    ns, blocks = {}, {}
+    for node in tree.body:
+        if not isinstance(node, ast.Assign):
+            continue
+        try:
+            value = eval(  # noqa: S307 - our own shipped source, on purpose
+                compile(ast.Expression(node.value), "<blocks>", "eval"), {}, ns
+            )
+        except Exception:
+            continue
+        for tgt in node.targets:
+            if isinstance(tgt, ast.Name) and isinstance(value, str):
+                ns[tgt.id] = value
+                if tgt.id in EXPECTED_BLOCKS:
+                    blocks[tgt.id] = value
     return blocks
 
 
@@ -98,7 +123,7 @@ def test_injected_block_carries_the_idempotency_marker(name):
     assert extract_blocks()[name].lstrip("\n").startswith("# TRUECLOUD_PATCH")
 
 
-@pytest.mark.parametrize("name", ["SNAPSHOT_BLOCK", "CRUD_BLOCK", "SYNC_BLOCK"])
+@pytest.mark.parametrize("name", NESTED_BLOCKS)
 def test_nested_blocks_degrade_safely_without_the_module(name):
     # If _truecloud_nested failed to install, every nested block must no-op.
     # Critically this includes CRUD_BLOCK: relaxing the guard without the
@@ -119,20 +144,21 @@ class TestSnapshotLeak:
         # On a staging failure, sync.py's `snapshot, local_path = await
         # create_snapshot(...)` never completes, so its local `snapshot` stays
         # None and its finally deletes nothing. We must sweep it ourselves.
-        block = extract_blocks()["SNAPSHOT_BLOCK"]
+        block = extract_blocks()["SNAPSHOT_ASYNC"]
         assert "except Exception:" in block
         assert "delete_snapshot_tree" in block
         assert "raise" in block
 
     def test_sync_block_cleans_up_on_every_path(self):
-        block = extract_blocks()["SYNC_BLOCK"]
+        block = extract_blocks()["SYNC_ASYNC"]
         assert "finally:" in block
         assert "cleanup_task" in block
 
 
 def test_crud_block_is_scoped_to_cloud_backup():
     # cloudsync has no staging teardown wired in, so its guard must stay.
-    assert '!= "cloud_backup"' in extract_blocks()["CRUD_BLOCK"]
+    for name in ("CRUD_ASYNC", "CRUD_SYNC"):
+        assert '!= "cloud_backup"' in extract_blocks()[name]
 
 
 class TestIndependentModules:
@@ -220,7 +246,7 @@ class TestIndependentModules:
         # find the string in our own patch and never detect native support.
         sh = self._sh()
         assert "split('\\n# TRUECLOUD_PATCH', 1)[0]" in sh
-        assert "no further nesting" in extract_blocks()["CRUD_BLOCK"], (
+        assert "no further nesting" in extract_blocks()["CRUD_ASYNC"], (
             "if this ever stops being true, the probe comment is stale"
         )
 
@@ -264,7 +290,7 @@ class TestOptIn:
         # The guard-relaxing crud.py patch must be inside the enabled branch.
         src = heredoc_source()
         gate = src.index("if not nested_needed:")
-        crud = src.index("patch_file(crud_py, CRUD_BLOCK)")
+        crud = src.index("patch_file(crud_py, _crud_block)")
         assert gate < crud, "crud.py patch must sit inside the opt-in branch"
 
     def test_disabling_REVERTS_the_patch_rather_than_merely_skipping_it(self):
@@ -282,7 +308,7 @@ class TestOptIn:
         assert "from mw_patch import patch_file, revert_nested" in src
         gate = src.index("if not nested_needed:")
         revert = src.index("reverted = revert_nested(")
-        patch = src.index("patch_file(crud_py, CRUD_BLOCK)")
+        patch = src.index("patch_file(crud_py, _crud_block)")
         assert gate < revert < patch, "revert belongs in the not-needed branch"
 
     def test_import_failure_skips_the_patch_rather_than_crashing(self):
@@ -302,9 +328,9 @@ def test_guard_is_relaxed_only_after_traversal_is_installed():
     src = heredoc_source()
     order = [
         src.index("shutil.copyfile(nested_src, nested_dst)"),
-        src.index("patch_file(snapshot_py, SNAPSHOT_BLOCK)"),
-        src.index("patch_file(sync_path, SYNC_BLOCK)"),
-        src.index("patch_file(crud_py, CRUD_BLOCK)"),
+        src.index("patch_file(snapshot_py, _snapshot_block)"),
+        src.index("patch_file(sync_path, _sync_block)"),
+        src.index("patch_file(crud_py, _crud_block)"),
     ]
     assert order == sorted(order), "crud.py must be patched last"
 
@@ -324,7 +350,7 @@ class TestWrappersDoNotHardcodeStockArity:
     """
 
     def test_restic_backup_forwards_rather_than_naming_stock_params(self):
-        block = extract_blocks()["SYNC_BLOCK"]
+        block = extract_blocks()["SYNC_ASYNC"]
         assert "async def restic_backup(middleware, job, cloud_backup, *args, **kwargs)" in block
         assert "_tc_orig_restic_backup(middleware, job, cloud_backup, *args, **kwargs)" in block
 

tests/test_compat.py

@@ -129,14 +129,11 @@ class TestFalseOkWouldBreakBackups:
         }))
         assert is_broken(r[PROVIDERS])
 
-    def test_async_to_sync_is_caught(self):
-        # THE TrueNAS 26 change.
+    def test_a_vanished_symbol_is_broken(self):
         r = check_files(with_(**{
-            "plugins/cloud/snapshot.py":
-                'def create_snapshot(middleware, path, name="x"):\n    return 1, 2\n',
+            "plugins/cloud/snapshot.py": "def something_else():\n    pass\n",
         }))
         assert is_broken(r[NESTED])
-        assert "async def" in r[NESTED]["problems"][0]["detail"]
 
 
 class TestFalseBrokenWouldDisableWorkingBoxes:
@@ -172,8 +169,7 @@ class TestFalseBrokenWouldDisableWorkingBoxes:
         r = check_files(with_(**{
             "plugins/cloud/snapshot.py": Unreadable("HTTP 429"),
             "plugins/cloud_backup/sync.py":
-                "def restic_backup(middleware, job, cloud_backup, dry_run=False, "
-                "rate_limit=None):\n    pass\n",
+                "async def restic_backup(job, middleware, cloud_backup):\n    pass\n",
         }))
         assert is_broken(r[NESTED]), "unknown must not launder away a proven break"
 
@@ -189,7 +185,7 @@ class TestTheNativeVerdict:
         r = check_files(with_(**{
             "plugins/cloud/crud.py":
                 "class CloudTaskServiceMixin:\n"
-                "    def _validate(self, app, verrors, name, data):\n"
+                "    async def _validate(self, verrors, name):\n"
                 "        verrors.add('x', 'no children allowed')\n",
         }))
         assert r[NESTED]["native"]
@@ -231,3 +227,47 @@ class TestUpdateReadmeCannotPublishAGuess:
         with pytest.raises(Unreadable):
             compat.update_readme(rows, path=str(readme))
         assert "old" in readme.read_text(), "a blip must not repaint the matrix"
+
+
+class TestAsyncFlavour:
+    """TrueNAS <= 25.10 is async; 26 is synchronous. Both are supported -- apply.sh
+    injects the wrapper that matches. So asyncness is DETECTED, never assumed."""
+
+    def test_async_middleware_is_detected(self):
+        assert compat.async_flavour(loader(GOOD)) is True
+
+    def test_sync_middleware_is_detected(self):
+        sync = dict(GOOD)
+        sync["plugins/cloud/snapshot.py"] = (
+            'def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+        )
+        sync["plugins/cloud/crud.py"] = (
+            "class CloudTaskServiceMixin:\n"
+            "    def _validate(self, app, verrors, name, data):\n"
+            "        verrors.add('x', 'no further nesting')\n"
+        )
+        sync["plugins/cloud_backup/sync.py"] = (
+            "def restic_backup(middleware, job, cloud_backup, dry_run=False, "
+            "rate_limit=None):\n    pass\n"
+        )
+        assert compat.async_flavour(loader(sync)) is False
+
+    def test_a_HALF_converted_middleware_is_refused(self):
+        # The dangerous middle. If iX converts create_snapshot but not restic_backup,
+        # there is no single wrapper flavour that works -- and guessing means either
+        # a coroutine unpacked as a tuple, or the event loop blocked. None means
+        # "do not patch"; apply.sh turns that into a skip, not a guess.
+        half = dict(GOOD)
+        half["plugins/cloud/snapshot.py"] = (
+            'def create_snapshot(middleware, path, name="x"):\n    return "s", "p"\n'
+        )
+        assert compat.async_flavour(loader(half)) is None
+
+    def test_an_unreadable_source_refuses_rather_than_guesses(self):
+        broken = dict(GOOD)
+        broken["plugins/cloud_backup/sync.py"] = Unreadable("HTTP 429")
+        assert compat.async_flavour(loader(broken)) is None
+
+    def test_the_real_truenas_versions(self):
+        # Pinning the actual fact this whole port exists for.
+        assert compat.async_flavour(loader(GOOD)) is True

tests/test_truecloud_nested.py

@@ -12,7 +12,6 @@ Two rules are under test above all else:
    on EVERY run.
 """
 
-import asyncio
 import os
 import sys
 
@@ -199,12 +198,20 @@ class TestSnapshotTreeNames:
 
 
 class FakeMiddleware:
+    """middlewared as this module actually uses it: `call_sync`, from a thread.
+
+    The module is synchronous on purpose -- see the orchestration note in
+    truecloud_nested.py. TrueNAS <= 25.10 reaches it through
+    `await middleware.run_in_thread(...)` and TrueNAS 26 calls it directly, but the
+    logic below the boundary is the same code either way, so it is tested once.
+    """
+
     def __init__(self, snapshots=None):
         self.snapshots = list(snapshots or [])
         self.calls = []
         self.logger = None
 
-    async def call(self, method, *args):
+    def call_sync(self, method, *args):
         self.calls.append((method, args))
         if method == "zfs.snapshot.query":
             return [{"name": n} for n in self.snapshots]
@@ -222,8 +229,35 @@ class FakeMiddleware:
             return True
         raise AssertionError(f"unexpected call {method}")
 
-    async def run_in_thread(self, fn, *args):
-        return fn(*args)
+
+def stub_core(monkeypatch, tn, *, plan=None, order=None, plan_raises=None):
+    """Replace the blocking core (plan/apply/verify/teardown) with recorders.
+
+    stage_nested calls these directly now, so they are patched by NAME rather than
+    intercepted at a `run_in_thread` boundary that no longer exists.
+    """
+    def record(name, result):
+        def fn(*args, **kwargs):
+            if order is not None:
+                order.append(name)
+            if name == "plan_staging" and plan_raises is not None:
+                raise plan_raises
+            return result() if callable(result) else result
+        fn.__name__ = name
+        return fn
+
+    real_write = tn._write_sidecar
+
+    def write_sidecar(*args, **kwargs):
+        if order is not None:
+            order.append("_write_sidecar")
+        return real_write(*args, **kwargs)
+
+    monkeypatch.setattr(tn, "_write_sidecar", write_sidecar)
+    monkeypatch.setattr(tn, "plan_staging", record("plan_staging", plan or ([], [])))
+    monkeypatch.setattr(tn, "apply_plan", record("apply_plan", True))
+    monkeypatch.setattr(tn, "verify_staged", record("verify_staged", True))
+    monkeypatch.setattr(tn, "teardown", record("teardown", []))
 
 
 class TestDeleteSnapshotTree:
@@ -231,20 +265,20 @@ class TestDeleteSnapshotTree:
         mw = FakeMiddleware([
             "Tap@snap", "Tap/apps@snap", "Tap/apps/lidarr@snap", "Tap@keepme",
         ])
-        asyncio.run(delete_snapshot_tree(mw, "Tap@snap"))
+        delete_snapshot_tree(mw, "Tap@snap")
         assert mw.snapshots == ["Tap@keepme"]
 
     def test_is_idempotent_when_stock_already_removed_the_parent(self):
         # Stock's finally can win the race once our mounts are released.
         mw = FakeMiddleware(["Tap/apps@snap", "Tap/apps/lidarr@snap"])
-        asyncio.run(delete_snapshot_tree(mw, "Tap@snap"))
+        delete_snapshot_tree(mw, "Tap@snap")
         assert mw.snapshots == []
 
     def test_uses_a_single_recursive_delete_not_252_individual_ones(self):
         # 252 sequential deletes are slow AND not atomic: a run killed part-way
         # through leaves exactly the orphans this function exists to prevent.
         mw = FakeMiddleware(["Tap@snap", "Tap/apps@snap", "Tap/apps/lidarr@snap"])
-        asyncio.run(delete_snapshot_tree(mw, "Tap@snap"))
+        delete_snapshot_tree(mw, "Tap@snap")
         assert mw.snapshots == []
         deletes = [a for m, a in mw.calls if m == "zfs.snapshot.delete"]
         assert len(deletes) == 1, "should be ONE recursive call, not one per snapshot"
@@ -255,20 +289,20 @@ class TestDeleteSnapshotTree:
 
     def test_survives_recursive_and_query_failure_by_deleting_the_parent(self):
         class Broken(FakeMiddleware):
-            async def call(self, method, *args):
+            def call_sync(self, method, *args):
                 if method == "zfs.snapshot.query":
                     raise RuntimeError("boom")
                 if method == "zfs.snapshot.delete" and len(args) > 1:
                     raise RuntimeError("recursive delete unavailable")
-                return await super().call(method, *args)
+                return super().call_sync(method, *args)
 
         mw = Broken(["Tap@snap"])
-        asyncio.run(delete_snapshot_tree(mw, "Tap@snap"))
+        delete_snapshot_tree(mw, "Tap@snap")
         assert mw.snapshots == []
 
     def test_leaves_unrelated_snapshots_alone_when_the_tree_is_gone(self):
         mw = FakeMiddleware(["Tap@unrelated"])
-        asyncio.run(delete_snapshot_tree(mw, "Tap@snap"))
+        delete_snapshot_tree(mw, "Tap@snap")
         assert mw.snapshots == ["Tap@unrelated"]
 
 
@@ -281,20 +315,13 @@ class TestStageNestedOrdering:
 
         monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
         order = []
+        stub_core(monkeypatch, tn, order=order,
+                  plan=([("/src", str(tmp_path / "cloud_backup-5"))], []))
 
-        class Recorder(FakeMiddleware):
-            async def run_in_thread(self, fn, *args):
-                order.append(fn.__name__)
-                if fn.__name__ == "plan_staging":
-                    return ([("/src", str(tmp_path / "cloud_backup-5"))], [])
-                if fn.__name__ in ("apply_plan", "verify_staged", "teardown"):
-                    return [] if fn.__name__ == "teardown" else True
-                return fn(*args)
-
-        asyncio.run(tn.stage_nested(
-            Recorder(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap",
+        tn.stage_nested(
+            FakeMiddleware(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap",
             "cloud_backup-5", DATASETS,
-        ))
+        )
 
         assert order.index("_write_sidecar") < order.index("apply_plan")
 
@@ -312,28 +339,12 @@ class TestStageNestedOrdering:
             fh.write("Tap@old-crashed-run")
 
         mw = FakeMiddleware(["Tap@old-crashed-run", "Tap/apps@old-crashed-run"])
+        stub_core(monkeypatch, tn, plan=([("/src", root)], []))
 
-        class Stub(FakeMiddleware):
-            def __init__(self, inner):
-                super().__init__()
-                self.inner = inner
-
-            async def call(self, method, *args):
-                return await self.inner.call(method, *args)
-
-            async def run_in_thread(self, fn, *args):
-                if fn.__name__ == "plan_staging":
-                    return ([("/src", root)], [])
-                if fn.__name__ == "teardown":
-                    return []
-                if fn.__name__ in ("apply_plan", "verify_staged"):
-                    return True
-                return fn(*args)
-
-        asyncio.run(tn.stage_nested(
-            Stub(mw), "/mnt/Tap", "Tap@new", "Tap", "/mnt/Tap",
+        tn.stage_nested(
+            mw, "/mnt/Tap", "Tap@new", "Tap", "/mnt/Tap",
             "cloud_backup-5", DATASETS,
-        ))
+        )
 
         assert mw.snapshots == [], "the crashed run's snapshot tree must be reclaimed"
 
@@ -342,18 +353,13 @@ class TestStageNestedOrdering:
 
         monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path))
         root = tn.staging_root_for("cloud_backup-5")
-
-        class Failing(FakeMiddleware):
-            async def run_in_thread(self, fn, *args):
-                if fn.__name__ == "plan_staging":
-                    raise StagingError("boom")
-                return fn(*args)
+        stub_core(monkeypatch, tn, plan_raises=StagingError("boom"))
 
         with pytest.raises(StagingError):
-            asyncio.run(tn.stage_nested(
-                Failing(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap",
+            tn.stage_nested(
+                FakeMiddleware(), "/mnt/Tap", "Tap@snap", "Tap", "/mnt/Tap",
                 "cloud_backup-5", DATASETS,
-            ))
+            )
 
         assert not os.path.exists(sidecar_for(root))
 
@@ -374,7 +380,7 @@ class TestCleanupTask:
         mw = FakeMiddleware(["Tap@snap", "Tap/apps@snap"])
         monkeypatch.setattr(tn, "teardown", lambda *_a, **_k: [])
 
-        asyncio.run(cleanup_task(mw, "cloud_backup-5"))
+        cleanup_task(mw, "cloud_backup-5")
 
         assert mw.snapshots == []
         assert not os.path.exists(sidecar_for(root))
@@ -384,7 +390,7 @@ class TestCleanupTask:
 
         monkeypatch.setattr(tn, "STAGING_BASE", str(tmp_path / "nope"))
         mw = FakeMiddleware(["Tap@snap"])
-        asyncio.run(cleanup_task(mw, "cloud_backup-5"))
+        cleanup_task(mw, "cloud_backup-5")
         assert mw.calls == []
         assert mw.snapshots == ["Tap@snap"]