flan
cc43b2831d
sitecustomize.py: when find_spec resolves real_spec as None (module absent after a TrueNAS update), record a FAIL status and mark the module done so hook_status.json is still written and cmd_verify shows a diagnostic FAIL instead of the ambiguous "no status file found". sitecustomize.py: the AttributeError fallback in the URL-fix wrapper now writes a WARNING to stderr before returning the unmodified result, making the unexpected ResticConfig type visible in journalctl. apply.sh: after falling back to bare python3, verify that python3 can also import middlewared; if not, emit a second warning so the operator knows the backend patch may be installed in the wrong site-packages directory. patch_ui.py: abort (return without writing) when FIND.subn produces a count other than 1, instead of committing a doubly-patched bundle and having subsequent runs silently accept it via the MARKER check. uninstall.sh: when a vendor sitecustomize.py backup exists, use mv to atomically overwrite our file rather than rm-then-mv; eliminates the window where a read-only /usr causes rm to fail under set -e, aborting before the backup is restored.
299 lines
12 KiB
Python
299 lines
12 KiB
Python
"""
|
|
TrueCloud provider patch — sitecustomize.py
|
|
Installed into Python site-packages on every boot by apply.sh.
|
|
|
|
Hooks two middlewared module imports using the find_spec / exec_module API
|
|
(required for Python 3.12+, which ships with TrueNAS SCALE 25.x / Debian 13):
|
|
|
|
middlewared.rclone.remote.b2
|
|
Adds get_restic_config() so the native restic B2 backend works.
|
|
Restic repo URL: b2:<bucket>/<folder>
|
|
Auth: B2_ACCOUNT_ID, B2_ACCOUNT_KEY
|
|
|
|
middlewared.plugins.cloud_backup.restic
|
|
Fixes the URL builder for providers with no hostname component.
|
|
Stock code: f"{rclone_type}:{url}/{remote_path}" → "b2:/bucket/path" (broken)
|
|
Patched: "b2:bucket/path" when url == ""
|
|
|
|
Both patches are no-ops if the module already provides the functionality
|
|
(i.e. a future TrueNAS version adds native support). All errors are caught
|
|
and written to stderr so middlewared always starts regardless of patch state.
|
|
"""
|
|
|
|
import sys
|
|
|
|
|
|
# ── Import hook ───────────────────────────────────────────────────────────────
|
|
|
|
class _Finder:
|
|
"""
|
|
find_spec-based meta path finder (Python 3.4+, required for 3.12+).
|
|
Intercepts specific module imports, loads them normally, then patches.
|
|
"""
|
|
|
|
_targets = frozenset({
|
|
"middlewared.rclone.remote.b2",
|
|
"middlewared.plugins.cloud_backup.restic",
|
|
})
|
|
|
|
def __init__(self):
|
|
self._loading = set() # guards against re-entrant imports
|
|
self._done = set() # modules already patched
|
|
|
|
def find_spec(self, fullname, path, target=None): # noqa: ARG002
|
|
if (
|
|
fullname in self._targets
|
|
and fullname not in self._done
|
|
and fullname not in self._loading
|
|
):
|
|
import importlib.machinery
|
|
import importlib.util
|
|
|
|
# Find the real file spec HERE, before Python adds the module to
|
|
# sys.modules. If we deferred this to exec_module, find_spec would
|
|
# short-circuit via sys.modules[fullname].__spec__ (our own spec) and
|
|
# exec_module would call itself recursively forever.
|
|
self._loading.add(fullname)
|
|
try:
|
|
real_spec = importlib.util.find_spec(fullname)
|
|
finally:
|
|
self._loading.discard(fullname)
|
|
|
|
if real_spec is None:
|
|
# Module absent in this Python installation (e.g. removed in a
|
|
# TrueNAS update). Record a FAIL so hook_status.json is still
|
|
# written and cmd_verify gives a diagnostic instead of "no file".
|
|
_record_status(fullname, ok=False,
|
|
detail="module not found in this Python installation")
|
|
self._mark_done(fullname)
|
|
return None
|
|
|
|
return importlib.machinery.ModuleSpec(
|
|
fullname,
|
|
_Loader(self, fullname, real_spec),
|
|
origin=real_spec.origin,
|
|
is_package=real_spec.submodule_search_locations is not None,
|
|
)
|
|
return None
|
|
|
|
def _mark_done(self, fullname):
|
|
self._done.add(fullname)
|
|
if self._done >= self._targets:
|
|
try:
|
|
sys.meta_path.remove(self)
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
class _Loader:
|
|
def __init__(self, finder, fullname, real_spec):
|
|
self._finder = finder
|
|
self._fullname = fullname
|
|
self._real_spec = real_spec
|
|
|
|
def create_module(self, spec): # noqa: ARG002
|
|
return None # use Python's default module creation
|
|
|
|
def exec_module(self, module):
|
|
fullname = self._fullname
|
|
real_spec = self._real_spec
|
|
|
|
try:
|
|
real_spec.loader.exec_module(module)
|
|
# Fix module metadata so it looks like a normal import.
|
|
module.__spec__ = real_spec
|
|
module.__loader__ = real_spec.loader
|
|
if real_spec.origin:
|
|
module.__file__ = real_spec.origin
|
|
except Exception as exc:
|
|
# Mark done to prevent infinite retry loops, then surface the failure
|
|
# in hook_status.json so cmd_verify shows a diagnostic FAIL rather
|
|
# than "no status file found".
|
|
self._finder._mark_done(fullname)
|
|
_record_status(fullname, ok=False,
|
|
detail=f"module load failed: {exc}")
|
|
raise
|
|
self._finder._mark_done(fullname)
|
|
|
|
# exec_module succeeded — apply our patch.
|
|
# Must stay in sync with _Finder._targets.
|
|
try:
|
|
if fullname == "middlewared.rclone.remote.b2":
|
|
_patch_b2(module)
|
|
elif fullname == "middlewared.plugins.cloud_backup.restic":
|
|
_patch_restic(module)
|
|
except Exception as exc:
|
|
sys.stderr.write(
|
|
f"[truecloud-patch] patch failed for {fullname}: {exc}\n"
|
|
)
|
|
_record_status(fullname, ok=False, detail=str(exc))
|
|
|
|
|
|
# ── Patch functions ───────────────────────────────────────────────────────────
|
|
|
|
def _patch_b2(module):
|
|
cls = module.B2RcloneRemote
|
|
|
|
if hasattr(cls, "get_restic_config"):
|
|
# A future TrueNAS version already added native B2 restic support.
|
|
_record_status("middlewared.rclone.remote.b2", ok=True,
|
|
detail="native support present; patch not needed")
|
|
return
|
|
|
|
def _b2_restic_config(task):
|
|
p = task["credentials"]["provider"]
|
|
missing = [f for f in ("account", "key") if f not in p]
|
|
if missing:
|
|
raise KeyError(
|
|
f"truecloud-patch: B2 provider missing field(s) {missing!r}; "
|
|
f"schema may have changed. Present: {sorted(p)!r}"
|
|
)
|
|
return "", {"B2_ACCOUNT_ID": p["account"], "B2_ACCOUNT_KEY": p["key"]}
|
|
|
|
cls.get_restic_config = staticmethod(_b2_restic_config)
|
|
cls.restic = True
|
|
sys.stderr.write("[truecloud-patch] B2 restic support enabled\n")
|
|
_record_status("middlewared.rclone.remote.b2", ok=True,
|
|
detail="method attached; credential fields verified at first backup")
|
|
|
|
|
|
def _patch_restic(module):
|
|
if getattr(module.get_restic_config, "_truecloud_patched", False):
|
|
_record_status("middlewared.plugins.cloud_backup.restic", ok=True,
|
|
detail="already patched in this process")
|
|
return
|
|
|
|
import dataclasses
|
|
|
|
_orig = module.get_restic_config
|
|
|
|
def get_restic_config(cloud_backup):
|
|
# Call the original — it handles transfer_setting, cache, RESTIC_PASSWORD,
|
|
# env construction, and everything else we don't own.
|
|
result = _orig(cloud_backup)
|
|
|
|
# Fix stray leading slash in repo URL: "b2:/bucket" → "b2:bucket".
|
|
# "scheme://path" (Storj) must not be touched.
|
|
# Covers all flag forms restic accepts (--repository is a documented synonym):
|
|
# -r <url> --repo <url> --repo=<url>
|
|
# --repository <url> --repository=<url>
|
|
cmd = list(result.cmd)
|
|
for i, part in enumerate(cmd):
|
|
if part.startswith("--repo=") or part.startswith("--repository="):
|
|
prefix, _, url = part.partition("=")
|
|
prefix += "="
|
|
scheme, sep, rest = url.partition(":")
|
|
if sep and rest.startswith("/") and not rest.startswith("//"):
|
|
cmd[i] = f"{prefix}{scheme}:{rest[1:]}"
|
|
try:
|
|
return dataclasses.replace(result, cmd=cmd)
|
|
except TypeError:
|
|
try:
|
|
return result._replace(cmd=cmd)
|
|
except AttributeError:
|
|
sys.stderr.write(
|
|
"[truecloud-patch] WARNING: cannot fix repo URL; "
|
|
f"unexpected ResticConfig type {type(result).__name__!r}\n"
|
|
)
|
|
return result
|
|
break
|
|
if i and cmd[i - 1] in ("-r", "--repo", "--repository"):
|
|
scheme, sep, rest = part.partition(":")
|
|
if sep and rest.startswith("/") and not rest.startswith("//"):
|
|
cmd[i] = f"{scheme}:{rest[1:]}"
|
|
try:
|
|
return dataclasses.replace(result, cmd=cmd)
|
|
except TypeError:
|
|
try:
|
|
return result._replace(cmd=cmd)
|
|
except AttributeError:
|
|
sys.stderr.write(
|
|
"[truecloud-patch] WARNING: cannot fix repo URL; "
|
|
f"unexpected ResticConfig type {type(result).__name__!r}\n"
|
|
)
|
|
return result
|
|
break
|
|
return result
|
|
|
|
get_restic_config._truecloud_patched = True
|
|
module.get_restic_config = get_restic_config
|
|
sys.stderr.write("[truecloud-patch] restic URL fix applied\n")
|
|
_record_status("middlewared.plugins.cloud_backup.restic", ok=True)
|
|
|
|
|
|
_STATUS_FILE = "/data/truecloud-patch/hook_status.json"
|
|
_hook_status: dict = {}
|
|
|
|
|
|
def _record_status(fullname: str, ok: bool, detail: str = "") -> None:
|
|
import json
|
|
import os
|
|
import time
|
|
|
|
if fullname in _hook_status:
|
|
return # idempotent: first call wins
|
|
_hook_status[fullname] = {"ok": ok, "detail": detail}
|
|
if len(_hook_status) < len(_Finder._targets):
|
|
return # wait for all patches; _Finder._targets is the canonical count
|
|
|
|
payload = {
|
|
"patched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"patches": _hook_status,
|
|
}
|
|
try:
|
|
tmp = _STATUS_FILE + ".tmp"
|
|
with open(tmp, "w", encoding="utf-8") as fh:
|
|
json.dump(payload, fh, indent=2)
|
|
os.replace(tmp, _STATUS_FILE) # atomic on POSIX
|
|
except OSError:
|
|
pass # non-fatal — status file is informational only
|
|
|
|
|
|
# ── Entry point ───────────────────────────────────────────────────────────────
|
|
|
|
def _install():
|
|
import os
|
|
if os.path.exists("/data/truecloud-patch/disabled"):
|
|
return # kill switch
|
|
# Scope to the middlewared service process only — not midclt, debug scripts,
|
|
# or other tools that happen to share the same venv.
|
|
# Also covers python -m middlewared where argv[0] is the __main__.py path.
|
|
_argv0 = (sys.argv or [""])[0]
|
|
_base = os.path.basename(_argv0)
|
|
if not (
|
|
_base == "middlewared"
|
|
or (_base == "__main__.py"
|
|
and os.path.basename(os.path.dirname(_argv0)) == "middlewared")
|
|
):
|
|
return
|
|
import importlib.util
|
|
if importlib.util.find_spec("middlewared") is None:
|
|
return
|
|
# If apply.sh displaced an existing sitecustomize.py, exec it first so any
|
|
# vendor startup code (path additions, codec registrations, etc.) still runs.
|
|
# __file__ is absent in some embedded contexts; the empty fallback is safe.
|
|
_self = globals().get("__file__", "")
|
|
if _self:
|
|
_pre = _self + ".pre-truecloud-patch"
|
|
if os.path.isfile(_pre):
|
|
try:
|
|
import builtins
|
|
with open(_pre, encoding="utf-8") as _fh:
|
|
# Separate globals dict prevents the exec'd code from
|
|
# shadowing our names; sys.path changes still take effect
|
|
# via the shared sys module object.
|
|
exec( # noqa: S102
|
|
compile(_fh.read(), _pre, "exec"),
|
|
{"__builtins__": builtins, "__file__": _pre,
|
|
"__name__": "sitecustomize"},
|
|
)
|
|
except Exception:
|
|
pass
|
|
sys.meta_path.append(_Finder())
|
|
|
|
|
|
try:
|
|
_install()
|
|
except Exception:
|
|
pass # never raise from sitecustomize.py — it would prevent Python from starting
|