Initial implementation: extend TrueCloud Backup to S3 and B2 providers
Patches middlewared at runtime via sitecustomize.py (no file edits to /usr/) and widens the UI credential dropdown from Storj-only to S3+B2+Storj. Persists across TrueNAS updates via PREINIT initshutdownscript stored in DB.
This commit is contained in:
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
# /data/truecloud-patch/apply.sh
|
||||
#
|
||||
# PREINIT script registered via TrueNAS initshutdownscript.
|
||||
# Runs on every boot before middlewared starts, re-applying patches that
|
||||
# TrueNAS updates wipe from /usr/.
|
||||
#
|
||||
# Two things are patched:
|
||||
# 1. sitecustomize.py → monkey-patches B2 restic support + URL fix into
|
||||
# middlewared at Python startup time (backend)
|
||||
# 2. Angular JS bundle → adds S3 and B2 to the credential dropdown in
|
||||
# the TrueCloud Backup task form (UI)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PATCH_DIR="/data/truecloud-patch"
|
||||
LOG="$PATCH_DIR/apply.log"
|
||||
|
||||
{
|
||||
echo "=== $(date -Iseconds) ==="
|
||||
|
||||
# ── 1. Backend: install sitecustomize.py ─────────────────────────────
|
||||
SITE_PKG=$(python3 -c "import site; print(site.getsitepackages()[0])" 2>/dev/null || true)
|
||||
|
||||
if [ -z "$SITE_PKG" ]; then
|
||||
echo "WARNING: could not determine site-packages path, skipping backend patch"
|
||||
else
|
||||
# If an unrelated sitecustomize.py exists, back it up once.
|
||||
if [ -f "$SITE_PKG/sitecustomize.py" ] && \
|
||||
! grep -q "truecloud-patch" "$SITE_PKG/sitecustomize.py" 2>/dev/null; then
|
||||
cp "$SITE_PKG/sitecustomize.py" "$SITE_PKG/sitecustomize.py.pre-truecloud-patch"
|
||||
echo "Backed up existing sitecustomize.py"
|
||||
fi
|
||||
|
||||
cp "$PATCH_DIR/sitecustomize.py" "$SITE_PKG/sitecustomize.py"
|
||||
echo "Installed sitecustomize.py → $SITE_PKG/sitecustomize.py"
|
||||
fi
|
||||
|
||||
# ── 2. UI: patch Angular bundle ──────────────────────────────────────
|
||||
python3 "$PATCH_DIR/patch_ui.py"
|
||||
|
||||
echo "=== done ==="
|
||||
|
||||
} >> "$LOG" 2>&1
|
||||
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
create_task.py — create TrueNAS TrueCloud Backup tasks with S3 or B2 credentials.
|
||||
|
||||
The TrueNAS UI restricts the credential dropdown to Storj only. This script
|
||||
talks directly to the REST API so you can use any compatible credential.
|
||||
|
||||
Requires: an API key from TrueNAS UI → System → API Keys.
|
||||
|
||||
Examples
|
||||
--------
|
||||
List available cloud credentials:
|
||||
python3 create_task.py --host 192.168.1.1 --api-key <key> list-credentials
|
||||
|
||||
Create a task backed by a B2 credential (id=3):
|
||||
python3 create_task.py --host 192.168.1.1 --api-key <key> create \\
|
||||
--name "tank-to-b2" \\
|
||||
--path /mnt/tank/data \\
|
||||
--credential 3 \\
|
||||
--bucket my-bucket \\
|
||||
--folder backups/tank \\
|
||||
--password "restic-repo-password" \\
|
||||
--keep-last 14
|
||||
|
||||
Create a task using an S3-compatible credential (Wasabi, R2, etc.):
|
||||
python3 create_task.py --host 192.168.1.1 --api-key <key> create \\
|
||||
--name "tank-to-wasabi" \\
|
||||
--path /mnt/tank/data \\
|
||||
--credential 5 \\
|
||||
--bucket my-bucket \\
|
||||
--folder backups \\
|
||||
--password "restic-repo-password"
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
def make_client(host, api_key, insecure=False):
|
||||
base = f"https://{host}/api/v2.0"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
ctx = ssl.create_default_context()
|
||||
if insecure:
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def call(method, path, body=None):
|
||||
url = base + path
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=ctx) as resp:
|
||||
return json.loads(resp.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode()
|
||||
print(f"HTTP {exc.code} {exc.reason}: {detail}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except urllib.error.URLError as exc:
|
||||
print(f"Connection error: {exc.reason}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
def cmd_list_credentials(client, _args):
|
||||
creds = client("GET", "/cloudsync/credentials")
|
||||
if not creds:
|
||||
print("No cloud credentials configured.")
|
||||
return
|
||||
print(f"{'ID':>4} {'Provider':<14} Name")
|
||||
print("─" * 50)
|
||||
for c in sorted(creds, key=lambda x: x["id"]):
|
||||
print(f"{c['id']:>4} {c['provider']['type']:<14} {c['name']}")
|
||||
|
||||
|
||||
def cmd_list_tasks(client, _args):
|
||||
tasks = client("GET", "/cloud_backup")
|
||||
if not tasks:
|
||||
print("No TrueCloud Backup tasks configured.")
|
||||
return
|
||||
print(f"{'ID':>4} {'Enabled':<8} {'Provider':<14} Name")
|
||||
print("─" * 55)
|
||||
for t in sorted(tasks, key=lambda x: x["id"]):
|
||||
ptype = t["credentials"]["provider"]["type"] if t.get("credentials") else "?"
|
||||
enabled = "yes" if t.get("enabled") else "no"
|
||||
print(f"{t['id']:>4} {enabled:<8} {ptype:<14} {t['description']}")
|
||||
|
||||
|
||||
def cmd_create(client, args):
|
||||
parts = args.schedule.split()
|
||||
if len(parts) != 5:
|
||||
print("--schedule must be a 5-field cron expression, e.g. '0 2 * * *'", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
minute, hour, dom, month, dow = parts
|
||||
|
||||
body = {
|
||||
"description": args.name,
|
||||
"path": args.path,
|
||||
"credentials": args.credential,
|
||||
"attributes": {
|
||||
"bucket": args.bucket,
|
||||
"folder": args.folder,
|
||||
},
|
||||
"password": args.password,
|
||||
"keep_last": args.keep_last,
|
||||
"transfer_setting": args.transfer_setting,
|
||||
"schedule": {
|
||||
"minute": minute,
|
||||
"hour": hour,
|
||||
"dom": dom,
|
||||
"month": month,
|
||||
"dow": dow,
|
||||
},
|
||||
"snapshot": args.snapshot,
|
||||
"absolute_paths": args.absolute_paths,
|
||||
"enabled": not args.disabled,
|
||||
}
|
||||
|
||||
result = client("POST", "/cloud_backup", body)
|
||||
print(f"Created task id={result['id']} name={result['description']!r}")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(
|
||||
description="Manage TrueNAS TrueCloud Backup tasks (S3/B2/Storj)",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
p.add_argument("--host", required=True, metavar="HOST", help="TrueNAS hostname or IP")
|
||||
p.add_argument("--api-key", required=True, metavar="KEY", help="TrueNAS API key")
|
||||
p.add_argument("--insecure", action="store_true", help="Skip TLS certificate verification")
|
||||
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sub.add_parser("list-credentials", help="List configured cloud credentials")
|
||||
sub.add_parser("list-tasks", help="List TrueCloud Backup tasks")
|
||||
|
||||
c = sub.add_parser("create", help="Create a TrueCloud Backup task")
|
||||
c.add_argument("--name", required=True, help="Task description shown in the UI")
|
||||
c.add_argument("--path", required=True, help="Local dataset path, e.g. /mnt/tank/data")
|
||||
c.add_argument("--credential", required=True, type=int, metavar="ID",
|
||||
help="Cloud credential ID (from list-credentials)")
|
||||
c.add_argument("--bucket", required=True, help="Bucket or container name")
|
||||
c.add_argument("--folder", default="", help="Folder path within the bucket (default: root)")
|
||||
c.add_argument("--password", required=True, help="Restic repository encryption password")
|
||||
c.add_argument("--keep-last", type=int, default=14, metavar="N",
|
||||
help="Snapshots to retain after each run (default: 14)")
|
||||
c.add_argument("--schedule", default="0 2 * * *",
|
||||
help="Cron schedule (default: '0 2 * * *' — daily at 02:00)")
|
||||
c.add_argument("--transfer-setting",
|
||||
choices=["DEFAULT", "PERFORMANCE", "FAST_STORAGE"], default="DEFAULT")
|
||||
c.add_argument("--snapshot", action="store_true",
|
||||
help="Create a ZFS snapshot before each backup")
|
||||
c.add_argument("--absolute-paths", action="store_true",
|
||||
help="Preserve absolute paths inside the restic repo")
|
||||
c.add_argument("--disabled", action="store_true",
|
||||
help="Create the task in a disabled state")
|
||||
|
||||
args = p.parse_args()
|
||||
client = make_client(args.host, args.api_key, args.insecure)
|
||||
|
||||
dispatch = {
|
||||
"list-credentials": cmd_list_credentials,
|
||||
"list-tasks": cmd_list_tasks,
|
||||
"create": cmd_create,
|
||||
}
|
||||
dispatch[args.cmd](client, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Patches the TrueNAS webui Angular bundle to show S3 and B2 credentials
|
||||
in the TrueCloud Backup task form, instead of Storj only.
|
||||
|
||||
The compiled bundle contains:
|
||||
"filterByProviders",["STORJ_IX"]
|
||||
which is the template binding [filterByProviders]="[CloudSyncProviderName.Storj]".
|
||||
We replace the array with ["STORJ_IX","S3","B2"] so all three providers appear.
|
||||
|
||||
Run automatically by apply.sh on every boot. Safe to run multiple times.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
WEBUI_CANDIDATES = [
|
||||
"/usr/share/truenas/webui",
|
||||
"/usr/share/truenas-ui",
|
||||
"/var/www/truenas",
|
||||
]
|
||||
|
||||
# The pattern as compiled by Angular's Ivy into minified JS.
|
||||
# String literals survive minification; the function name before the comma
|
||||
# is mangled and not part of our match.
|
||||
FIND = re.compile(r'("filterByProviders",)\["STORJ_IX"\]')
|
||||
REPLACE = r'\1["STORJ_IX","S3","B2"]'
|
||||
|
||||
# Presence of this string means we already patched this file.
|
||||
MARKER = '"STORJ_IX","S3","B2"'
|
||||
|
||||
|
||||
def find_webui():
|
||||
for d in WEBUI_CANDIDATES:
|
||||
if os.path.isdir(d):
|
||||
return d
|
||||
return None
|
||||
|
||||
|
||||
def find_bundle(webui):
|
||||
for root, _, names in os.walk(webui):
|
||||
for name in names:
|
||||
if not name.endswith(".js"):
|
||||
continue
|
||||
path = os.path.join(root, name)
|
||||
try:
|
||||
with open(path) as fh:
|
||||
content = fh.read()
|
||||
if FIND.search(content):
|
||||
return path, content
|
||||
except (UnicodeDecodeError, PermissionError, OSError):
|
||||
continue
|
||||
return None, None
|
||||
|
||||
|
||||
def main():
|
||||
webui = find_webui()
|
||||
if not webui:
|
||||
print("[truecloud-patch] WARNING: webui directory not found, skipping UI patch")
|
||||
sys.exit(0)
|
||||
|
||||
path, content = find_bundle(webui)
|
||||
if path is None:
|
||||
print(
|
||||
"[truecloud-patch] WARNING: filterByProviders pattern not found in webui bundle.\n"
|
||||
"[truecloud-patch] The UI patch may need updating for this TrueNAS version.\n"
|
||||
"[truecloud-patch] File an issue at https://github.com/sudolulo/truenas-truecloud-patch"
|
||||
)
|
||||
sys.exit(0)
|
||||
|
||||
if MARKER in content:
|
||||
print(f"[truecloud-patch] UI already patched: {path}")
|
||||
sys.exit(0)
|
||||
|
||||
backup = path + ".pre-truecloud-patch"
|
||||
if not os.path.exists(backup):
|
||||
shutil.copy2(path, backup)
|
||||
|
||||
patched, count = FIND.subn(REPLACE, content)
|
||||
with open(path, "w") as fh:
|
||||
fh.write(patched)
|
||||
|
||||
print(f"[truecloud-patch] UI bundle patched ({count} replacement(s)): {path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
TrueCloud provider patch — sitecustomize.py
|
||||
Installed into Python site-packages on every boot by apply.sh.
|
||||
|
||||
Hooks the import of two middlewared modules and patches them in-place:
|
||||
|
||||
middlewared.rclone.remote.b2
|
||||
Adds get_restic_config() so the native B2 restic backend works.
|
||||
Restic URL: b2:<bucket>/<folder>
|
||||
Auth env: B2_ACCOUNT_ID, B2_ACCOUNT_KEY
|
||||
|
||||
middlewared.plugins.cloud_backup.restic
|
||||
Fixes URL construction for providers that have no hostname component
|
||||
(url == ""). Stock code builds "b2:/bucket/path" (broken double-slash);
|
||||
patched code builds "b2:bucket/path".
|
||||
|
||||
Safe for all Python processes on the system: if middlewared is absent the
|
||||
hook installs but never fires, and all errors are caught and logged to stderr.
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
def _install():
|
||||
_pending = {
|
||||
"middlewared.rclone.remote.b2",
|
||||
"middlewared.plugins.cloud_backup.restic",
|
||||
}
|
||||
_loading = set()
|
||||
|
||||
class _Hook:
|
||||
def find_module(self, fullname, path=None): # noqa: ARG002
|
||||
if fullname in _pending and fullname not in _loading:
|
||||
return self
|
||||
return None
|
||||
|
||||
def load_module(self, fullname):
|
||||
if fullname in sys.modules:
|
||||
module = sys.modules[fullname]
|
||||
else:
|
||||
_loading.add(fullname)
|
||||
try:
|
||||
__import__(fullname)
|
||||
finally:
|
||||
_loading.discard(fullname)
|
||||
module = sys.modules[fullname]
|
||||
|
||||
_pending.discard(fullname)
|
||||
|
||||
try:
|
||||
if fullname == "middlewared.rclone.remote.b2":
|
||||
_patch_b2(module)
|
||||
elif fullname == "middlewared.plugins.cloud_backup.restic":
|
||||
_patch_restic(module)
|
||||
except Exception as exc:
|
||||
sys.stderr.write(f"[truecloud-patch] patch failed for {fullname}: {exc}\n")
|
||||
|
||||
if not _pending:
|
||||
try:
|
||||
sys.meta_path.remove(hook)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return module
|
||||
|
||||
hook = _Hook()
|
||||
sys.meta_path.append(hook)
|
||||
|
||||
|
||||
def _patch_b2(module):
|
||||
cls = module.B2RcloneRemote
|
||||
|
||||
if hasattr(cls, "get_restic_config"):
|
||||
return # future TrueNAS version already added it
|
||||
|
||||
def get_restic_config(self, task):
|
||||
p = task["credentials"]["provider"]
|
||||
return "", {
|
||||
"B2_ACCOUNT_ID": p["account"],
|
||||
"B2_ACCOUNT_KEY": p["key"],
|
||||
}
|
||||
|
||||
cls.get_restic_config = get_restic_config
|
||||
cls.restic = True
|
||||
sys.stderr.write("[truecloud-patch] B2 restic support enabled\n")
|
||||
|
||||
|
||||
def _patch_restic(module):
|
||||
orig = module.get_restic_config
|
||||
if getattr(orig, "_truecloud_patched", False):
|
||||
return
|
||||
|
||||
# Capture module-level references; REMOTES is the same mutable dict
|
||||
# object that remotes.setup() will populate later.
|
||||
_REMOTES = module.REMOTES
|
||||
_get_remote_path = module.get_remote_path
|
||||
_ResticConfig = module.ResticConfig
|
||||
|
||||
def get_restic_config(cloud_backup):
|
||||
remote = _REMOTES[cloud_backup["credentials"]["provider"]["type"]]
|
||||
remote_path = _get_remote_path(remote, cloud_backup["attributes"])
|
||||
url, env = remote.get_restic_config(cloud_backup)
|
||||
|
||||
if cloud_backup["cache_path"]:
|
||||
cache = ["--cache-dir", cloud_backup["cache_path"]]
|
||||
else:
|
||||
cache = ["--no-cache"]
|
||||
|
||||
# Fix: stock code does f"{rclone_type}:{url}/{remote_path}" which
|
||||
# produces "b2:/bucket/path" when url is empty.
|
||||
repo = (
|
||||
f"{remote.rclone_type}:{url}/{remote_path}"
|
||||
if url
|
||||
else f"{remote.rclone_type}:{remote_path}"
|
||||
)
|
||||
cmd = ["restic"] + cache + ["--json", "-r", repo]
|
||||
env["RESTIC_PASSWORD"] = cloud_backup["password"]
|
||||
return _ResticConfig(cmd, env)
|
||||
|
||||
get_restic_config._truecloud_patched = True
|
||||
module.get_restic_config = get_restic_config
|
||||
sys.stderr.write("[truecloud-patch] restic URL fix applied\n")
|
||||
|
||||
|
||||
try:
|
||||
import importlib.util
|
||||
if importlib.util.find_spec("middlewared") is not None:
|
||||
_install()
|
||||
except Exception:
|
||||
pass
|
||||
Reference in New Issue
Block a user