flan
092bdeae29
Release-candidate tags would have been installed as stable ----------------------------------------------------------- git's version sort ranks v0.5.0-rc1 ABOVE v0.5.0 (verified empirically), and the release workflow deliberately supports rc/beta/alpha tags. update.sh would have offered an RC as "the newest release". Tag selection is now filtered to plain vX.Y.Z. update.sh would have died mid-update on an untracked file ---------------------------------------------------------- The dirty-tree guard uses --untracked-files=no, so an untracked file that the TARGET tracks slips past it -- and `git checkout` then aborts. Under set -e the script died with a raw git error, after already recording the rollback point. Not hypothetical: a hand-copied patch/wait_restart.sh blocked a pull on a real box in exactly this way. It is now detected up front, by name. Gitignored files are correctly not treated as blockers, since git overwrites those silently. Special case: if update.sh ITSELF is the blocker, it was hand-copied in to bootstrap -- and "delete update.sh, then re-run update.sh" is impossible. It now says so and prints the git commands that bootstrap it properly. --rollback skipped that check entirely and would have hit the identical failure. The check is now a shared function used by both paths, and rollback also validates that the recorded revision still exists. Also: install.sh's chmod aborted under set -e if a listed file was missing (the file set changes between versions, so --rollback must not be killed by a name this version happens to know about), and --to with no value was silently ignored. Verified end to end in a throwaway clone: forward v0.4.1 -> v0.4.2 and rollback back, with files appearing and disappearing correctly; both guards fire. 132 tests, ruff and shellcheck -S style clean.
393 lines
16 KiB
Python
Executable File
393 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
create_task.py — create TrueNAS TrueCloud Backup tasks with S3 or B2 credentials.
|
|
|
|
The TrueNAS UI normally restricts the credential dropdown to Storj only.
|
|
This script bypasses that restriction by talking to the TrueNAS middleware
|
|
directly via `midclt` (the /api/v2.0 REST API is removed in TrueNAS 26.04).
|
|
|
|
Compatible providers (after the truecloud-patch backend patch is applied):
|
|
S3 — any S3-compatible endpoint (AWS, Wasabi, Cloudflare R2, MinIO, …)
|
|
B2 — Backblaze B2 native API
|
|
STORJ_IX — Storj (unchanged, always worked)
|
|
|
|
Run this ON the TrueNAS host — it uses the local middleware socket via `midclt`,
|
|
so no host address or API key is needed.
|
|
|
|
Examples
|
|
--------
|
|
List available cloud credentials:
|
|
python3 create_task.py list-credentials
|
|
|
|
Create a task backed by a B2 credential (id=3):
|
|
python3 create_task.py create \\
|
|
--name "tank-to-b2" \\
|
|
--path /mnt/tank/data \\
|
|
--credential 3 \\
|
|
--bucket my-bucket \\
|
|
--folder backups/tank \\
|
|
--password-stdin \\
|
|
--keep-last 14
|
|
(pipe the password in: echo -n "s3cret" | python3 create_task.py create ... )
|
|
|
|
Create a task using an S3-compatible credential (Wasabi, R2, etc.):
|
|
python3 create_task.py create \\
|
|
--name "tank-to-wasabi" \\
|
|
--path /mnt/tank/data \\
|
|
--credential 5 \\
|
|
--bucket my-bucket \\
|
|
--folder backups \\
|
|
--password-stdin
|
|
|
|
List existing TrueCloud Backup tasks:
|
|
python3 create_task.py list-tasks
|
|
"""
|
|
|
|
import argparse
|
|
import calendar
|
|
import getpass
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
__version__ = "0.4.1"
|
|
|
|
_PATCH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
_STATUS_FILE = os.path.join(_PATCH_DIR, "hook_status.json")
|
|
|
|
|
|
def midclt_call(method, *args):
|
|
"""Call a middleware method on the local host.
|
|
|
|
Uses `truenas_api_client` -- the library that backs `midclt` itself -- rather
|
|
than shelling out to `midclt`.
|
|
|
|
This is a SECURITY requirement, not a style choice. `midclt call <method>
|
|
<json>` puts its arguments in the process's **argv**, and `cloud_backup.create`
|
|
carries the restic repository password. argv is world-readable via `ps`, so
|
|
shelling out would expose the key to the entire backup repo to every local
|
|
user for the duration of the call. Going through the client library keeps it
|
|
in this process's memory.
|
|
"""
|
|
try:
|
|
from truenas_api_client import Client
|
|
except ImportError:
|
|
print(
|
|
"ERROR: `truenas_api_client` not importable — run this script ON the\n"
|
|
" TrueNAS host. (It ships with midclt.)",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
|
|
try:
|
|
with Client() as client:
|
|
return client.call(method, *args)
|
|
except Exception as exc: # noqa: BLE001 - surface any middleware error verbatim
|
|
# Never echo `args` here: for cloud_backup.create it contains the password.
|
|
print(f"ERROR: {method}: {exc}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
# ── Sub-commands ──────────────────────────────────────────────────────────────
|
|
|
|
def _middlewared_start_epoch():
|
|
"""Epoch timestamp of the running middlewared main process, or None."""
|
|
try:
|
|
# Partial path (S607) is fine here: this runs as root on TrueNAS, so an
|
|
# attacker who can poison PATH already has root. Hard-coding a path would
|
|
# be less portable (/bin vs /usr/bin) for no security gain.
|
|
pid = int(subprocess.run(
|
|
["systemctl", "show", "--property=MainPID", "--value", "middlewared"], # noqa: S607
|
|
capture_output=True, text=True, timeout=10, check=True,
|
|
).stdout.strip())
|
|
if pid <= 0:
|
|
return None
|
|
with open(f"/proc/{pid}/stat", encoding="ascii", errors="replace") as fh:
|
|
stat = fh.read()
|
|
# Field 22 (starttime, in clock ticks since boot); the comm field may
|
|
# contain spaces, so split after the closing paren.
|
|
start_ticks = float(stat.rsplit(")", 1)[1].split()[19])
|
|
# Base on /proc/stat btime, not uptime: starttime ticks count from the
|
|
# kernel boot, which uptime does not match inside containers.
|
|
with open("/proc/stat", encoding="ascii") as fh:
|
|
btime = next(float(line.split()[1]) for line in fh
|
|
if line.startswith("btime "))
|
|
return btime + start_ticks / os.sysconf("SC_CLK_TCK")
|
|
except (OSError, ValueError, IndexError, StopIteration,
|
|
subprocess.SubprocessError):
|
|
return None
|
|
|
|
|
|
def cmd_verify():
|
|
"""Print the hook status written by apply.sh at boot."""
|
|
if not os.path.exists(_STATUS_FILE):
|
|
print("No hook status file found.")
|
|
print("Either the patch has never loaded (middlewared not yet restarted")
|
|
print("after install) or the status file was deleted.")
|
|
print(f" Expected: {_STATUS_FILE}")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
with open(_STATUS_FILE, encoding="utf-8") as fh:
|
|
status = json.load(fh)
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
print(f"Could not read status file: {exc}")
|
|
print(f" Path: {_STATUS_FILE}")
|
|
print("Try restarting middlewared to regenerate it.")
|
|
sys.exit(1)
|
|
|
|
print(f"Hook status (recorded at {status.get('patched_at', 'unknown')})")
|
|
print()
|
|
all_ok = True
|
|
any_active = False
|
|
for module, info in status.get("patches", {}).items():
|
|
ok = info.get("ok", False)
|
|
# A module can be inactive because TrueNAS now does it natively, or
|
|
# because it is opt-in and switched off. Neither is a failure.
|
|
active = info.get("active", True)
|
|
label = "OK " if ok else "FAIL"
|
|
if ok and not active:
|
|
label = "SKIP"
|
|
detail = f" — {info['detail']}" if info.get("detail") else ""
|
|
print(f" [{label}] {module}{detail}")
|
|
if not ok:
|
|
all_ok = False
|
|
if active:
|
|
any_active = True
|
|
|
|
# The disk status alone can false-positive: at boot the files are patched
|
|
# while middlewared is already running with the stock modules imported.
|
|
# The running process only has the patch if it started AFTER patched_at.
|
|
try:
|
|
patched_epoch = calendar.timegm(
|
|
time.strptime(status.get("patched_at", ""), "%Y-%m-%dT%H:%M:%SZ"))
|
|
except ValueError:
|
|
patched_epoch = None
|
|
mw_start = _middlewared_start_epoch()
|
|
|
|
proc_stale = False
|
|
if not any_active:
|
|
# Nothing is patched into middlewared, so whether it restarted since is
|
|
# irrelevant -- there is nothing for it to have loaded.
|
|
print(" [-- ] running middlewared process — no active module; nothing to load")
|
|
elif patched_epoch is None or mw_start is None:
|
|
print(" [?? ] running middlewared process — could not compare start time;")
|
|
print(" the results above reflect the on-disk state only")
|
|
elif mw_start + 2 < patched_epoch:
|
|
proc_stale = True
|
|
print(" [FAIL] running middlewared process — started BEFORE the patch was applied,")
|
|
print(" so it is running the stock (unpatched) modules")
|
|
else:
|
|
print(" [OK ] running middlewared process — started after the patch was applied")
|
|
|
|
print()
|
|
if all_ok and not proc_stale:
|
|
print("All patches installed. Run a test backup to confirm end-to-end.")
|
|
elif all_ok:
|
|
print("The patch is on disk but not loaded. Right after boot, the deferred")
|
|
print("restart (unit truecloud-mw-restart) may still be pending — re-check in a")
|
|
print("minute. Otherwise run: systemctl restart middlewared")
|
|
sys.exit(1)
|
|
else:
|
|
print("One or more patches failed to apply.")
|
|
print(f"Check {os.path.join(_PATCH_DIR, 'apply.log')} and journalctl -u middlewared")
|
|
sys.exit(1)
|
|
|
|
def _provider_type(cred):
|
|
"""Provider type string across schemas (<=24.10 plain str, >=25.04 dict)."""
|
|
p = (cred or {}).get("provider")
|
|
if isinstance(p, dict):
|
|
return p.get("type", "?")
|
|
return p or "?"
|
|
|
|
|
|
def cmd_list_credentials(_args):
|
|
creds = midclt_call("cloudsync.credentials.query")
|
|
if not creds:
|
|
print("No cloud credentials configured.")
|
|
return
|
|
print(f"{'ID':>4} {'Provider':<14} Name")
|
|
print("─" * 55)
|
|
for c in sorted(creds, key=lambda x: x["id"]):
|
|
print(f"{c['id']:>4} {_provider_type(c):<14} {c['name']}")
|
|
|
|
|
|
def cmd_list_tasks(_args):
|
|
tasks = midclt_call("cloud_backup.query")
|
|
if not tasks:
|
|
print("No TrueCloud Backup tasks configured.")
|
|
return
|
|
print(f"{'ID':>4} {'Enabled':<8} {'Provider':<14} Name")
|
|
print("─" * 60)
|
|
for t in sorted(tasks, key=lambda x: x["id"]):
|
|
ptype = _provider_type(t.get("credentials"))
|
|
enabled = "yes" if t.get("enabled") else "no"
|
|
print(f"{t['id']:>4} {enabled:<8} {ptype:<14} {t.get('description', '')}")
|
|
|
|
|
|
def _resolve_password(args):
|
|
"""Get the restic repo password without writing it to the user's shell history.
|
|
|
|
That password is the key to the whole backup repository. `--password <secret>`
|
|
persists it in ~/.bash_history and exposes it in `ps` for the lifetime of the
|
|
shell command, so it is accepted but warned about; stdin and an interactive
|
|
prompt are the safe paths.
|
|
"""
|
|
if args.password_stdin:
|
|
if args.password:
|
|
print("ERROR: use either --password or --password-stdin, not both.",
|
|
file=sys.stderr)
|
|
sys.exit(1)
|
|
password = sys.stdin.readline().rstrip("\n")
|
|
elif args.password:
|
|
print(
|
|
"WARNING: --password puts the restic repository password in your shell\n"
|
|
" history. Prefer: echo -n 'pw' | ... --password-stdin",
|
|
file=sys.stderr,
|
|
)
|
|
password = args.password
|
|
else:
|
|
password = getpass.getpass("Restic repository password: ")
|
|
|
|
if not password:
|
|
print("ERROR: the restic repository password must not be empty.",
|
|
file=sys.stderr)
|
|
sys.exit(1)
|
|
return password
|
|
|
|
|
|
def cmd_create(args):
|
|
parts = args.schedule.split()
|
|
if len(parts) != 5:
|
|
print(
|
|
"ERROR: --schedule must be a 5-field cron expression, e.g. '0 2 * * *'",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
minute, hour, dom, month, dow = parts
|
|
|
|
password = _resolve_password(args)
|
|
|
|
body = {
|
|
"description": args.name,
|
|
"path": args.path,
|
|
"credentials": args.credential,
|
|
"attributes": {
|
|
"bucket": args.bucket,
|
|
"folder": args.folder,
|
|
},
|
|
"password": password,
|
|
"keep_last": args.keep_last,
|
|
"transfer_setting": args.transfer_setting,
|
|
"schedule": {
|
|
"minute": minute,
|
|
"hour": hour,
|
|
"dom": dom,
|
|
"month": month,
|
|
"dow": dow,
|
|
},
|
|
"snapshot": args.snapshot,
|
|
"absolute_paths": args.absolute_paths,
|
|
"enabled": not args.disabled,
|
|
}
|
|
|
|
if args.cache_path:
|
|
body["cache_path"] = args.cache_path
|
|
else:
|
|
print(
|
|
"WARNING: no --cache-path given. TrueNAS will run restic with --no-cache, "
|
|
"which is very slow for large repositories (it re-reads all repo metadata "
|
|
"from the provider every run). Set --cache-path to a writable dir on a pool "
|
|
"with free space.",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
result = midclt_call("cloud_backup.create", body)
|
|
try:
|
|
print(f"Created task id={result['id']} name={result['description']!r}")
|
|
except (KeyError, TypeError):
|
|
print(f"Task created but response schema was unexpected: {result}")
|
|
|
|
|
|
# ── CLI ───────────────────────────────────────────────────────────────────────
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(
|
|
description="Manage TrueNAS TrueCloud Backup tasks (S3 / B2 / Storj)",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog=__doc__.split("Examples")[1] if __doc__ and "Examples" in __doc__ else "",
|
|
)
|
|
p.add_argument("--version", "-V", action="version", version=f"truecloud-patch {__version__}")
|
|
# Deprecated & ignored: the tool now uses the local middleware via `midclt` (the
|
|
# /api/v2.0 REST API is removed in TrueNAS 26.04), so it must run ON the TrueNAS
|
|
# host and needs no host/API key. Kept accepted-but-ignored for compatibility.
|
|
p.add_argument("--host", default=None, help=argparse.SUPPRESS)
|
|
p.add_argument("--api-key", default=None, help=argparse.SUPPRESS)
|
|
p.add_argument("--insecure", action="store_true", help=argparse.SUPPRESS)
|
|
|
|
sub = p.add_subparsers(dest="cmd", required=True)
|
|
|
|
sub.add_parser("verify", help="Check that the backend hook loaded correctly")
|
|
sub.add_parser("list-credentials", help="List configured cloud credentials")
|
|
sub.add_parser("list-tasks", help="List TrueCloud Backup tasks")
|
|
|
|
c = sub.add_parser("create", help="Create a new TrueCloud Backup task")
|
|
c.add_argument("--name", required=True,
|
|
help="Task description shown in the UI")
|
|
c.add_argument("--path", required=True,
|
|
help="Local dataset path (e.g. /mnt/tank/data)")
|
|
c.add_argument("--credential", required=True, type=int, metavar="ID",
|
|
help="Cloud credential ID — get it from list-credentials")
|
|
c.add_argument("--bucket", required=True,
|
|
help="Bucket (S3) or container (B2) name")
|
|
c.add_argument("--folder", default="",
|
|
help="Path within the bucket (default: root)")
|
|
c.add_argument("--password", default=None,
|
|
help="Restic repository password. UNSAFE: it lands in your shell "
|
|
"history. Prefer --password-stdin, or omit both and be prompted.")
|
|
c.add_argument("--password-stdin", action="store_true",
|
|
help="Read the restic repository password from stdin (recommended)")
|
|
c.add_argument("--keep-last", type=int, default=14, metavar="N",
|
|
help="Snapshots to retain after each run (default: 14)")
|
|
c.add_argument("--schedule", default="0 2 * * *",
|
|
help="Cron schedule (default: '0 2 * * *' — daily at 02:00)")
|
|
c.add_argument("--cache-path", default="", metavar="PATH",
|
|
help="restic cache directory (e.g. /mnt/pool/.restic-cache). "
|
|
"STRONGLY recommended: without it TrueNAS runs restic with "
|
|
"--no-cache, which re-fetches all repo metadata from the "
|
|
"provider every run and is extremely slow on large repos.")
|
|
c.add_argument("--transfer-setting",
|
|
choices=["DEFAULT", "PERFORMANCE", "FAST_STORAGE"],
|
|
default="DEFAULT",
|
|
help="Pack-size / concurrency preset (default: DEFAULT)")
|
|
c.add_argument("--snapshot", action="store_true",
|
|
help="Create a ZFS snapshot before each backup run")
|
|
c.add_argument("--absolute-paths", action="store_true",
|
|
help="Preserve absolute paths inside the restic repository")
|
|
c.add_argument("--disabled", action="store_true",
|
|
help="Create the task in a disabled state")
|
|
|
|
args = p.parse_args()
|
|
|
|
if args.cmd == "verify":
|
|
cmd_verify()
|
|
return
|
|
|
|
if args.host or args.api_key or args.insecure:
|
|
print("NOTE: --host/--api-key/--insecure are deprecated and ignored; this tool "
|
|
"now uses the local middleware (midclt) and must run on the TrueNAS host.",
|
|
file=sys.stderr)
|
|
|
|
if args.cmd == "list-credentials":
|
|
cmd_list_credentials(args)
|
|
elif args.cmd == "list-tasks":
|
|
cmd_list_tasks(args)
|
|
elif args.cmd == "create":
|
|
cmd_create(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|