truenas-truecloud-patch/patch/create_task.py

#!/usr/bin/env python3
"""
create_task.py — create TrueNAS TrueCloud Backup tasks with S3 or B2 credentials.

The TrueNAS UI normally restricts the credential dropdown to Storj only.
This script bypasses that restriction by talking to the TrueNAS middleware
directly via `midclt` (the /api/v2.0 REST API is removed in TrueNAS 26.04).

Compatible providers (after the truecloud-patch backend patch is applied):
  S3     — any S3-compatible endpoint (AWS, Wasabi, Cloudflare R2, MinIO, …)
  B2     — Backblaze B2 native API
  STORJ_IX — Storj (unchanged, always worked)

Run this ON the TrueNAS host — it uses the local middleware socket via `midclt`,
so no host address or API key is needed.

Examples
--------
List available cloud credentials:
    python3 create_task.py list-credentials

Create a task backed by a B2 credential (id=3):
    python3 create_task.py create \\
        --name "tank-to-b2" \\
        --path /mnt/tank/data \\
        --credential 3 \\
        --bucket my-bucket \\
        --folder backups/tank \\
        --password "restic-repo-password" \\
        --keep-last 14

Create a task using an S3-compatible credential (Wasabi, R2, etc.):
    python3 create_task.py create \\
        --name "tank-to-wasabi" \\
        --path /mnt/tank/data \\
        --credential 5 \\
        --bucket my-bucket \\
        --folder backups \\
        --password "restic-repo-password"

List existing TrueCloud Backup tasks:
    python3 create_task.py list-tasks
"""

import argparse
import calendar
import json
import os
import subprocess
import sys
import time

__version__ = "0.2.0"

_PATCH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_STATUS_FILE = os.path.join(_PATCH_DIR, "hook_status.json")


def midclt_call(method, *args):
    """Call a middleware method locally via `midclt`, the supported JSON-RPC transport
    that replaces the deprecated /api/v2.0 REST API (removed in TrueNAS 26.04). Must run
    on the TrueNAS host. Each arg is JSON-encoded (a dict for create; none for queries).
    Exits with a clear message on failure."""
    cmd = ["midclt", "call", method] + [json.dumps(a) for a in args]
    try:
        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
    except FileNotFoundError:
        print("ERROR: `midclt` not found — run this script ON the TrueNAS host.",
              file=sys.stderr)
        sys.exit(1)
    except subprocess.SubprocessError as exc:
        print(f"ERROR: midclt call failed: {exc}", file=sys.stderr)
        sys.exit(1)
    if proc.returncode != 0:
        print(f"ERROR: midclt {method}: {(proc.stderr or proc.stdout).strip()}",
              file=sys.stderr)
        sys.exit(1)
    out = proc.stdout.strip()
    return json.loads(out) if out else None


# ── Sub-commands ──────────────────────────────────────────────────────────────

def _middlewared_start_epoch():
    """Epoch timestamp of the running middlewared main process, or None."""
    try:
        pid = int(subprocess.run(
            ["systemctl", "show", "--property=MainPID", "--value", "middlewared"],
            capture_output=True, text=True, timeout=10, check=True,
        ).stdout.strip())
        if pid <= 0:
            return None
        with open(f"/proc/{pid}/stat", encoding="ascii", errors="replace") as fh:
            stat = fh.read()
        # Field 22 (starttime, in clock ticks since boot); the comm field may
        # contain spaces, so split after the closing paren.
        start_ticks = float(stat.rsplit(")", 1)[1].split()[19])
        # Base on /proc/stat btime, not uptime: starttime ticks count from the
        # kernel boot, which uptime does not match inside containers.
        with open("/proc/stat", encoding="ascii") as fh:
            btime = next(float(line.split()[1]) for line in fh
                         if line.startswith("btime "))
        return btime + start_ticks / os.sysconf("SC_CLK_TCK")
    except (OSError, ValueError, IndexError, StopIteration,
            subprocess.SubprocessError):
        return None


def cmd_verify():
    """Print the hook status written by apply.sh at boot."""
    if not os.path.exists(_STATUS_FILE):
        print("No hook status file found.")
        print("Either the patch has never loaded (middlewared not yet restarted")
        print("after install) or the status file was deleted.")
        print(f"  Expected: {_STATUS_FILE}")
        sys.exit(1)

    try:
        with open(_STATUS_FILE, encoding="utf-8") as fh:
            status = json.load(fh)
    except (OSError, json.JSONDecodeError) as exc:
        print(f"Could not read status file: {exc}")
        print(f"  Path: {_STATUS_FILE}")
        print("Try restarting middlewared to regenerate it.")
        sys.exit(1)

    print(f"Hook status (recorded at {status.get('patched_at', 'unknown')})")
    print()
    all_ok = True
    any_active = False
    for module, info in status.get("patches", {}).items():
        ok = info.get("ok", False)
        # A module can be inactive because TrueNAS now does it natively, or
        # because it is opt-in and switched off. Neither is a failure.
        active = info.get("active", True)
        label = "OK  " if ok else "FAIL"
        if ok and not active:
            label = "SKIP"
        detail = f"  — {info['detail']}" if info.get("detail") else ""
        print(f"  [{label}] {module}{detail}")
        if not ok:
            all_ok = False
        if active:
            any_active = True

    # The disk status alone can false-positive: at boot the files are patched
    # while middlewared is already running with the stock modules imported.
    # The running process only has the patch if it started AFTER patched_at.
    try:
        patched_epoch = calendar.timegm(
            time.strptime(status.get("patched_at", ""), "%Y-%m-%dT%H:%M:%SZ"))
    except ValueError:
        patched_epoch = None
    mw_start = _middlewared_start_epoch()

    proc_stale = False
    if not any_active:
        # Nothing is patched into middlewared, so whether it restarted since is
        # irrelevant -- there is nothing for it to have loaded.
        print("  [--  ] running middlewared process — no active module; nothing to load")
    elif patched_epoch is None or mw_start is None:
        print("  [??  ] running middlewared process — could not compare start time;")
        print("         the results above reflect the on-disk state only")
    elif mw_start + 2 < patched_epoch:
        proc_stale = True
        print("  [FAIL] running middlewared process — started BEFORE the patch was applied,")
        print("         so it is running the stock (unpatched) modules")
    else:
        print("  [OK  ] running middlewared process — started after the patch was applied")

    print()
    if all_ok and not proc_stale:
        print("All patches installed. Run a test backup to confirm end-to-end.")
    elif all_ok:
        print("The patch is on disk but not loaded. Right after boot, the deferred")
        print("restart (unit truecloud-mw-restart) may still be pending — re-check in a")
        print("minute. Otherwise run: systemctl restart middlewared")
        sys.exit(1)
    else:
        print("One or more patches failed to apply.")
        print(f"Check {os.path.join(_PATCH_DIR, 'apply.log')} and journalctl -u middlewared")
        sys.exit(1)

def _provider_type(cred):
    """Provider type string across schemas (<=24.10 plain str, >=25.04 dict)."""
    p = (cred or {}).get("provider")
    if isinstance(p, dict):
        return p.get("type", "?")
    return p or "?"


def cmd_list_credentials(_args):
    creds = midclt_call("cloudsync.credentials.query")
    if not creds:
        print("No cloud credentials configured.")
        return
    print(f"{'ID':>4}  {'Provider':<14}  Name")
    print("─" * 55)
    for c in sorted(creds, key=lambda x: x["id"]):
        print(f"{c['id']:>4}  {_provider_type(c):<14}  {c['name']}")


def cmd_list_tasks(_args):
    tasks = midclt_call("cloud_backup.query")
    if not tasks:
        print("No TrueCloud Backup tasks configured.")
        return
    print(f"{'ID':>4}  {'Enabled':<8}  {'Provider':<14}  Name")
    print("─" * 60)
    for t in sorted(tasks, key=lambda x: x["id"]):
        ptype = _provider_type(t.get("credentials"))
        enabled = "yes" if t.get("enabled") else "no"
        print(f"{t['id']:>4}  {enabled:<8}  {ptype:<14}  {t.get('description', '')}")


def cmd_create(args):
    parts = args.schedule.split()
    if len(parts) != 5:
        print(
            "ERROR: --schedule must be a 5-field cron expression, e.g. '0 2 * * *'",
            file=sys.stderr,
        )
        sys.exit(1)
    minute, hour, dom, month, dow = parts

    body = {
        "description": args.name,
        "path": args.path,
        "credentials": args.credential,
        "attributes": {
            "bucket": args.bucket,
            "folder": args.folder,
        },
        "password": args.password,
        "keep_last": args.keep_last,
        "transfer_setting": args.transfer_setting,
        "schedule": {
            "minute": minute,
            "hour": hour,
            "dom": dom,
            "month": month,
            "dow": dow,
        },
        "snapshot": args.snapshot,
        "absolute_paths": args.absolute_paths,
        "enabled": not args.disabled,
    }

    if args.cache_path:
        body["cache_path"] = args.cache_path
    else:
        print(
            "WARNING: no --cache-path given. TrueNAS will run restic with --no-cache, "
            "which is very slow for large repositories (it re-reads all repo metadata "
            "from the provider every run). Set --cache-path to a writable dir on a pool "
            "with free space.",
            file=sys.stderr,
        )

    result = midclt_call("cloud_backup.create", body)
    try:
        print(f"Created task id={result['id']}  name={result['description']!r}")
    except (KeyError, TypeError):
        print(f"Task created but response schema was unexpected: {result}")


# ── CLI ───────────────────────────────────────────────────────────────────────

def main():
    p = argparse.ArgumentParser(
        description="Manage TrueNAS TrueCloud Backup tasks (S3 / B2 / Storj)",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__.split("Examples")[1] if __doc__ and "Examples" in __doc__ else "",
    )
    p.add_argument("--version", "-V", action="version", version=f"truecloud-patch {__version__}")
    # Deprecated & ignored: the tool now uses the local middleware via `midclt` (the
    # /api/v2.0 REST API is removed in TrueNAS 26.04), so it must run ON the TrueNAS
    # host and needs no host/API key. Kept accepted-but-ignored for compatibility.
    p.add_argument("--host", default=None, help=argparse.SUPPRESS)
    p.add_argument("--api-key", default=None, help=argparse.SUPPRESS)
    p.add_argument("--insecure", action="store_true", help=argparse.SUPPRESS)

    sub = p.add_subparsers(dest="cmd", required=True)

    sub.add_parser("verify",           help="Check that the backend hook loaded correctly")
    sub.add_parser("list-credentials", help="List configured cloud credentials")
    sub.add_parser("list-tasks",       help="List TrueCloud Backup tasks")

    c = sub.add_parser("create", help="Create a new TrueCloud Backup task")
    c.add_argument("--name",        required=True,
                   help="Task description shown in the UI")
    c.add_argument("--path",        required=True,
                   help="Local dataset path (e.g. /mnt/tank/data)")
    c.add_argument("--credential",  required=True, type=int, metavar="ID",
                   help="Cloud credential ID — get it from list-credentials")
    c.add_argument("--bucket",      required=True,
                   help="Bucket (S3) or container (B2) name")
    c.add_argument("--folder",      default="",
                   help="Path within the bucket (default: root)")
    c.add_argument("--password",    required=True,
                   help="Restic repository encryption password (choose a strong one)")
    c.add_argument("--keep-last",   type=int, default=14, metavar="N",
                   help="Snapshots to retain after each run (default: 14)")
    c.add_argument("--schedule",    default="0 2 * * *",
                   help="Cron schedule (default: '0 2 * * *' — daily at 02:00)")
    c.add_argument("--cache-path",  default="", metavar="PATH",
                   help="restic cache directory (e.g. /mnt/pool/.restic-cache). "
                        "STRONGLY recommended: without it TrueNAS runs restic with "
                        "--no-cache, which re-fetches all repo metadata from the "
                        "provider every run and is extremely slow on large repos.")
    c.add_argument("--transfer-setting",
                   choices=["DEFAULT", "PERFORMANCE", "FAST_STORAGE"],
                   default="DEFAULT",
                   help="Pack-size / concurrency preset (default: DEFAULT)")
    c.add_argument("--snapshot",    action="store_true",
                   help="Create a ZFS snapshot before each backup run")
    c.add_argument("--absolute-paths", action="store_true",
                   help="Preserve absolute paths inside the restic repository")
    c.add_argument("--disabled",    action="store_true",
                   help="Create the task in a disabled state")

    args = p.parse_args()

    if args.cmd == "verify":
        cmd_verify()
        return

    if args.host or args.api_key or args.insecure:
        print("NOTE: --host/--api-key/--insecure are deprecated and ignored; this tool "
              "now uses the local middleware (midclt) and must run on the TrueNAS host.",
              file=sys.stderr)

    if args.cmd == "list-credentials":
        cmd_list_credentials(args)
    elif args.cmd == "list-tasks":
        cmd_list_tasks(args)
    elif args.cmd == "create":
        cmd_create(args)


if __name__ == "__main__":
    main()