From 2c9adc1ce5f9dc7775dcf25d5684abdc2fd8b426 Mon Sep 17 00:00:00 2001 From: sudolulo Date: Mon, 15 Jun 2026 03:23:31 +0000 Subject: [PATCH] Fix five quality findings; expand recovery and restore documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit patch_ui.py: - Write Angular bundle atomically via tmp + os.replace, matching the pattern already used by _record_status. Prevents a corrupt bundle if the write is interrupted mid-boot. sitecustomize.py: - Tighten _record_status count barrier comment to name _Finder._targets as the canonical count, making the coupling visible to future editors. uninstall.sh: - Verify middlewared restarted cleanly after uninstall, with journalctl guidance on failure — matching recover.sh's existing pattern. apply.sh: - Change second line of site-packages error block from WARNING: prefix (misleading for an instructional message) to a plain Run: hint. create_task.py: - Guard __doc__ against None in epilog extraction so -OO does not crash. README.md: - Split Emergency recovery into three named subsections: middlewared won't start, web UI is blank or broken (corrupt bundle recovery), and backend verify shows FAIL. Each gives direct commands and escalation steps. - Add Restoring from a TrueCloud Backup section: finding the restic binary, gathering credentials, provider-specific env var setup for B2 and S3, listing and restoring snapshots, and operational notes on restore hygiene. - Clarify that hook_status.json is written once both target modules have loaded (not necessarily at the instant middlewared starts). --- README.md | 171 ++++++++++++++++++++++++++++++++++++++--- patch/apply.sh | 2 +- patch/create_task.py | 2 +- patch/patch_ui.py | 8 +- patch/sitecustomize.py | 2 +- uninstall.sh | 13 +++- 6 files changed, 182 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0ad824d..373dfa4 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,100 @@ restart. --- +## Restoring from a TrueCloud Backup + +TrueCloud Backup uses [restic](https://restic.net/) under the hood. Restores +are done with the `restic` command directly — TrueNAS does not yet expose a +restore UI for TrueCloud Backup tasks. + +### 1. Find the restic binary + +```bash +which restic 2>/dev/null || find /usr -name restic -type f 2>/dev/null | head -1 +``` + +Use that path in the commands below (referred to as `restic`). + +### 2. Gather your repository details + +You need three things from the task you created: + +| Detail | Where to find it | +|---|---| +| **Bucket** and **folder** | TrueNAS UI → Data Protection → TrueCloud Backup → edit the task → Attributes | +| **Credentials** (key ID + secret) | TrueNAS UI → Credentials → Backup Credentials → edit the credential | +| **Repository password** | The `--password` value you supplied when creating the task | + +### 3. Set environment variables + +**Backblaze B2:** +```bash +export B2_ACCOUNT_ID="your-key-id" +export B2_ACCOUNT_KEY="your-application-key" +export RESTIC_PASSWORD="your-repo-password" +REPO="b2:your-bucket/your-folder" +``` + +**S3-compatible (AWS S3, Wasabi, Cloudflare R2, MinIO, etc.):** +```bash +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +export RESTIC_PASSWORD="your-repo-password" +# Use just the hostname as the endpoint — no https:// prefix: +REPO="s3:s3.wasabisys.com/your-bucket/your-folder" # Wasabi example +# REPO="s3:s3.amazonaws.com/your-bucket/your-folder" # AWS S3 +# REPO="s3:.r2.cloudflarestorage.com/your-bucket/your-folder" # R2 +``` + +### 4. List snapshots + +```bash +restic -r "$REPO" snapshots +``` + +Output example: +``` +ID Time Host Tags Paths +────────────────────────────────────────────────────────── +a1b2c3d4 2026-06-01 02:00:05 truenas /mnt/tank/data +e5f6a7b8 2026-06-08 02:00:07 truenas /mnt/tank/data +``` + +### 5. Restore files + +**Restore everything from the latest snapshot to a temporary location:** +```bash +restic -r "$REPO" restore latest --target /mnt/tank/restore-tmp +``` + +**Restore a specific snapshot by ID:** +```bash +restic -r "$REPO" restore a1b2c3d4 --target /mnt/tank/restore-tmp +``` + +**Restore only specific paths from within a snapshot:** +```bash +restic -r "$REPO" restore latest \ + --include /mnt/tank/data/important-dir \ + --target /mnt/tank/restore-tmp +``` + +**Browse a snapshot without extracting (useful for finding the right file):** +```bash +restic -r "$REPO" ls latest +``` + +### 6. Notes + +- Restore to a **different path** first, then move files into place after + verifying. Restoring directly over a live dataset can cause data loss if + the snapshot is incomplete or from the wrong point in time. +- If you created the task with `--snapshot` (ZFS snapshot before each run), + the restic snapshot captures the dataset at a consistent point in time. +- Use `restic check -r "$REPO"` periodically to verify repository integrity. + +--- + ## After a TrueNAS update 1. Check the log: `cat /data/truecloud-patch/apply.log | tail -30` @@ -169,17 +263,19 @@ restart. ## Emergency recovery -If middlewared stops starting after installing this patch, run this from the -TrueNAS shell (local console, SSH, or the debug shell in the UI): +### middlewared won't start + +Run this from the TrueNAS shell (local console, SSH, or the debug shell in +the UI): ```bash bash /data/truecloud-patch/recover.sh ``` -That creates a kill-switch file (`/data/truecloud-patch/disabled`) that -`sitecustomize.py` checks at startup. With the switch set, the import hook is -skipped entirely and middlewared starts clean. Your system returns to -Storj-only TrueCloud Backup — nothing else is affected. +This creates a kill-switch file (`/data/truecloud-patch/disabled`). +`sitecustomize.py` checks for it at Python startup; if present, the import +hook is skipped entirely and middlewared starts clean with Storj-only support. +Nothing else on your system is affected. If you cannot run a script and only have a bare shell prompt: @@ -188,7 +284,14 @@ touch /data/truecloud-patch/disabled systemctl restart middlewared ``` -To re-enable the patch after investigating: +If middlewared **still** won't start after the kill switch is set, the problem +is unrelated to this patch. Check: + +```bash +journalctl -u middlewared -n 50 +``` + +To re-enable the patch once you have investigated: ```bash rm /data/truecloud-patch/disabled @@ -197,6 +300,53 @@ bash /data/truecloud-patch/apply.sh --- +### Web UI is blank or broken + +If the TrueNAS web interface loads blank or shows JavaScript errors, the +Angular bundle may have been interrupted mid-write (e.g. power cut during +boot). The original bundle is always backed up before patching, so recovery +is straightforward: + +```bash +# Find the backup (the path varies by TrueNAS version): +find /usr/share/truenas /var/www/truenas -name "*.js.pre-truecloud-patch" 2>/dev/null + +# Restore it — substitute the actual path from the find output: +mv /usr/share/truenas/webui/main.XXXXXXXX.js.pre-truecloud-patch \ + /usr/share/truenas/webui/main.XXXXXXXX.js +``` + +Refresh your browser. The UI will return to normal (Storj-only until the +patch re-runs at next reboot, or you run `bash /data/truecloud-patch/apply.sh` +manually). + +--- + +### Backend verify shows FAIL + +```bash +python3 /data/truecloud-patch/create_task.py verify +``` + +If one or more entries show `[FAIL]`: + +1. **Check the apply log** for errors during the last boot: + ```bash + cat /data/truecloud-patch/apply.log | tail -40 + ``` +2. **Check middlewared's own log** for Python tracebacks: + ```bash + journalctl -u middlewared -n 50 + ``` +3. **A FAIL is non-fatal.** middlewared runs normally; the affected provider + falls back to Storj-only. Your existing backups are not at risk. +4. **If the detail says the module doesn't exist**, a TrueNAS update renamed + or restructured the internal API. + [Open an issue](https://github.com/sudolulo/truenas-truecloud-patch/issues) + with your TrueNAS version number and the full verify output. + +--- + ## Troubleshooting **Apply log** (check after each reboot or install): @@ -208,8 +358,11 @@ cat /data/truecloud-patch/apply.log ```bash python3 /data/truecloud-patch/create_task.py verify ``` -This reads `/data/truecloud-patch/hook_status.json`, written by the import hook -at middlewared startup. It shows which patches applied and any failure details. +This reads `/data/truecloud-patch/hook_status.json`, written by the import +hook once both target modules have been loaded by middlewared. If it reports +"No hook status file found" immediately after install, restart middlewared +and try again — the file is written when middlewared imports the relevant +modules, which happens at service start. Does not require `--host` or `--api-key`. **Verify the UI patch** (should print your TrueNAS version): diff --git a/patch/apply.sh b/patch/apply.sh index fb9fa41..b39f590 100755 --- a/patch/apply.sh +++ b/patch/apply.sh @@ -77,7 +77,7 @@ SITE_PKG=$("$PYTHON" -c "import site; print(site.getsitepackages()[0])" 2>/dev/n if [ -z "$SITE_PKG" ]; then echo "WARNING: Cannot determine site-packages directory; skipping backend patch." - echo "WARNING: Verify that '$PYTHON -c \"import site; print(site.getsitepackages())\"' works." + echo " Run: $PYTHON -c \"import site; print(site.getsitepackages())\"" else # Back up any pre-existing sitecustomize.py that isn't ours. if [ -f "$SITE_PKG/sitecustomize.py" ] && \ diff --git a/patch/create_task.py b/patch/create_task.py index 2839e69..6e26683 100755 --- a/patch/create_task.py +++ b/patch/create_task.py @@ -188,7 +188,7 @@ def main(): p = argparse.ArgumentParser( description="Manage TrueNAS TrueCloud Backup tasks (S3 / B2 / Storj)", formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=__doc__.split("Examples")[1] if "Examples" in __doc__ else "", + epilog=__doc__.split("Examples")[1] if __doc__ and "Examples" in __doc__ else "", ) p.add_argument("--host", default=None, metavar="HOST", help="TrueNAS hostname or IP address (required except for verify)") diff --git a/patch/patch_ui.py b/patch/patch_ui.py index 79efb69..d6491b5 100644 --- a/patch/patch_ui.py +++ b/patch/patch_ui.py @@ -109,11 +109,17 @@ def main(): patched, count = FIND.subn(REPLACE, content) + tmp = path + ".tmp" try: - with open(path, "w", encoding="utf-8") as fh: + with open(tmp, "w", encoding="utf-8") as fh: fh.write(patched) + os.replace(tmp, path) except OSError as exc: print(f"[truecloud-patch] ERROR: Could not write {path}: {exc}") + try: + os.unlink(tmp) + except OSError: + pass return print(f"[truecloud-patch] UI bundle patched ({count} replacement(s)): {path}") diff --git a/patch/sitecustomize.py b/patch/sitecustomize.py index 938aea8..b5ab4b0 100644 --- a/patch/sitecustomize.py +++ b/patch/sitecustomize.py @@ -189,7 +189,7 @@ def _record_status(fullname: str, ok: bool, detail: str = "") -> None: return # idempotent: first call wins _hook_status[fullname] = {"ok": ok, "detail": detail} if len(_hook_status) < len(_Finder._targets): - return # wait until all patches have reported before writing + return # wait for all patches; _Finder._targets is the canonical count payload = { "patched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), diff --git a/uninstall.sh b/uninstall.sh index 431cfd7..9f8cb2e 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -106,6 +106,13 @@ fi echo "" echo "Restarting middlewared ..." -systemctl restart middlewared -echo "" -echo "Uninstall complete. Refresh your browser to see the restored UI." +if systemctl restart middlewared; then + echo "" + echo "Uninstall complete. Refresh your browser to see the restored UI." +else + echo "" + echo "WARNING: middlewared did not start cleanly after uninstall." + echo "Check the system log for details:" + echo " journalctl -u middlewared -n 50" + exit 1 +fi