Compatibility watch: check the patch's assumptions against every TrueNAS release line

TrueNAS 26 rewrites cloud_backup from async to sync. Every block the nested
module injects is an async wrapper around an awaited original, so on 26 it hands
sync.py a coroutine where it unpacks a tuple.

tools/compat.py records what each module assumes and checks it two ways: CI runs
it against iX's source at every release line (including master and the current
BETA) and files a bug report when an unreleased line breaks; apply.sh runs it
against the middlewared actually installed and refuses to apply a module whose
assumptions no longer hold. Stock TrueNAS without a feature beats TrueNAS with a
broken one.

Workflows run on both forges; only the release/issue API calls differ.

.github/workflows/compat.yml

@@ -0,0 +1,193 @@
+name: TrueNAS compatibility
+
+# Find out that iX broke us BEFORE their release ships, not after a user's backup
+# fails.
+#
+# This patch appends code to middlewared's internal modules. There is no stability
+# contract: TrueNAS 26 rewrote the whole cloud_backup path from async to sync, and
+# every block the nested module injects is an `async def` wrapping an `await`ed
+# original. Nobody would have found out until a restore did not work.
+#
+# tools/compat.py records what the patch assumes and checks it against iX's actual
+# source at every release line -- including master and the current BETA/RC, which is
+# where a break shows up first. When an UNRELEASED line breaks, this opens a bug
+# report so there is time to fix it before that version reaches anyone.
+#
+# Runs on both forges: Gitea (canonical) and GitHub (mirror). Only the "file an
+# issue" call differs.
+
+on:
+  schedule:
+    - cron: "17 6 * * *"     # daily, off the hour: everyone crons on the hour
+  workflow_dispatch:
+  push:
+    paths:
+      # The manifest itself changed -- re-check immediately rather than waiting a day.
+      - "tools/compat.py"
+      - ".github/workflows/compat.yml"
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  compat:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      # ONE pass over the network. Every other step renders from this JSON -- calling
+      # compat.py three times would re-fetch every file from every release line three
+      # times, and could even disagree with itself if iX pushed mid-run.
+      #
+      # The exit code is CAPTURED, not allowed to abort the step: a nonzero exit means
+      # "a shipped release is broken", which is a result to report, not a reason to
+      # die before reporting it. (Actions runs `bash -e`, so `cmd > out` followed by
+      # `echo $?` never reaches the echo.) It becomes a job failure at the end, after
+      # the bug report has been filed.
+      - name: check every TrueNAS release line
+        id: check
+        run: |
+          rc=0
+          python3 tools/compat.py --matrix --json > /tmp/matrix.json || rc=$?
+          echo "shipped_broken=$rc" >> "$GITHUB_OUTPUT"
+
+      - name: build the report
+        id: report
+        run: |
+          python3 - <<'PY' >> "$GITHUB_OUTPUT"
+          import json, sys
+
+          sys.path.insert(0, "tools")
+          import compat
+
+          with open("/tmp/matrix.json") as fh:
+              rows = json.load(fh)
+
+          with open("/tmp/matrix.md", "w") as fh:
+              fh.write(compat.render_markdown(rows))
+
+          broken = [
+              r for r in rows
+              if any(not m["ok"] and not m["native"] for m in r["modules"].values())
+          ]
+          native = [
+              r for r in rows
+              if any(m["native"] for m in r["modules"].values())
+          ]
+
+          print(f"broken={'1' if broken else '0'}")
+          print(f"refs={','.join(r['ref'] for r in broken)}")
+
+          lines = []
+          for r in broken:
+              lines.append(f"### {r['ref']}")
+              lines.append("")
+              for mod, m in sorted(r["modules"].items()):
+                  if m["ok"] or m["native"]:
+                      continue
+                  lines.append(f"**{mod}** — the patch will not apply:")
+                  lines.append("")
+                  for p in m["problems"]:
+                      lines.append(f"- `{p['id']}`: {p['detail']}")
+                      lines.append(f"  - why it matters: {p['why']}")
+                  lines.append("")
+          if native:
+              lines.append("### Native support detected")
+              lines.append("")
+              for r in native:
+                  for mod, m in sorted(r["modules"].items()):
+                      if m["native"]:
+                          lines.append(
+                              f"- `{r['ref']}`: **{mod}** appears to be native now — "
+                              f"retire the module rather than fixing it."
+                          )
+
+          # GITHUB_OUTPUT is line-based; a multi-line value needs a heredoc marker.
+          print("body<<__EOF__")
+          print("\n".join(lines))
+          print("__EOF__")
+          PY
+
+      - name: matrix
+        run: cat /tmp/matrix.md
+
+      # A broken SHIPPED release is an outage: users are on it right now.
+      - name: fail if a shipped release is broken
+        if: ${{ steps.check.outputs.shipped_broken != '0' }}
+        run: |
+          echo "::error::The patch is broken on a SHIPPED TrueNAS release."
+          exit 1
+
+      - name: file a bug report (GitHub)
+        if: ${{ steps.report.outputs.broken == '1' && contains(github.server_url, 'github.com') }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TITLE: "Incompatible with upcoming TrueNAS: ${{ steps.report.outputs.refs }}"
+        run: |
+          # One issue per set of broken refs, reopened/updated rather than duplicated
+          # daily -- a bot that files the same issue every morning gets muted, and
+          # then it is not a warning system any more.
+          existing="$(gh issue list --state all --label compat --search "$TITLE" \
+                        --json number,title,state \
+                        --jq ".[] | select(.title == \"$TITLE\") | .number" | head -1)"
+
+          {
+            echo "\`tools/compat.py\` found that the patch's assumptions no longer hold."
+            echo
+            cat /tmp/matrix.md
+            echo
+            echo "${{ steps.report.outputs.body }}"
+            echo
+            echo "_Filed automatically by \`.github/workflows/compat.yml\`._"
+          } > /tmp/issue.md
+
+          if [ -n "$existing" ]; then
+            gh issue comment "$existing" --body-file /tmp/issue.md
+            gh issue reopen "$existing" 2>/dev/null || true
+          else
+            gh issue create --title "$TITLE" --body-file /tmp/issue.md --label compat
+          fi
+
+      - name: file a bug report (Gitea)
+        if: ${{ steps.report.outputs.broken == '1' && !contains(github.server_url, 'github.com') }}
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+          TITLE: "Incompatible with upcoming TrueNAS: ${{ steps.report.outputs.refs }}"
+        run: |
+          {
+            echo "\`tools/compat.py\` found that the patch's assumptions no longer hold."
+            echo
+            cat /tmp/matrix.md
+            echo
+            echo "${{ steps.report.outputs.body }}"
+            echo
+            echo "_Filed automatically by \`.github/workflows/compat.yml\`._"
+          } > /tmp/issue.md
+
+          body="$(jq -Rs . < /tmp/issue.md)"
+          title="$(printf '%s' "$TITLE" | jq -Rs .)"
+
+          # Same title => same issue. Comment on it instead of filing a new one.
+          number="$(curl -sf -H "Authorization: token $TOKEN" \
+            "$API/issues?state=all&type=issues" \
+            | jq -r --arg t "$TITLE" '.[] | select(.title == $t) | .number' | head -1)"
+
+          if [ -n "$number" ]; then
+            curl -sS -X POST "$API/issues/$number/comments" \
+              -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
+              -d "$(printf '{"body":%s}' "$body")" -o /dev/null -w 'comment -> %{http_code}\n'
+            curl -sS -X PATCH "$API/issues/$number" \
+              -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
+              -d '{"state":"open"}' -o /dev/null -w 'reopen -> %{http_code}\n'
+          else
+            curl -sS -X POST "$API/issues" \
+              -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
+              -d "$(printf '{"title":%s,"body":%s}' "$title" "$body")" \
+              -o /dev/null -w 'create -> %{http_code}\n'
+          fi

.github/workflows/release.yml

@@ -85,32 +85,17 @@ jobs:
       - name: "gate: this commit was a release candidate"
         run: python3 tools/release_gate.py "${{ steps.tag.outputs.tag }}" -C .
 
-      # ...and the candidate has to have actually passed. Only CI can see this, so
+      # There is deliberately NO "did the candidate's CI run pass?" gate here.
-      # it cannot live in release_gate.py with the rest.
+      #
-      - name: "gate: that candidate's CI run passed"
+      # It would have to query the forge's run history, which is the one thing that
-        if: ${{ !contains(steps.tag.outputs.tag, '-rc') && !contains(steps.tag.outputs.tag, '-beta') && !contains(steps.tag.outputs.tag, '-alpha') }}
+      # differs between GitHub and Gitea -- and it adds nothing: the steps above
-        env:
+      # re-run ruff, pytest and the shell checks against the TAGGED COMMIT, and
-          GH_TOKEN: ${{ github.token }}
+      # release_gate.py has already proved a candidate points at that same commit.
-          TAG: ${{ steps.tag.outputs.tag }}
+      # If the code passes now, it passed then; they are the same code.
-        run: |
+      #
-          sha="$(git rev-list -n 1 "$TAG")"
+      # What a candidate really buys is the thing no CI can check: that a human
-          # Every rc tag on this exact commit -- release_gate.py already proved
+      # installed it on a real box and exercised it. The barrier makes room for
-          # there is at least one.
+      # that; it cannot verify it.
-          rcs="$(git tag --points-at "$sha" | grep -E -- '-rc[0-9]+$' || true)"
-
-          for rc in $rcs; do
-            concl="$(gh run list --workflow=release.yml --branch "$rc" \
-                       --json conclusion --jq '.[0].conclusion // ""' 2>/dev/null || true)"
-            echo "candidate $rc -> ${concl:-<no run found>}"
-            if [ "$concl" = "success" ]; then
-              echo "::notice::$TAG is promoting $rc, whose release run passed."
-              exit 0
-            fi
-          done
-
-          echo "::error::No release candidate on $sha has a passing release run."
-          echo "::error::Candidates found: ${rcs:-none}. Wait for CI, or cut a new one."
-          exit 1
 
       - name: extract release notes from CHANGELOG
         run: |
@@ -118,12 +103,17 @@ jobs:
           echo "--- release body ---"
           cat /tmp/notes.md
 
-      - name: create or update the release
+      # This repo is canonically hosted on Gitea (git.onetick.ninja) and mirrored to
+      # GitHub, and BOTH run this workflow -- Gitea reads .github/workflows too. So
+      # the publish step has to work on whichever forge it lands on. Everything
+      # above is forge-agnostic; only the "create a release" API differs.
+
+      - name: publish the release (GitHub)
+        if: ${{ contains(github.server_url, 'github.com') }}
         env:
           GH_TOKEN: ${{ github.token }}
           TAG: ${{ steps.tag.outputs.tag }}
         run: |
-          # Pre-1.0 and any -rc/-beta suffix ship as prereleases, not "Latest".
           prerelease=""
           case "$TAG" in
             *-rc*|*-beta*|*-alpha*) prerelease="--prerelease" ;;
@@ -134,8 +124,42 @@ jobs:
             gh release edit "$TAG" --notes-file /tmp/notes.md
           else
             # shellcheck disable=SC2086
-            gh release create "$TAG" \
+            gh release create "$TAG" --title "$TAG" --notes-file /tmp/notes.md $prerelease
-              --title "$TAG" \
+          fi
-              --notes-file /tmp/notes.md \
+
-              $prerelease
+      - name: publish the release (Gitea)
+        if: ${{ !contains(github.server_url, 'github.com') }}
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
+          TAG: ${{ steps.tag.outputs.tag }}
+          API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
+        run: |
+          prerelease=false
+          case "$TAG" in
+            *-rc*|*-beta*|*-alpha*) prerelease=true ;;
+          esac
+
+          # jq -Rs so the notes are JSON-encoded properly: the changelog is full of
+          # quotes, backticks and newlines, and hand-built JSON would mangle them.
+          body="$(jq -Rs . < /tmp/notes.md)"
+          payload="$(printf '{"tag_name":%s,"name":%s,"body":%s,"prerelease":%s}' \
+            "$(printf '%s' "$TAG" | jq -Rs .)" \
+            "$(printf '%s' "$TAG" | jq -Rs .)" \
+            "$body" "$prerelease")"
+
+          existing="$(curl -sf -H "Authorization: token $TOKEN" \
+            "$API/releases/tags/$TAG" 2>/dev/null || true)"
+
+          if [ -n "$existing" ]; then
+            id="$(printf '%s' "$existing" | jq -r .id)"
+            echo "Release $TAG exists (id=$id) — updating notes."
+            curl -sS -X PATCH "$API/releases/$id" \
+              -H "Authorization: token $TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "$payload" -o /dev/null -w 'PATCH -> %{http_code}\n'
+          else
+            curl -sS -X POST "$API/releases" \
+              -H "Authorization: token $TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "$payload" -o /dev/null -w 'POST -> %{http_code}\n'
           fi

tests/test_release_notes.py

@@ -108,9 +108,14 @@ class TestAgainstTheRealRepo:
             f"scripts say v{version} but the newest CHANGELOG entry is v{newest}"
         )
 
-    def test_check_passes_for_the_current_version(self):
+    def test_the_repo_is_always_releasable_as_a_candidate(self):
+        # Deliberately checked as an rc, not as a stable release. `main` carries
+        # work under `## Unreleased` most of the time, and the stable gate refuses
+        # that on purpose -- shipping with work stranded mid-section is how you get
+        # a release that needs another release. So the invariant main must uphold is
+        # the candidate one: versions agree, and the CHANGELOG section exists.
         version = next(iter({normalise(v) for v in script_versions(REPO).values()}))
-        assert check(version, REPO) == []
+        assert check(f"v{version}-rc1", REPO) == []
 
 
 class TestCheckCatchesMistakes: