diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 099c4b4..17d113d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,9 +9,31 @@ on: permissions: contents: read +# ONE job, deliberately. This was four (shell + a 3-way python matrix) and they +# started within the same second on the self-hosted Gitea runner, which is what +# made CI unreliable in two separate ways: +# +# 1. The act action-cache race. `act` caches each ACTION as a single shared git +# clone under /root/.cache/act/ and re-pulls it per job, so concurrent +# jobs using the same action fight over that directory and the loser dies +# with `lstat /root/.cache/act//: no such file or directory` -- +# a red `main` with zero suite output, and a different victim each push +# (3.12 on one, 3.11 on the next). Dropping one action only shrank the +# surface: every job still used actions/checkout. Concurrency is the actual +# ingredient, so removing it removes the whole class -- a single job cannot +# race itself, no matter which actions it uses. +# +# 2. Docker Hub 429s. The runner force-pulls its base image per job, so four +# jobs meant four anonymous pulls per push. A few pushes and re-runs in an +# afternoon exhausted the anonymous limit and every job failed before it +# started -- including the shell job, which nothing had touched. One job is +# one pull. +# +# The cost is wall-clock parallelism, and this repo does not need it: the suite +# is ~1.5s, so container start and interpreter downloads dominate either way. jobs: - shell: - name: shell (shellcheck + syntax) + ci: + name: ci (shell + python 3.11-3.13) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -31,36 +53,15 @@ jobs: env: SHELLCHECK_OPTS: -S warning -e SC1091 - python: - name: python ${{ matrix.python }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - # TrueNAS SCALE middleware runs 3.11+; keep the patch importable across - # the versions it may be injected into. - python: ["3.11", "3.12", "3.13"] - steps: - - uses: actions/checkout@v4 - # uv-managed interpreters instead of actions/setup-python: the prebuilt-CPython # download path setup-python relies on does not work on the self-hosted Gitea # runner (all three matrix jobs failed at setup there while passing on GitHub); # uv works identically on both. # - # Installed by a plain `run:` step rather than astral-sh/setup-uv, because - # act -- the engine behind the Gitea runner -- caches each ACTION as a single - # shared git clone under /root/.cache/act/ and re-pulls it per job. The - # three matrix jobs start within the same second on one runner, so they race - # on that directory and whichever loses dies with - # - # lstat /root/.cache/act//.npmrc: no such file or directory - # - # before a single test runs -- a red `main` with zero suite output, and a - # different victim each time (3.12 on one push, 3.11 on the next). A `run:` - # step has no action-cache entry, so it cannot race. Serialising the matrix - # would have been the other option; it costs 3x the wall clock and still - # leaves actions/checkout sharing a cache across the four jobs. + # Installed by a plain `run:` step rather than astral-sh/setup-uv: one fewer + # action is one fewer thing to go wrong, and the action was only ever + # fetching a binary -- the interpreter is chosen per command by `uvx + # --python`, never by the action. # # Pinned for the same reason ruff is pinned below: an unpinned uv means any # upstream release can turn main red with no code change here. @@ -76,10 +77,33 @@ jobs: # with no code change. run: uvx ruff@0.16.1 check patch tests tools + # TrueNAS SCALE middleware runs 3.11+; keep the patch importable across the + # versions it may be injected into. Every version runs even after one + # fails -- that is what `fail-fast: false` bought when this was a matrix, + # and losing it would mean a 3.11 break hides whether 3.12 and 3.13 are + # fine, which is exactly the information you want at that moment. - name: pytest - run: uvx --python ${{ matrix.python }} pytest tests -v + env: + PYTHONS: "3.11 3.12 3.13" + run: | + fail=0 + for v in $PYTHONS; do + echo "::group::pytest on python $v" + uvx --python "$v" pytest tests -v \ + || { echo "::error::suite failed on python $v"; fail=1; } + echo "::endgroup::" + done + exit $fail - name: verify injected middleware blocks compile # Belt-and-braces: the *_BLOCK strings are appended into live middlewared # modules. A syntax error there would break the box at boot. - run: uvx --python ${{ matrix.python }} pytest tests/test_apply_blocks.py -v + env: + PYTHONS: "3.11 3.12 3.13" + run: | + fail=0 + for v in $PYTHONS; do + uvx --python "$v" pytest tests/test_apply_blocks.py -v \ + || { echo "::error::injected blocks failed to compile on python $v"; fail=1; } + done + exit $fail diff --git a/CHANGELOG.md b/CHANGELOG.md index cb7cb5a..bb97dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,23 +39,33 @@ worse than no alert, because one day it carries a security fix. ### Fixed -- **CI turned `main` red on two of three pushes without running a single test.** - `act`, the engine behind the self-hosted Gitea runner, caches each *action* as - one shared git clone under `/root/.cache/act/` and re-pulls it per job. - The three matrix jobs start within the same second on one runner, so they race - on that directory and whichever loses dies with `lstat - /root/.cache/act//.npmrc: no such file or directory` — before any suite - output exists, with a different victim each push (3.12 on one, 3.11 on the - next). A red gate that is usually noise is worse than no gate, because the one - time it means something nobody looks. +- **CI turned `main` red on two of three pushes without running a single test, + then stopped running at all.** Two failures, one cause: four concurrent jobs on + one self-hosted runner. - `uv` is now installed by a plain `run:` step instead of `astral-sh/setup-uv`. - A `run:` step has no action-cache entry and cannot race, and the action was - only ever fetching a binary — the matrix interpreter is chosen per command by - `uvx --python`, not by the action. Serialising the matrix was the alternative; - it costs 3x the wall clock and still leaves `actions/checkout` shared across - the four jobs. The uv version is pinned under the same rule as ruff: an - unpinned tool lets an upstream release turn `main` red with no change here. + `act`, the engine behind the Gitea runner, caches each *action* as one shared + git clone under `/root/.cache/act/` and re-pulls it per job. Jobs + starting in the same second fight over that directory, and the loser dies with + `lstat /root/.cache/act//.npmrc: no such file or directory` — before any + suite output exists, with a different victim each push (3.12 on one, 3.11 on + the next). Separately, the runner force-pulls its base image per job, so four + jobs meant four anonymous Docker Hub pulls per push; a few pushes and re-runs + in one afternoon hit `429 Too Many Requests` and *every* job began failing + before it started — including the shell job, which nothing had touched. + + CI is now a single job. Dropping one action (uv is installed by a `run:` step + rather than `astral-sh/setup-uv`, which was only ever fetching a binary) only + shrank the surface, because every job still used `actions/checkout`. + Concurrency is the actual ingredient, so removing it removes the whole class: + one job cannot race itself whatever actions it uses, and one job is one pull. + The Python sweep moved inside that job and still runs every version after one + fails — that is what `fail-fast: false` bought, and losing it would mean a 3.11 + break hides whether 3.12 and 3.13 are fine. The cost is wall-clock + parallelism, which this repo does not need: the suite is ~1.5s, so container + start and interpreter downloads dominate either way. + + A red gate that is usually noise is worse than no gate, because the one time + it means something, nobody looks. - **The patch survived being applied and then silently stopped existing, because something else remounted `/usr` four seconds later.** On a box running diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 38f9c59..03c81f6 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -175,15 +175,56 @@ class TestCompatCannotSilentlyPass: class TestActionCacheRace: - """act caches each ACTION as one shared clone and re-pulls it per job. + """CI must not run concurrent jobs on the self-hosted runner. - Matrix jobs start within the same second on the self-hosted Gitea runner, so - they race on `/root/.cache/act/` and the loser dies with `lstat - .../: no such file or directory` before any test runs -- a red `main` - with zero suite output and a different victim each push. Fewer actions in a - fan-out job means fewer directories to race on. + `act` caches each ACTION as one shared clone under /root/.cache/act/ + and re-pulls it per job, so jobs starting together fight over that directory + and the loser dies with `lstat .../: no such file or directory` before + any test runs -- a red `main` with zero suite output and a different victim + each push. The runner also force-pulls its base image per job, so job count + is also Docker Hub pull count, and four-per-push exhausted the anonymous + limit in an afternoon. Both problems have the same cure: one job. """ + def _ci(self): + with open(os.path.join(WORKFLOWS, "ci.yml"), encoding="utf-8") as fh: + return fh.read() + + def test_ci_runs_as_exactly_one_job(self): + """The fix is the absence of concurrency, not the absence of one action. + + Dropping astral-sh/setup-uv only shrank the surface -- every job still + used actions/checkout. A single job cannot race itself whatever actions + it uses, which is why this, and not the action count, is the invariant. + """ + ci = self._ci() + # Scope to the jobs: block -- `on:` has two-space keys of its own + # (push/pull_request/workflow_dispatch) that look identical otherwise. + body = ci[ci.index("\njobs:"):] + jobs = re.findall(r"^ (\w[\w-]*):$", body, re.M) + assert len(jobs) == 1, ( + f"ci.yml defines {len(jobs)} jobs ({jobs}); concurrent jobs on the " + "self-hosted runner race on act's shared action cache and multiply " + "Docker Hub pulls. Keep CI to one job." + ) + + def test_no_matrix_reintroduces_parallel_jobs(self): + ci = self._ci() + assert "strategy:" not in ci and "matrix:" not in ci, ( + "a matrix fans out into concurrent jobs again -- sweep versions " + "inside one job instead" + ) + + def test_every_python_version_still_runs_after_one_fails(self): + """`fail-fast: false` is what the loop has to preserve. + + A 3.11 break must not hide whether 3.12 and 3.13 are fine; that is + precisely the information you want at that moment. + """ + ci = self._ci() + assert 'PYTHONS: "3.11 3.12 3.13"' in ci + assert ci.count("fail=1") >= 2, "the sweeps must collect failures, not exit early" + def test_uv_is_installed_without_an_action(self): """Checks `uses:` directives, not prose. @@ -194,21 +235,18 @@ class TestActionCacheRace: environment holds pytest and nothing else, so a third-party import here fails on the runner while passing locally. """ - with open(os.path.join(WORKFLOWS, "ci.yml"), encoding="utf-8") as fh: - ci = fh.read() + ci = self._ci() used = re.findall(r"^\s*-?\s*uses:\s*(\S+)", ci, re.M) assert not [u for u in used if "setup-uv" in u], ( - "installing uv via an action reintroduces the act action-cache race " - "that turned main red on two of three pushes; install it in a run: step" + "the action was only fetching a binary; a run: step does the same " + "with one less moving part" ) assert "astral.sh/uv/" in ci def test_the_uv_version_is_pinned(self): - with open(os.path.join(WORKFLOWS, "ci.yml"), encoding="utf-8") as fh: - ci = fh.read() + ci = self._ci() assert re.search(r'UV_VERSION:\s*"\d+\.\d+\.\d+"', ci), ( "an unpinned uv lets any upstream release turn main red with no " "code change here -- the same rule ruff is pinned under" ) - # The version must be used, not just declared. - assert 'https://astral.sh/uv/${UV_VERSION}/install.sh' in ci + assert "https://astral.sh/uv/${UV_VERSION}/install.sh" in ci