From 14ef25e3c6b5d3f54a1f8193f74b75a6c3c9d224 Mon Sep 17 00:00:00 2001 From: sudolulo Date: Wed, 26 Aug 2026 05:34:11 +0000 Subject: [PATCH] Run CI as a single job so concurrent jobs cannot race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CI failures shared one cause: four jobs starting together on the self-hosted runner. act caches each action as one shared clone under /root/.cache/act/ and re-pulls it per job, so concurrent jobs fight over that directory and the loser dies with "lstat .../: no such file or directory" before any test runs — a different victim each push. And the runner force-pulls its base image per job, so four jobs meant four anonymous Docker Hub pulls per push; that hit 429 Too Many Requests and every job started failing before it began, including the shell job nothing had touched. Installing uv without an action only shrank the surface, since every job still used actions/checkout. Concurrency is the ingredient, so this removes it: one job cannot race itself whatever actions it uses, and one job is one pull. The version sweep moves inside the job and still runs every version after one fails, preserving what fail-fast: false bought. --- .github/workflows/ci.yml | 82 ++++++++++++++++++++++++++-------------- CHANGELOG.md | 42 ++++++++++++-------- tests/test_workflows.py | 66 +++++++++++++++++++++++++------- 3 files changed, 131 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 099c4b4..17d113d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,9 +9,31 @@ on: permissions: contents: read +# ONE job, deliberately. This was four (shell + a 3-way python matrix) and they +# started within the same second on the self-hosted Gitea runner, which is what +# made CI unreliable in two separate ways: +# +# 1. The act action-cache race. `act` caches each ACTION as a single shared git +# clone under /root/.cache/act/ and re-pulls it per job, so concurrent +# jobs using the same action fight over that directory and the loser dies +# with `lstat /root/.cache/act//: no such file or directory` -- +# a red `main` with zero suite output, and a different victim each push +# (3.12 on one, 3.11 on the next). Dropping one action only shrank the +# surface: every job still used actions/checkout. Concurrency is the actual +# ingredient, so removing it removes the whole class -- a single job cannot +# race itself, no matter which actions it uses. +# +# 2. Docker Hub 429s. The runner force-pulls its base image per job, so four +# jobs meant four anonymous pulls per push. A few pushes and re-runs in an +# afternoon exhausted the anonymous limit and every job failed before it +# started -- including the shell job, which nothing had touched. One job is +# one pull. +# +# The cost is wall-clock parallelism, and this repo does not need it: the suite +# is ~1.5s, so container start and interpreter downloads dominate either way. jobs: - shell: - name: shell (shellcheck + syntax) + ci: + name: ci (shell + python 3.11-3.13) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -31,36 +53,15 @@ jobs: env: SHELLCHECK_OPTS: -S warning -e SC1091 - python: - name: python ${{ matrix.python }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - # TrueNAS SCALE middleware runs 3.11+; keep the patch importable across - # the versions it may be injected into. - python: ["3.11", "3.12", "3.13"] - steps: - - uses: actions/checkout@v4 - # uv-managed interpreters instead of actions/setup-python: the prebuilt-CPython # download path setup-python relies on does not work on the self-hosted Gitea # runner (all three matrix jobs failed at setup there while passing on GitHub); # uv works identically on both. # - # Installed by a plain `run:` step rather than astral-sh/setup-uv, because - # act -- the engine behind the Gitea runner -- caches each ACTION as a single - # shared git clone under /root/.cache/act/ and re-pulls it per job. The - # three matrix jobs start within the same second on one runner, so they race - # on that directory and whichever loses dies with - # - # lstat /root/.cache/act//.npmrc: no such file or directory - # - # before a single test runs -- a red `main` with zero suite output, and a - # different victim each time (3.12 on one push, 3.11 on the next). A `run:` - # step has no action-cache entry, so it cannot race. Serialising the matrix - # would have been the other option; it costs 3x the wall clock and still - # leaves actions/checkout sharing a cache across the four jobs. + # Installed by a plain `run:` step rather than astral-sh/setup-uv: one fewer + # action is one fewer thing to go wrong, and the action was only ever + # fetching a binary -- the interpreter is chosen per command by `uvx + # --python`, never by the action. # # Pinned for the same reason ruff is pinned below: an unpinned uv means any # upstream release can turn main red with no code change here. @@ -76,10 +77,33 @@ jobs: # with no code change. run: uvx ruff@0.16.1 check patch tests tools + # TrueNAS SCALE middleware runs 3.11+; keep the patch importable across the + # versions it may be injected into. Every version runs even after one + # fails -- that is what `fail-fast: false` bought when this was a matrix, + # and losing it would mean a 3.11 break hides whether 3.12 and 3.13 are + # fine, which is exactly the information you want at that moment. - name: pytest - run: uvx --python ${{ matrix.python }} pytest tests -v + env: + PYTHONS: "3.11 3.12 3.13" + run: | + fail=0 + for v in $PYTHONS; do + echo "::group::pytest on python $v" + uvx --python "$v" pytest tests -v \ + || { echo "::error::suite failed on python $v"; fail=1; } + echo "::endgroup::" + done + exit $fail - name: verify injected middleware blocks compile # Belt-and-braces: the *_BLOCK strings are appended into live middlewared # modules. A syntax error there would break the box at boot. - run: uvx --python ${{ matrix.python }} pytest tests/test_apply_blocks.py -v + env: + PYTHONS: "3.11 3.12 3.13" + run: | + fail=0 + for v in $PYTHONS; do + uvx --python "$v" pytest tests/test_apply_blocks.py -v \ + || { echo "::error::injected blocks failed to compile on python $v"; fail=1; } + done + exit $fail diff --git a/CHANGELOG.md b/CHANGELOG.md index cb7cb5a..bb97dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,23 +39,33 @@ worse than no alert, because one day it carries a security fix. ### Fixed -- **CI turned `main` red on two of three pushes without running a single test.** - `act`, the engine behind the self-hosted Gitea runner, caches each *action* as - one shared git clone under `/root/.cache/act/` and re-pulls it per job. - The three matrix jobs start within the same second on one runner, so they race - on that directory and whichever loses dies with `lstat - /root/.cache/act//.npmrc: no such file or directory` — before any suite - output exists, with a different victim each push (3.12 on one, 3.11 on the - next). A red gate that is usually noise is worse than no gate, because the one - time it means something nobody looks. +- **CI turned `main` red on two of three pushes without running a single test, + then stopped running at all.** Two failures, one cause: four concurrent jobs on + one self-hosted runner. - `uv` is now installed by a plain `run:` step instead of `astral-sh/setup-uv`. - A `run:` step has no action-cache entry and cannot race, and the action was - only ever fetching a binary — the matrix interpreter is chosen per command by - `uvx --python`, not by the action. Serialising the matrix was the alternative; - it costs 3x the wall clock and still leaves `actions/checkout` shared across - the four jobs. The uv version is pinned under the same rule as ruff: an - unpinned tool lets an upstream release turn `main` red with no change here. + `act`, the engine behind the Gitea runner, caches each *action* as one shared + git clone under `/root/.cache/act/` and re-pulls it per job. Jobs + starting in the same second fight over that directory, and the loser dies with + `lstat /root/.cache/act//.npmrc: no such file or directory` — before any + suite output exists, with a different victim each push (3.12 on one, 3.11 on + the next). Separately, the runner force-pulls its base image per job, so four + jobs meant four anonymous Docker Hub pulls per push; a few pushes and re-runs + in one afternoon hit `429 Too Many Requests` and *every* job began failing + before it started — including the shell job, which nothing had touched. + + CI is now a single job. Dropping one action (uv is installed by a `run:` step + rather than `astral-sh/setup-uv`, which was only ever fetching a binary) only + shrank the surface, because every job still used `actions/checkout`. + Concurrency is the actual ingredient, so removing it removes the whole class: + one job cannot race itself whatever actions it uses, and one job is one pull. + The Python sweep moved inside that job and still runs every version after one + fails — that is what `fail-fast: false` bought, and losing it would mean a 3.11 + break hides whether 3.12 and 3.13 are fine. The cost is wall-clock + parallelism, which this repo does not need: the suite is ~1.5s, so container + start and interpreter downloads dominate either way. + + A red gate that is usually noise is worse than no gate, because the one time + it means something, nobody looks. - **The patch survived being applied and then silently stopped existing, because something else remounted `/usr` four seconds later.** On a box running diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 38f9c59..03c81f6 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -175,15 +175,56 @@ class TestCompatCannotSilentlyPass: class TestActionCacheRace: - """act caches each ACTION as one shared clone and re-pulls it per job. + """CI must not run concurrent jobs on the self-hosted runner. - Matrix jobs start within the same second on the self-hosted Gitea runner, so - they race on `/root/.cache/act/` and the loser dies with `lstat - .../: no such file or directory` before any test runs -- a red `main` - with zero suite output and a different victim each push. Fewer actions in a - fan-out job means fewer directories to race on. + `act` caches each ACTION as one shared clone under /root/.cache/act/ + and re-pulls it per job, so jobs starting together fight over that directory + and the loser dies with `lstat .../: no such file or directory` before + any test runs -- a red `main` with zero suite output and a different victim + each push. The runner also force-pulls its base image per job, so job count + is also Docker Hub pull count, and four-per-push exhausted the anonymous + limit in an afternoon. Both problems have the same cure: one job. """ + def _ci(self): + with open(os.path.join(WORKFLOWS, "ci.yml"), encoding="utf-8") as fh: + return fh.read() + + def test_ci_runs_as_exactly_one_job(self): + """The fix is the absence of concurrency, not the absence of one action. + + Dropping astral-sh/setup-uv only shrank the surface -- every job still + used actions/checkout. A single job cannot race itself whatever actions + it uses, which is why this, and not the action count, is the invariant. + """ + ci = self._ci() + # Scope to the jobs: block -- `on:` has two-space keys of its own + # (push/pull_request/workflow_dispatch) that look identical otherwise. + body = ci[ci.index("\njobs:"):] + jobs = re.findall(r"^ (\w[\w-]*):$", body, re.M) + assert len(jobs) == 1, ( + f"ci.yml defines {len(jobs)} jobs ({jobs}); concurrent jobs on the " + "self-hosted runner race on act's shared action cache and multiply " + "Docker Hub pulls. Keep CI to one job." + ) + + def test_no_matrix_reintroduces_parallel_jobs(self): + ci = self._ci() + assert "strategy:" not in ci and "matrix:" not in ci, ( + "a matrix fans out into concurrent jobs again -- sweep versions " + "inside one job instead" + ) + + def test_every_python_version_still_runs_after_one_fails(self): + """`fail-fast: false` is what the loop has to preserve. + + A 3.11 break must not hide whether 3.12 and 3.13 are fine; that is + precisely the information you want at that moment. + """ + ci = self._ci() + assert 'PYTHONS: "3.11 3.12 3.13"' in ci + assert ci.count("fail=1") >= 2, "the sweeps must collect failures, not exit early" + def test_uv_is_installed_without_an_action(self): """Checks `uses:` directives, not prose. @@ -194,21 +235,18 @@ class TestActionCacheRace: environment holds pytest and nothing else, so a third-party import here fails on the runner while passing locally. """ - with open(os.path.join(WORKFLOWS, "ci.yml"), encoding="utf-8") as fh: - ci = fh.read() + ci = self._ci() used = re.findall(r"^\s*-?\s*uses:\s*(\S+)", ci, re.M) assert not [u for u in used if "setup-uv" in u], ( - "installing uv via an action reintroduces the act action-cache race " - "that turned main red on two of three pushes; install it in a run: step" + "the action was only fetching a binary; a run: step does the same " + "with one less moving part" ) assert "astral.sh/uv/" in ci def test_the_uv_version_is_pinned(self): - with open(os.path.join(WORKFLOWS, "ci.yml"), encoding="utf-8") as fh: - ci = fh.read() + ci = self._ci() assert re.search(r'UV_VERSION:\s*"\d+\.\d+\.\d+"', ci), ( "an unpinned uv lets any upstream release turn main red with no " "code change here -- the same rule ruff is pinned under" ) - # The version must be used, not just declared. - assert 'https://astral.sh/uv/${UV_VERSION}/install.sh' in ci + assert "https://astral.sh/uv/${UV_VERSION}/install.sh" in ci