flan
841e0364fd
An edit matched the literal '## Unreleased' inside a backticked phrase in a prose bullet and spliced a whole new section into the middle of it, splitting the sentence in half. The release body IS this file, so that would have shipped to every user. Tests now assert: no empty version section, versions descend, no heading is indented inside a list item, and every bullet's bold phrases are balanced (ignoring code spans -- '*args, **kwargs' is a literal, not markup).
294 lines
10 KiB
Python
294 lines
10 KiB
Python
"""Tests for the release automation.
|
|
|
|
The release workflow refuses to publish unless these hold, so a bad tag fails
|
|
loudly in CI instead of shipping a release whose notes are empty, wrong, or whose
|
|
scripts announce a different version than the tag.
|
|
|
|
That last one is not hypothetical: VERSION= drifted to three different values
|
|
across install.sh / uninstall.sh / recover.sh / apply.sh and nothing noticed.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools"))
|
|
|
|
from release_notes import ( # noqa: E402
|
|
changelog_versions,
|
|
check,
|
|
extract_notes,
|
|
normalise,
|
|
script_versions,
|
|
significance,
|
|
version_tuple,
|
|
)
|
|
|
|
REPO = os.path.join(os.path.dirname(__file__), "..")
|
|
|
|
SAMPLE = """\
|
|
# Changelog
|
|
|
|
## v0.3.0 — 2026-07-13
|
|
|
|
### Added
|
|
|
|
- the new thing
|
|
|
|
## v0.2.1 — 2026-07-09
|
|
|
|
### Fixed
|
|
|
|
- the old thing
|
|
|
|
## v0.2.0 — 2026-07-08
|
|
|
|
- first
|
|
"""
|
|
|
|
|
|
class TestExtractNotes:
|
|
def test_returns_only_that_versions_body(self):
|
|
body = extract_notes(SAMPLE, "v0.3.0")
|
|
assert "the new thing" in body
|
|
assert "the old thing" not in body
|
|
# The version heading itself is dropped (GitHub renders its own title),
|
|
# but sub-headings like "### Added" must survive.
|
|
assert not body.startswith("## v")
|
|
assert body.startswith("### Added")
|
|
|
|
def test_stops_at_the_next_version_heading(self):
|
|
body = extract_notes(SAMPLE, "v0.2.1")
|
|
assert "the old thing" in body
|
|
assert "first" not in body
|
|
|
|
def test_last_section_runs_to_end_of_file(self):
|
|
assert "first" in extract_notes(SAMPLE, "v0.2.0")
|
|
|
|
def test_accepts_the_tag_with_or_without_the_v(self):
|
|
assert extract_notes(SAMPLE, "0.3.0") == extract_notes(SAMPLE, "v0.3.0")
|
|
|
|
def test_unknown_version_raises_rather_than_returning_empty(self):
|
|
# An empty release body is worse than a failed release.
|
|
with pytest.raises(KeyError, match="no section"):
|
|
extract_notes(SAMPLE, "v9.9.9")
|
|
|
|
|
|
class TestChangelogVersions:
|
|
def test_lists_versions_newest_first(self):
|
|
assert changelog_versions(SAMPLE) == ["0.3.0", "0.2.1", "0.2.0"]
|
|
|
|
|
|
class TestAgainstTheRealRepo:
|
|
"""These run against the actual files, so drift breaks the build."""
|
|
|
|
def test_every_script_declares_a_version(self):
|
|
from release_notes import VERSIONED_FILES
|
|
|
|
found = script_versions(REPO)
|
|
missing = [f for f in VERSIONED_FILES if f not in found]
|
|
assert not missing, f"no VERSION= in: {missing}"
|
|
|
|
def test_all_scripts_agree_on_the_version(self):
|
|
versions = {normalise(v) for v in script_versions(REPO).values()}
|
|
assert len(versions) == 1, f"scripts disagree on version: {sorted(versions)}"
|
|
|
|
def test_the_current_version_has_a_changelog_section(self):
|
|
version = next(iter({normalise(v) for v in script_versions(REPO).values()}))
|
|
with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh:
|
|
body = extract_notes(fh.read(), version)
|
|
assert body, f"CHANGELOG.md has no content for v{version}"
|
|
|
|
def test_the_current_version_is_the_newest_changelog_entry(self):
|
|
version = next(iter({normalise(v) for v in script_versions(REPO).values()}))
|
|
with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh:
|
|
newest = changelog_versions(fh.read())[0]
|
|
assert newest == version, (
|
|
f"scripts say v{version} but the newest CHANGELOG entry is v{newest}"
|
|
)
|
|
|
|
def test_the_repo_is_always_releasable_as_a_candidate(self):
|
|
# Deliberately checked as an rc, not as a stable release. `main` carries
|
|
# work under `## Unreleased` most of the time, and the stable gate refuses
|
|
# that on purpose -- shipping with work stranded mid-section is how you get
|
|
# a release that needs another release. So the invariant main must uphold is
|
|
# the candidate one: versions agree, and the CHANGELOG section exists.
|
|
version = next(iter({normalise(v) for v in script_versions(REPO).values()}))
|
|
assert check(f"v{version}-rc1", REPO) == []
|
|
|
|
|
|
class TestCheckCatchesMistakes:
|
|
def test_reports_a_tag_that_no_script_matches(self):
|
|
problems = check("v9.9.9", REPO)
|
|
assert problems
|
|
assert any("declares VERSION" in p for p in problems)
|
|
|
|
def test_reports_a_missing_changelog_section(self):
|
|
problems = check("v9.9.9", REPO)
|
|
assert any("no section" in p for p in problems)
|
|
|
|
|
|
class TestSignificance:
|
|
"""Drives the TrueNAS update alert: what is worth bothering a human about.
|
|
|
|
The rule: a release whose CHANGELOG only has a "### Docs" section changed no
|
|
code, and nobody should get an alert because a README was reworded.
|
|
"""
|
|
|
|
TEXT = """\
|
|
# Changelog
|
|
|
|
## v0.4.2 — 2026-07-13
|
|
|
|
### Docs
|
|
|
|
- reworded the README
|
|
|
|
## v0.4.1 — 2026-07-13
|
|
|
|
### Fixed
|
|
|
|
- a real bug
|
|
|
|
## v0.4.0 — 2026-07-13
|
|
|
|
### Added
|
|
|
|
- a feature
|
|
|
|
## v0.3.3 — 2026-07-13
|
|
|
|
### Security
|
|
|
|
- keep a password out of argv
|
|
|
|
## v0.3.2 — 2026-07-13
|
|
|
|
### Fixed
|
|
|
|
- something
|
|
"""
|
|
|
|
def test_docs_only_release_does_not_alert(self):
|
|
level, versions, _ = significance(self.TEXT, "0.4.1", "0.4.2")
|
|
assert level == "docs"
|
|
assert versions == ["0.4.2"]
|
|
|
|
def test_a_real_fix_alerts(self):
|
|
level, _v, _h = significance(self.TEXT, "0.4.0", "0.4.1")
|
|
assert level == "notable"
|
|
|
|
def test_security_in_range_escalates(self):
|
|
level, _v, _h = significance(self.TEXT, "0.3.2", "0.3.3")
|
|
assert level == "security"
|
|
|
|
def test_security_wins_even_when_the_newest_release_is_docs_only(self):
|
|
# A docs-only v0.4.2 sitting on top of a security-fixing v0.3.3 must still
|
|
# be reported as security — classify the whole span, not just the tip.
|
|
level, versions, _ = significance(self.TEXT, "0.3.2", "0.4.2")
|
|
assert level == "security"
|
|
assert set(versions) == {"0.3.3", "0.4.0", "0.4.1", "0.4.2"}
|
|
|
|
def test_same_version_is_never_notable(self):
|
|
level, versions, _ = significance(self.TEXT, "0.4.2", "0.4.2")
|
|
assert level == "docs"
|
|
assert versions == []
|
|
|
|
def test_range_is_exclusive_of_current_inclusive_of_latest(self):
|
|
_l, versions, _h = significance(self.TEXT, "0.4.0", "0.4.2")
|
|
assert "0.4.0" not in versions
|
|
assert "0.4.2" in versions
|
|
|
|
def test_version_tuple_orders_correctly(self):
|
|
assert version_tuple("v0.10.0") > version_tuple("v0.9.9")
|
|
assert version_tuple("0.4.2") > version_tuple("0.4.1")
|
|
# Pre-release suffixes are dropped, not ranked above the release.
|
|
assert version_tuple("v0.5.0-rc1") == version_tuple("v0.5.0")
|
|
|
|
|
|
class TestCandidateNotesResolveToTheBaseVersion:
|
|
"""`notes v0.6.0-rc1` must return v0.6.0's section.
|
|
|
|
A candidate ships the same code as the release it is a candidate for, and the
|
|
CHANGELOG only ever has the one section. Without this, the release workflow cut
|
|
v0.6.0-rc1, passed every gate, and then died extracting the body -- so the tag
|
|
existed but nothing was ever published. Caught in an rc, which is the entire
|
|
point of having them.
|
|
"""
|
|
|
|
CHANGELOG = "# C\n\n## v0.6.0 — 2026-07-13\n\n### Added\n- the thing\n\n## v0.5.1 — 2026-07-13\n\n- older\n"
|
|
|
|
def test_an_rc_resolves_to_its_base_version(self):
|
|
body = extract_notes(self.CHANGELOG, "v0.6.0-rc1")
|
|
assert "the thing" in body
|
|
assert "older" not in body
|
|
|
|
def test_rc10_too(self):
|
|
assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0-rc10")
|
|
|
|
def test_the_plain_version_still_works(self):
|
|
assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0")
|
|
|
|
def test_a_genuinely_missing_section_still_raises(self):
|
|
with pytest.raises(KeyError):
|
|
extract_notes(self.CHANGELOG, "v9.9.9-rc1")
|
|
|
|
|
|
class TestTheChangelogIsStructurallySound:
|
|
"""The release body IS this file, so a mangled section ships to every user.
|
|
|
|
It has been mangled once: an edit matched the literal `## Unreleased` inside a
|
|
backticked phrase in a prose bullet and spliced a whole new section into the middle
|
|
of it, splitting the sentence in half.
|
|
"""
|
|
|
|
def changelog(self):
|
|
with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh:
|
|
return fh.read()
|
|
|
|
def test_no_version_section_is_empty(self):
|
|
text = self.changelog()
|
|
for v in changelog_versions(text):
|
|
assert extract_notes(text, v).strip(), f"v{v} has an empty section"
|
|
|
|
def test_versions_are_in_descending_order(self):
|
|
from release_notes import version_tuple
|
|
versions = changelog_versions(self.changelog())
|
|
assert versions == sorted(versions, key=version_tuple, reverse=True), (
|
|
"CHANGELOG versions are out of order — a section was spliced in wrong"
|
|
)
|
|
|
|
def test_headings_are_at_the_start_of_a_line_and_not_inside_prose(self):
|
|
# A `### Fixed` that ends up indented under a bullet is a section nobody sees.
|
|
for i, line in enumerate(self.changelog().splitlines(), 1):
|
|
if line.lstrip().startswith(("## ", "### ")) and line != line.lstrip():
|
|
raise AssertionError(
|
|
f"line {i}: heading is indented, so it is inside a list item "
|
|
f"rather than being a section: {line!r}"
|
|
)
|
|
|
|
def test_every_bullet_that_opens_a_bold_phrase_closes_it(self):
|
|
# The splice cut `- **A stable release ... under \`## Unreleased` in half,
|
|
# leaving an unterminated ** and a dangling sentence.
|
|
#
|
|
# A bullet is the `- ` line plus everything up to the next top-level bullet or
|
|
# heading -- bold phrases routinely wrap across lines, so a per-line check
|
|
# would flag every long bullet in the file.
|
|
text = self.changelog()
|
|
bullets = re.split(r"^(?=- |#{2,3} )", text, flags=re.M)
|
|
bad = []
|
|
for b in bullets:
|
|
if not b.startswith("- "):
|
|
continue
|
|
# Code spans are not markup: `*args, **kwargs` is a literal, not a bold
|
|
# phrase, and counting its ** would flag a perfectly well-formed bullet.
|
|
prose = re.sub(r"`[^`]*`", "", b)
|
|
if prose.count("**") % 2:
|
|
bad.append(b.splitlines()[0][:70])
|
|
assert not bad, (
|
|
"unbalanced ** in a bullet — a section was probably spliced into the "
|
|
"middle of it:\n " + "\n ".join(bad)
|
|
)
|