"""Tests for the release automation. The release workflow refuses to publish unless these hold, so a bad tag fails loudly in CI instead of shipping a release whose notes are empty, wrong, or whose scripts announce a different version than the tag. That last one is not hypothetical: VERSION= drifted to three different values across install.sh / uninstall.sh / recover.sh / apply.sh and nothing noticed. """ import os import re import sys import pytest sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools")) from release_notes import ( # noqa: E402 changelog_versions, check, extract_notes, normalise, script_versions, significance, version_tuple, ) REPO = os.path.join(os.path.dirname(__file__), "..") SAMPLE = """\ # Changelog ## v0.3.0 — 2026-07-13 ### Added - the new thing ## v0.2.1 — 2026-07-09 ### Fixed - the old thing ## v0.2.0 — 2026-07-08 - first """ class TestExtractNotes: def test_returns_only_that_versions_body(self): body = extract_notes(SAMPLE, "v0.3.0") assert "the new thing" in body assert "the old thing" not in body # The version heading itself is dropped (GitHub renders its own title), # but sub-headings like "### Added" must survive. assert not body.startswith("## v") assert body.startswith("### Added") def test_stops_at_the_next_version_heading(self): body = extract_notes(SAMPLE, "v0.2.1") assert "the old thing" in body assert "first" not in body def test_last_section_runs_to_end_of_file(self): assert "first" in extract_notes(SAMPLE, "v0.2.0") def test_accepts_the_tag_with_or_without_the_v(self): assert extract_notes(SAMPLE, "0.3.0") == extract_notes(SAMPLE, "v0.3.0") def test_unknown_version_raises_rather_than_returning_empty(self): # An empty release body is worse than a failed release. with pytest.raises(KeyError, match="no section"): extract_notes(SAMPLE, "v9.9.9") class TestChangelogVersions: def test_lists_versions_newest_first(self): assert changelog_versions(SAMPLE) == ["0.3.0", "0.2.1", "0.2.0"] class TestAgainstTheRealRepo: """These run against the actual files, so drift breaks the build.""" def test_every_script_declares_a_version(self): from release_notes import VERSIONED_FILES found = script_versions(REPO) missing = [f for f in VERSIONED_FILES if f not in found] assert not missing, f"no VERSION= in: {missing}" def test_all_scripts_agree_on_the_version(self): versions = {normalise(v) for v in script_versions(REPO).values()} assert len(versions) == 1, f"scripts disagree on version: {sorted(versions)}" def test_the_current_version_has_a_changelog_section(self): version = next(iter({normalise(v) for v in script_versions(REPO).values()})) with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh: body = extract_notes(fh.read(), version) assert body, f"CHANGELOG.md has no content for v{version}" def test_the_current_version_is_the_newest_changelog_entry(self): version = next(iter({normalise(v) for v in script_versions(REPO).values()})) with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh: newest = changelog_versions(fh.read())[0] assert newest == version, ( f"scripts say v{version} but the newest CHANGELOG entry is v{newest}" ) def test_the_repo_is_always_releasable_as_a_candidate(self): # Deliberately checked as an rc, not as a stable release. `main` carries # work under `## Unreleased` most of the time, and the stable gate refuses # that on purpose -- shipping with work stranded mid-section is how you get # a release that needs another release. So the invariant main must uphold is # the candidate one: versions agree, and the CHANGELOG section exists. version = next(iter({normalise(v) for v in script_versions(REPO).values()})) assert check(f"v{version}-rc1", REPO) == [] class TestCheckCatchesMistakes: def test_reports_a_tag_that_no_script_matches(self): problems = check("v9.9.9", REPO) assert problems assert any("declares VERSION" in p for p in problems) def test_reports_a_missing_changelog_section(self): problems = check("v9.9.9", REPO) assert any("no section" in p for p in problems) class TestSignificance: """Drives the TrueNAS update alert: what is worth bothering a human about. The rule: a release whose CHANGELOG only has a "### Docs" section changed no code, and nobody should get an alert because a README was reworded. """ TEXT = """\ # Changelog ## v0.4.2 — 2026-07-13 ### Docs - reworded the README ## v0.4.1 — 2026-07-13 ### Fixed - a real bug ## v0.4.0 — 2026-07-13 ### Added - a feature ## v0.3.3 — 2026-07-13 ### Security - keep a password out of argv ## v0.3.2 — 2026-07-13 ### Fixed - something """ def test_docs_only_release_does_not_alert(self): level, versions, _ = significance(self.TEXT, "0.4.1", "0.4.2") assert level == "docs" assert versions == ["0.4.2"] def test_a_real_fix_alerts(self): level, _v, _h = significance(self.TEXT, "0.4.0", "0.4.1") assert level == "notable" def test_security_in_range_escalates(self): level, _v, _h = significance(self.TEXT, "0.3.2", "0.3.3") assert level == "security" def test_security_wins_even_when_the_newest_release_is_docs_only(self): # A docs-only v0.4.2 sitting on top of a security-fixing v0.3.3 must still # be reported as security — classify the whole span, not just the tip. level, versions, _ = significance(self.TEXT, "0.3.2", "0.4.2") assert level == "security" assert set(versions) == {"0.3.3", "0.4.0", "0.4.1", "0.4.2"} def test_same_version_is_never_notable(self): level, versions, _ = significance(self.TEXT, "0.4.2", "0.4.2") assert level == "docs" assert versions == [] def test_range_is_exclusive_of_current_inclusive_of_latest(self): _l, versions, _h = significance(self.TEXT, "0.4.0", "0.4.2") assert "0.4.0" not in versions assert "0.4.2" in versions def test_version_tuple_orders_correctly(self): assert version_tuple("v0.10.0") > version_tuple("v0.9.9") assert version_tuple("0.4.2") > version_tuple("0.4.1") # Pre-release suffixes are dropped, not ranked above the release. assert version_tuple("v0.5.0-rc1") == version_tuple("v0.5.0") class TestCandidateNotesResolveToTheBaseVersion: """`notes v0.6.0-rc1` must return v0.6.0's section. A candidate ships the same code as the release it is a candidate for, and the CHANGELOG only ever has the one section. Without this, the release workflow cut v0.6.0-rc1, passed every gate, and then died extracting the body -- so the tag existed but nothing was ever published. Caught in an rc, which is the entire point of having them. """ CHANGELOG = "# C\n\n## v0.6.0 — 2026-07-13\n\n### Added\n- the thing\n\n## v0.5.1 — 2026-07-13\n\n- older\n" def test_an_rc_resolves_to_its_base_version(self): body = extract_notes(self.CHANGELOG, "v0.6.0-rc1") assert "the thing" in body assert "older" not in body def test_rc10_too(self): assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0-rc10") def test_the_plain_version_still_works(self): assert "the thing" in extract_notes(self.CHANGELOG, "v0.6.0") def test_a_genuinely_missing_section_still_raises(self): with pytest.raises(KeyError): extract_notes(self.CHANGELOG, "v9.9.9-rc1") class TestTheChangelogIsStructurallySound: """The release body IS this file, so a mangled section ships to every user. It has been mangled once: an edit matched the literal `## Unreleased` inside a backticked phrase in a prose bullet and spliced a whole new section into the middle of it, splitting the sentence in half. """ def changelog(self): with open(os.path.join(REPO, "CHANGELOG.md"), encoding="utf-8") as fh: return fh.read() def test_no_version_section_is_empty(self): text = self.changelog() for v in changelog_versions(text): assert extract_notes(text, v).strip(), f"v{v} has an empty section" def test_versions_are_in_descending_order(self): from release_notes import version_tuple versions = changelog_versions(self.changelog()) assert versions == sorted(versions, key=version_tuple, reverse=True), ( "CHANGELOG versions are out of order — a section was spliced in wrong" ) def test_headings_are_at_the_start_of_a_line_and_not_inside_prose(self): # A `### Fixed` that ends up indented under a bullet is a section nobody sees. for i, line in enumerate(self.changelog().splitlines(), 1): if line.lstrip().startswith(("## ", "### ")) and line != line.lstrip(): raise AssertionError( f"line {i}: heading is indented, so it is inside a list item " f"rather than being a section: {line!r}" ) def test_every_bullet_that_opens_a_bold_phrase_closes_it(self): # The splice cut `- **A stable release ... under \`## Unreleased` in half, # leaving an unterminated ** and a dangling sentence. # # A bullet is the `- ` line plus everything up to the next top-level bullet or # heading -- bold phrases routinely wrap across lines, so a per-line check # would flag every long bullet in the file. text = self.changelog() bullets = re.split(r"^(?=- |#{2,3} )", text, flags=re.M) bad = [] for b in bullets: if not b.startswith("- "): continue # Code spans are not markup: `*args, **kwargs` is a literal, not a bold # phrase, and counting its ** would flag a perfectly well-formed bullet. prose = re.sub(r"`[^`]*`", "", b) if prose.count("**") % 2: bad.append(b.splitlines()[0][:70]) assert not bad, ( "unbalanced ** in a bullet — a section was probably spliced into the " "middle of it:\n " + "\n ".join(bad) )