From 7202aac5956ef92ab6a4efc6cb1917de075d771a Mon Sep 17 00:00:00 2001 From: Jens Tirsvad Nielsen Date: Tue, 13 Oct 2026 15:59:25 +0800 Subject: [PATCH] Add static workflow test for ci.yml Closes #34 Task: MIL-001#7 --- docs/artifact-registry.md | 2 +- .../reviews/rc-029-python-workflow-test.md | 59 +++++++ .../reviews/rc-030-ci-workflow-delta-24.md | 44 +++++ docs/sqa/traceability-matrix.md | 5 +- pyproject.toml | 3 +- tests/test_workflow.py | 167 ++++++++++++++++++ 6 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 docs/sqa/reviews/rc-029-python-workflow-test.md create mode 100644 docs/sqa/reviews/rc-030-ci-workflow-delta-24.md create mode 100644 tests/test_workflow.py diff --git a/docs/artifact-registry.md b/docs/artifact-registry.md index bd4c33b..6e7695a 100644 --- a/docs/artifact-registry.md +++ b/docs/artifact-registry.md @@ -16,7 +16,7 @@ document of a type. `Primary File` may contain a glob (e.g. | PP | Project Plan | docs/project-plan.md | 002 | | MIL | Milestone / Gateway | docs/milestones/*.md | 005 | | ADR | Architecture Decision Record | docs/adr/adr-NNNN-*.md | 0002 | -| RC | SQA Review Record | docs/sqa/reviews/rc-*.md | 029 | +| RC | SQA Review Record | docs/sqa/reviews/rc-*.md | 031 | | DICT | Domain Dictionary | docs/dictionary.md | 002 | | TM | Traceability Matrix | docs/sqa/traceability-matrix.md | 002 | | UCD | Use Case Diagram | docs/use-case-diagram.md | 002 | diff --git a/docs/sqa/reviews/rc-029-python-workflow-test.md b/docs/sqa/reviews/rc-029-python-workflow-test.md new file mode 100644 index 0000000..933539b --- /dev/null +++ b/docs/sqa/reviews/rc-029-python-workflow-test.md @@ -0,0 +1,59 @@ +# RC Review of the workflow test + +## Metadata +| Key | Value | +| --- | --- | +| ID | RC-029 | +| CrossReference | [PY-MIL-001-7], [QC-PY-001] | + +## Version History +| Date | Status | Author | Reviewer | Change | Commit | +| --- | --- | --- | --- | --- | --- | +| 2026-10-13 | Accepted | Jens Tirsvad Nielsen | AI as stand-in for S01 | Initial version | pending | + +--- + +## Artifact Under Review + +- Instance reviewed: [PY-MIL-001-7] +- Checklist used: [QC-PY-001] +- Scope: full review of tests/test_workflow.py and the dev extra of pyproject.toml +- Language and domain: n/a +- Language reviewer: none (the stand-in reviewer reads English and the `it` domain) +- Mode: autonomous, not independent (reviewer: AI as stand-in for S01) + +## Checklist Results + +| # | Criterion | Status | Evidence/Notes | +| --- | --- | --- | --- | +| 1 | Packages, modules, functions, variables, classes and constants follow PEP 8 casing (`snake_case`, `PascalCase`, `UPPER_SNAKE`) (Mandatory) | Pass | Functions, constants (`MAX_RUN_LINES`) and class (`UniqueKeyLoader`) follow PEP 8 casing. | +| 2 | Names state purpose in the domain's language; no unexplained abbreviations, no single-letter names outside tiny scopes (Mandatory) | Pass | Names state purpose (`exceeds_line_limit`, `load_workflow`). | +| 3 | Code is produced by the project's formatter and passes its linter with no unexplained suppressions (Mandatory) | Pass | `ruff check` and `ruff format --check` pass with no suppressions. | +| 4 | Every function and method signature is type-annotated, including `-> None` (Mandatory) | Pass | Every function is annotated; `mypy --strict` passes on 11 files; `Any` is justified by a comment. | +| 5 | No bare `except:`, no swallowed exceptions; specific exceptions are raised and the cause is kept (`raise ... from`) (Mandatory) | Pass | The duplicate-key loader raises `ValueError` with the key named; no bare except. | +| 6 | No mutable default arguments and no shadowed builtins (Mandatory) | Pass | No mutable defaults; no builtins shadowed. | +| 7 | Files, locks and connections are managed with context managers (Mandatory) | Pass | The workflow file is read through `pathlib` and the YAML loader, which close the file. | +| 8 | Public modules, classes and functions have docstrings that say what, not how (Optional) | Pass | Helpers and tests have docstrings. | +| 9 | Logging uses `logging`, not `print`; no secrets or personal data in log output (Mandatory) | Pass | No print and no logging; no secrets. | +| 10 | Classes and operations trace to the Design Class Diagram they implement; deviations are recorded (Mandatory) | N-A | Test code; no class or operation of a design class diagram. | +| 11 | Tests exist for new behaviour, are named for the behaviour, and do not depend on order or the network (Mandatory) | Pass | 14 tests, one behaviour each, plus a negative self-test of the line limit; no order or network dependence. | +| 12 | Type checker runs in strict mode without errors; `Any` is justified in a comment (Optional) | Pass | `mypy --strict` passes; `Any` is commented. | +| 13 | Dependencies are declared and pinned in the project's dependency file, none unused (Optional) | Pass | `pyyaml` and `types-PyYAML` are dev dependencies pinned exactly; no runtime dependency added. | +| 14 | Each piece of knowledge (a business rule, constant, format, validation or query) is defined in one place; code is merged only where it expresses the same knowledge, not where it merely looks alike (Mandatory) | Pass | The line limit is the single constant `MAX_RUN_LINES`, matching the rule in the YAML convention. | +| 15 | Imports point inward: business-rule modules import no framework, database, UI or delivery-mechanism code, and there are no import cycles (Mandatory) | Pass | The test imports the standard library and `yaml`; it imports no application code. | +| 16 | SOLID holds: a class or module has one reason to change; new behaviour is added by extension, not by editing a type test in several places; subclasses honour the contract of their base; protocols are narrow; high-level code depends on protocols or abstract types and receives its implementations (Mandatory) | Pass | One module of small single-purpose helpers and tests. | + +## Overall Verdict + +Go — all Mandatory criteria pass; criterion 10 is not applicable. + +## Action Items + +| Action | Owner | Due | +| --- | --- | --- | +| None | S01 | - | + +--- + +[PY-MIL-001-7]: ../../../tests/test_workflow.py +[QC-PY-001]: ../../../framework/qc/code/qc-programming-python.md diff --git a/docs/sqa/reviews/rc-030-ci-workflow-delta-24.md b/docs/sqa/reviews/rc-030-ci-workflow-delta-24.md new file mode 100644 index 0000000..62921fd --- /dev/null +++ b/docs/sqa/reviews/rc-030-ci-workflow-delta-24.md @@ -0,0 +1,44 @@ +# RC Review of ci.yml, delta re-review of criterion 24 + +## Metadata +| Key | Value | +| --- | --- | +| ID | RC-030 | +| CrossReference | [WF-MIL-001], [QC-WF-001] | + +## Version History +| Date | Status | Author | Reviewer | Change | Commit | +| --- | --- | --- | --- | --- | --- | +| 2026-10-13 | Accepted | Jens Tirsvad Nielsen | AI as stand-in for S01 | Initial version | pending | + +--- + +## Artifact Under Review + +- Instance reviewed: [WF-MIL-001] +- Checklist used: [QC-WF-001] +- Scope: delta re-review of criterion 24 of QC-WF-001; reason: action item of RC-010; earlier records RC-010 and RC-025 +- Language and domain: n/a +- Language reviewer: none (the stand-in reviewer reads English and the `it` domain) +- Mode: autonomous, not independent (reviewer: AI as stand-in for S01) + +## Checklist Results + +| # | Criterion | Status | Evidence/Notes | +| --- | --- | --- | --- | +| 24 | A static test of the repository's workflows exists and runs on every pull request: valid YAML, required keys, explicit permissions and runner, no inline script above the limit (Optional) | Pass | `tests/test_workflow.py` runs in the pytest step of `ci.yml` on every pull request: it checks safe-loader parsing, required keys, duplicate keys, explicit permissions and runner, timeout, step names, exact-tag pins and the five-line limit on `run:` blocks. 33 tests pass locally. | + +## Overall Verdict + +Go — criterion 24 (Optional) now passes; the other rows of RC-010 and RC-025 still hold. Closes the second action item of RC-010. + +## Action Items + +| Action | Owner | Due | +| --- | --- | --- | +| None | S01 | - | + +--- + +[WF-MIL-001]: ../../../.gitea/workflows/ci.yml +[QC-WF-001]: ../../../framework/qc/code/qc-workflow.md diff --git a/docs/sqa/traceability-matrix.md b/docs/sqa/traceability-matrix.md index 08693e2..24c66f5 100644 --- a/docs/sqa/traceability-matrix.md +++ b/docs/sqa/traceability-matrix.md @@ -40,7 +40,8 @@ updated whenever an artifact instance is created or reviewed. | [ADR-0001] | Architecture Decision Record | - | - | [BC-001], [SA-001], [MIL-002] | `src/squirrel/census.py` | [RC-017] | | `src/squirrel/census.py`, `cli.py`, `__main__.py`, `tests/` | Python Source Code | - | - | [UC-001], [SSD-001], [ADR-0001] | - | [RC-024] | | `src/squirrel/constants.py`, `tests/test_constants.py` | Python Source Code | - | - | [MIL-001] | - | [RC-008] | -| `.gitea/workflows/ci.yml` | CI Workflow | - | - | [MIL-001] | - | [RC-009], [RC-010], [RC-025] | +| `tests/test_workflow.py` | Python Source Code | - | - | [MIL-001], [RC-010] | - | [RC-029] | +| `.gitea/workflows/ci.yml` | CI Workflow | - | - | [MIL-001] | - | [RC-009], [RC-010], [RC-025], [RC-030] | ## Coverage Notes @@ -93,3 +94,5 @@ updated whenever an artifact instance is created or reviewed. [RC-026]: ./reviews/rc-026-business-case-wiki-path.md [RC-027]: ./reviews/rc-027-mil-004-wiki-path.md [RC-028]: ./reviews/rc-028-mil-001-task-7.md +[RC-029]: ./reviews/rc-029-python-workflow-test.md +[RC-030]: ./reviews/rc-030-ci-workflow-delta-24.md diff --git a/pyproject.toml b/pyproject.toml index 319be8b..f8768e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,8 @@ classifiers = [ ] [project.optional-dependencies] -dev = ["pytest==9.1.1", "ruff==0.17.0", "mypy==2.4.0"] +dev = ["pytest==9.1.1", "ruff==0.17.0", "mypy==2.4.0", + "pyyaml==6.0.3", "types-PyYAML==6.0.12.20260906"] [tool.setuptools.packages.find] where = ["src"] diff --git a/tests/test_workflow.py b/tests/test_workflow.py new file mode 100644 index 0000000..c76934b --- /dev/null +++ b/tests/test_workflow.py @@ -0,0 +1,167 @@ +"""Static checks of the CI workflow (QC-WF-001 criterion 24, MIL-001 task 7).""" + +import re +from collections.abc import Hashable, Iterator +from pathlib import Path +from typing import Any + +import pytest +import yaml + +ROOT = Path(__file__).parents[1] +WORKFLOW_PATH = ROOT / ".gitea" / "workflows" / "ci.yml" +GITHUB_WORKFLOW_PATH = ROOT / ".github" / "workflows" / "ci.yml" +MAX_RUN_LINES = 5 +EXACT_TAG = re.compile(r"@v\d+\.\d+\.\d+$") +FULL_SHA = re.compile(r"@[0-9a-f]{40}$") + +# Parsed YAML is untyped data, so Any is the honest type here. +Workflow = dict[Any, Any] + + +class UniqueKeyLoader(yaml.SafeLoader): + """Safe loader that rejects a mapping with a repeated key.""" + + def construct_mapping( + self, node: yaml.MappingNode, deep: bool = False + ) -> dict[Hashable, Any]: + """Build the mapping, raising ValueError on a duplicate key.""" + seen: set[Hashable] = set() + for key_node, _ in node.value: + key = self.construct_object(key_node, deep=deep) + if key in seen: + raise ValueError(f"duplicate key {key!r} at {key_node.start_mark}") + seen.add(key) + return super().construct_mapping(node, deep=deep) + + +def load_workflow() -> Workflow: + """Parse the workflow with the safe loader.""" + loaded = yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) + assert isinstance(loaded, dict) + return loaded + + +def triggers(workflow: Workflow) -> Any: + """Return the trigger section; YAML 1.1 reads the key `on` as True.""" + return workflow["on"] if "on" in workflow else workflow[True] + + +def trigger_names(workflow: Workflow) -> set[str]: + """Return the names of the events that trigger the workflow.""" + section = triggers(workflow) + if isinstance(section, str): + return {section} + return {str(name) for name in section} + + +def jobs(workflow: Workflow) -> dict[str, Workflow]: + """Return the jobs by id.""" + found: dict[str, Workflow] = workflow["jobs"] + return found + + +def steps(job: Workflow) -> list[Workflow]: + """Return the steps of a job.""" + found: list[Workflow] = job.get("steps", []) + return found + + +def all_steps(workflow: Workflow) -> Iterator[Workflow]: + """Yield every step of every job.""" + for job in jobs(workflow).values(): + yield from steps(job) + + +def exceeds_line_limit(script: str, limit: int = MAX_RUN_LINES) -> bool: + """Tell whether a script has more lines than the limit, as written. + + Blank lines and comments count. + """ + return len(script.splitlines()) > limit + + +def test_workflow_parses_with_required_keys() -> None: + """The file is valid YAML with a name, a trigger and jobs.""" + workflow = load_workflow() + assert "name" in workflow + assert "on" in workflow or True in workflow + assert "jobs" in workflow + + +def test_workflow_has_no_duplicate_keys() -> None: + """No mapping in the file repeats a key.""" + text = WORKFLOW_PATH.read_text(encoding="utf-8") + assert yaml.load(text, Loader=UniqueKeyLoader) is not None + + +def test_duplicate_key_loader_rejects_repeated_key() -> None: + """The duplicate-key loader bites on a repeated key.""" + with pytest.raises(ValueError, match="duplicate key"): + yaml.load("a: 1\na: 2\n", Loader=UniqueKeyLoader) + + +def test_permissions_are_declared() -> None: + """Permissions are set at workflow level or on every job.""" + workflow = load_workflow() + if "permissions" in workflow: + return + assert all("permissions" in job for job in jobs(workflow).values()) + + +def test_every_job_has_runner_and_timeout() -> None: + """Each job names its runner and a timeout.""" + for job_id, job in jobs(load_workflow()).items(): + assert "runs-on" in job, job_id + assert "timeout-minutes" in job, job_id + + +def test_every_step_has_a_name() -> None: + """Each step carries a name.""" + for job_id, job in jobs(load_workflow()).items(): + for step in steps(job): + assert "name" in step, f"{job_id}: {step}" + + +def test_step_names_are_unique_within_a_job() -> None: + """No two steps of one job share a name.""" + for job_id, job in jobs(load_workflow()).items(): + names = [step["name"] for step in steps(job) if "name" in step] + assert len(names) == len(set(names)), job_id + + +def test_uses_are_pinned_to_exact_release_or_sha() -> None: + """Each action is pinned to an exact tag or a full commit SHA.""" + for step in all_steps(load_workflow()): + if "uses" in step: + ref = str(step["uses"]) + assert EXACT_TAG.search(ref) or FULL_SHA.search(ref), ref + + +def test_run_blocks_stay_within_line_limit() -> None: + """No inline script is longer than the limit.""" + for step in all_steps(load_workflow()): + if "run" in step: + assert not exceeds_line_limit(str(step["run"])), step.get("name") + + +def test_line_limit_check_rejects_six_lines() -> None: + """The line-limit helper counts lines as written, blank lines included.""" + assert exceeds_line_limit("a\nb\nc\nd\ne\nf\n") + assert exceeds_line_limit("a\nb\nc\nd\ne\n\n") + assert not exceeds_line_limit("a\nb\n\nc\nd\n") + + +def test_workflow_uses_no_secrets() -> None: + """The file refers to no secret.""" + assert "secrets." not in WORKFLOW_PATH.read_text(encoding="utf-8") + + +def test_workflow_is_not_triggered_by_pull_request_target() -> None: + """The privileged pull_request_target trigger is not used.""" + assert "pull_request_target" not in trigger_names(load_workflow()) + + +def test_workflow_is_not_duplicated_for_github() -> None: + """The same workflow does not also live under .github/workflows.""" + assert not GITHUB_WORKFLOW_PATH.exists()