MIL-010 tasks 4 and 5: counter, coverage reader and badge renderer

Task: MIL-010#4
Task: MIL-010#5
Refs #158
Refs #159
This commit is contained in:
2026-10-14 13:48:04 +08:00
parent 6c3544837c
commit 5bc14d789d
4 changed files with 1200 additions and 0 deletions
+282
View File
@@ -0,0 +1,282 @@
"""Offline tests for tools/badges.py (ADR-0010): the coverage reader and the SVG renderer."""
import importlib.util
import json
import xml.etree.ElementTree as ElementTree
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parent.parent
_SPEC = importlib.util.spec_from_file_location(
"badges", REPO_ROOT / "tools" / "badges.py"
)
assert _SPEC is not None and _SPEC.loader is not None
badges = importlib.util.module_from_spec(_SPEC)
_SPEC.loader.exec_module(badges)
DTD = (
'<!DOCTYPE coverage SYSTEM "http://cobertura.sourceforge.net/xml/coverage-04.dtd">'
)
def report(
tmp_path: Path, attributes: str, root: str = "coverage", prolog: str = ""
) -> Path:
path = tmp_path / "coverage.xml"
path.write_text(
f'<?xml version="1.0" ?>\n{prolog}<{root} {attributes}><packages/></{root}>\n'
)
return path
class TestReadCoverage:
@pytest.mark.parametrize(
("covered", "valid", "percent"),
[
(0, 10, 0),
(1, 3, 33),
(2, 3, 67),
(5, 8, 63),
(87, 100, 87),
(1, 200, 1),
(10, 10, 100),
],
)
def test_percent_is_covered_over_valid_rounded_half_up(
self, tmp_path, covered, valid, percent
):
path = report(
tmp_path, f'lines-covered="{covered}" lines-valid="{valid}" line-rate="0"'
)
assert badges.read_coverage(path) == percent
def test_the_half_rounds_up(self, tmp_path):
assert (
badges.read_coverage(report(tmp_path, 'lines-covered="1" lines-valid="8"'))
== 13
)
def test_line_rate_is_used_when_the_counts_are_absent(self, tmp_path):
assert badges.read_coverage(report(tmp_path, 'line-rate="0.875"')) == 88
def test_a_document_type_declaration_is_accepted(self, tmp_path):
path = report(tmp_path, 'lines-covered="9" lines-valid="10"', prolog=DTD + "\n")
assert badges.read_coverage(path) == 90
def test_missing_file_names_the_file(self, tmp_path):
with pytest.raises(badges.ConfigurationError, match="nope.xml' does not exist"):
badges.read_coverage(tmp_path / "nope.xml")
def test_malformed_xml_names_the_file(self, tmp_path):
path = tmp_path / "bad.xml"
path.write_text("<coverage lines-valid=")
with pytest.raises(
badges.ConfigurationError, match="bad.xml' is not well-formed"
):
badges.read_coverage(path)
def test_other_xml_is_not_a_cobertura_report(self, tmp_path):
with pytest.raises(badges.ConfigurationError, match="not a Cobertura report"):
badges.read_coverage(report(tmp_path, 'lines-valid="1"', root="testsuite"))
def test_no_valid_lines_is_a_failure_and_not_zero_percent(self, tmp_path):
with pytest.raises(badges.ConfigurationError, match="has no valid lines"):
badges.read_coverage(report(tmp_path, 'lines-covered="0" lines-valid="0"'))
def test_covered_above_valid_is_refused(self, tmp_path):
with pytest.raises(badges.ConfigurationError, match="outside 0..lines-valid"):
badges.read_coverage(report(tmp_path, 'lines-covered="5" lines-valid="4"'))
def test_report_without_any_rate_is_refused(self, tmp_path):
with pytest.raises(badges.ConfigurationError, match="neither lines-valid"):
badges.read_coverage(report(tmp_path, 'branch-rate="1"'))
def test_line_rate_outside_zero_to_one_is_refused(self, tmp_path):
with pytest.raises(badges.ConfigurationError, match="outside 0..1"):
badges.read_coverage(report(tmp_path, 'line-rate="1.5"'))
def test_values_that_are_not_numbers_are_refused(self, tmp_path):
with pytest.raises(badges.ConfigurationError, match="not a number"):
badges.read_coverage(report(tmp_path, 'lines-covered="a" lines-valid="b"'))
with pytest.raises(badges.ConfigurationError, match="not a number"):
badges.read_coverage(report(tmp_path, 'line-rate="high"'))
def test_entity_declarations_are_refused(self, tmp_path):
prolog = '<!DOCTYPE coverage [<!ENTITY x "y">]>\n'
with pytest.raises(badges.ConfigurationError, match="declares entities"):
badges.read_coverage(
report(tmp_path, 'lines-covered="1" lines-valid="2"', prolog=prolog)
)
class TestColor:
@pytest.mark.parametrize(
("percent", "color"),
[
(0, badges.RED),
(59, badges.RED),
(60, badges.YELLOW),
(79, badges.YELLOW),
(80, badges.GREEN),
(100, badges.GREEN),
],
)
def test_thresholds_are_warn_and_good(self, percent, color):
assert badges.coverage_color(percent, 60, 80) == color
def test_thresholds_can_be_moved(self):
assert badges.coverage_color(50, 40, 50) == badges.GREEN
assert badges.coverage_color(39, 40, 50) == badges.RED
class TestRender:
def test_is_valid_xml_with_title_and_text_alternative(self):
svg = badges.render("code files", "1,234", badges.BLUE)
root = ElementTree.fromstring(svg)
ns = "{http://www.w3.org/2000/svg}"
assert root.tag == ns + "svg"
assert root.get("role") == "img"
assert root.get("aria-label") == "code files: 1,234"
assert root.find(ns + "title").text == "code files: 1,234"
def test_same_input_gives_the_same_bytes_and_has_no_script_or_link(self):
first = badges.render("coverage", "87%", badges.GREEN)
assert first == badges.render("coverage", "87%", badges.GREEN)
for forbidden in (
"<script",
"href",
"<image",
"<style",
"http://www.w3.org/1999/xlink",
):
assert forbidden not in first
def test_width_follows_the_number_of_characters(self):
narrow = ElementTree.fromstring(badges.render("a", "1", badges.BLUE)).get(
"width"
)
wide = ElementTree.fromstring(badges.render("a", "1,000", badges.BLUE)).get(
"width"
)
assert int(wide) - int(narrow) == 4 * badges.CHAR_WIDTH
def test_text_is_escaped(self):
svg = badges.render("a<b", 'c&"d', badges.BLUE)
assert "a&lt;b" in svg and "c&amp;&quot;d" in svg
ElementTree.fromstring(svg)
def test_text_colour_has_the_better_contrast(self):
assert badges._text_color("#555555") == "#ffffff"
assert badges._text_color(badges.GREEN) == "#000000"
def metrics_file(tmp_path: Path, files: int = 3, lines: int = 1200) -> Path:
path = tmp_path / "metrics.json"
path.write_text(
json.dumps(
{"files": files, "lines": lines, "comment": 5, "blank": 6, "languages": {}}
)
)
return path
class TestMain:
def test_writes_three_badges_and_the_metrics(self, tmp_path, capsys):
out = tmp_path / "out"
cov = report(tmp_path, 'lines-covered="87" lines-valid="100"')
code = badges.main(
[
"--metrics",
str(metrics_file(tmp_path)),
"--output-dir",
str(out),
"--coverage-report",
str(cov),
]
)
assert code == 0
assert sorted(p.name for p in out.iterdir()) == [
"code-files.svg",
"code-lines.svg",
"code-metrics.json",
"coverage.svg",
]
assert (
'aria-label="lines of code: 1,200"' in (out / "code-lines.svg").read_text()
)
assert 'aria-label="coverage: 87%"' in (out / "coverage.svg").read_text()
assert f'fill="{badges.GREEN}"' in (out / "coverage.svg").read_text()
assert b"\r" not in (out / "code-files.svg").read_bytes()
assert "wrote" in capsys.readouterr().out
def test_output_is_byte_identical_on_a_second_run(self, tmp_path):
args = [
"--metrics",
str(metrics_file(tmp_path)),
"--output-dir",
str(tmp_path / "o"),
]
badges.main(args)
first = {p.name: p.read_bytes() for p in (tmp_path / "o").iterdir()}
badges.main(args)
assert first == {p.name: p.read_bytes() for p in (tmp_path / "o").iterdir()}
def test_without_a_report_no_coverage_badge_and_a_notice(self, tmp_path, capsys):
out = tmp_path / "out"
assert (
badges.main(
["--metrics", str(metrics_file(tmp_path)), "--output-dir", str(out)]
)
== 0
)
assert not (out / "coverage.svg").exists()
assert "::notice::No coverage-report given" in capsys.readouterr().out
def test_a_bad_report_fails_and_writes_nothing(self, tmp_path, capsys):
out = tmp_path / "out"
code = badges.main(
[
"--metrics",
str(metrics_file(tmp_path)),
"--output-dir",
str(out),
"--coverage-report",
str(tmp_path / "missing.xml"),
]
)
assert code == 1
assert "::error::[configuration] coverage-report" in capsys.readouterr().out
assert not out.exists()
@pytest.mark.parametrize(
("warn", "good", "message"),
[
("x", "80", "coverage-warn 'x'"),
("60", "101", "coverage-good '101'"),
("-1", "80", "coverage-warn '-1'"),
("90", "80", "above coverage-good"),
],
)
def test_invalid_thresholds_fail(self, tmp_path, capsys, warn, good, message):
code = badges.main(
[
"--metrics",
str(metrics_file(tmp_path)),
"--output-dir",
str(tmp_path / "o"),
"--coverage-warn",
warn,
"--coverage-good",
good,
]
)
assert code == 1
assert message in capsys.readouterr().out
def test_unreadable_metrics_fail(self, tmp_path, capsys):
bad = tmp_path / "m.json"
bad.write_text('{"files": 1}')
code = badges.main(["--metrics", str(bad), "--output-dir", str(tmp_path / "o")])
assert code == 1
assert "cannot be read" in capsys.readouterr().out
+317
View File
@@ -0,0 +1,317 @@
"""Offline tests for tools/count_loc.py (ADR-0010): hand-counted fixtures per language."""
import importlib.util
import json
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parent.parent
_SPEC = importlib.util.spec_from_file_location(
"count_loc", REPO_ROOT / "tools" / "count_loc.py"
)
assert _SPEC is not None and _SPEC.loader is not None
count_loc = importlib.util.module_from_spec(_SPEC)
_SPEC.loader.exec_module(count_loc)
LANGUAGE = {language.name: language for language in count_loc.LANGUAGES}
# name -> (source, code, comment, blank), counted by hand.
FIXTURES = {
"Python": (
"# a comment\n"
"x = 1 # trailing comment\n"
"\n"
's = "a # not a comment"\n'
"t = 'it''s # still a string'\n"
'"""docstring\n'
"# inside a string\n"
"\n"
'"""\n'
"y = 2", # no final newline
7,
1,
2,
),
"Shell": (
"#!/usr/bin/env bash\n"
"# comment\n"
'echo "$#" ${#args[@]} a#b\n'
"\n"
" # indented comment\n"
"echo 'it # stays' # but this is a comment\n"
"ls | wc -l;# after a semicolon\n",
3,
3,
1,
),
"PowerShell": (
'<# block\ncomment #>\nWrite-Host "a # b"\n# line\n$x = 1 # trailing\n',
2,
3,
0,
),
"C": (
"/* header\n"
" * text\n"
" */\n"
"#include <stdio.h>\n"
"\n"
"int main(void) { /* inline */ return 0; }\n"
"char quote = '\"'; // a double quote in a char\n"
'const char *url = "http://example.com";\n'
"// done\n",
4,
4,
1,
),
"C++": (
'int a = 0; // one\n/* block */\nauto s = "/* not a comment */";\n',
2,
1,
0,
),
"C#": (
"// line\n"
"using System;\n"
"/// <summary>doc</summary>\n"
'class A { string s = "// no"; }\n',
2,
2,
0,
),
"Java": (
"/** doc */\nclass A {\n // c\n int x; /* t */\n}\n",
3,
2,
0,
),
"JavaScript": (
"const a = `line one\n"
"// in a template\n"
"line three`;\n"
"// real comment\n"
"const b = 'x // y';\n"
"/* a */ /* b */\n",
4,
2,
0,
),
"TypeScript": (
"let n: number = 1; // c\n\n/*\n * block\n */\n",
1,
3,
1,
),
"Go": (
"package main\n// c\nvar r = '/'\nvar raw = `// not a comment\n`\n",
4,
1,
0,
),
"Rust": (
"/* outer /* nested */ still a comment */\n"
"fn f<'a>(x: &'a str) -> &'a str { x } // lifetimes\n"
"let c = '\\'';\n"
"// end\n",
2,
2,
0,
),
"SQL": (
"-- query\n"
"SELECT '-- not a comment' FROM t; -- but this is\n"
"/* a\n"
" block */\n"
"\n",
1,
3,
1,
),
}
@pytest.mark.parametrize("name", sorted(FIXTURES))
def test_fixture_counts_equal_the_hand_count(name):
source, code, comment, blank = FIXTURES[name]
assert count_loc.classify_text(source, LANGUAGE[name]) == (code, comment, blank)
def test_every_language_has_a_fixture():
assert set(FIXTURES) == set(LANGUAGE)
class TestClassify:
def test_empty_text_has_no_lines(self):
assert count_loc.classify_text("", LANGUAGE["Python"]) == (0, 0, 0)
def test_windows_line_endings_are_one_line_each(self):
assert count_loc.classify_text("a = 1\r\n# c\r\n\r\n", LANGUAGE["Python"]) == (
1,
1,
1,
)
def test_blank_line_inside_a_block_comment_is_blank(self):
assert count_loc.classify_text("/*\n\n*/\n", LANGUAGE["C"]) == (0, 2, 1)
def test_code_after_a_closing_block_comment_is_code(self):
assert count_loc.classify_text("/* a\n b */ x();\n", LANGUAGE["C"]) == (1, 1, 0)
def test_unterminated_string_ends_at_end_of_line(self):
assert count_loc.classify_text('s = "open // x\n// c\n', LANGUAGE["C"]) == (
1,
1,
0,
)
def test_unterminated_backtick_string_continues(self):
assert count_loc.classify_text("a = `x\n// y\n", LANGUAGE["JavaScript"]) == (
2,
0,
0,
)
def test_escaped_quote_does_not_end_the_string(self):
assert count_loc.classify_text('s = "a \\" // b";\n', LANGUAGE["C"]) == (
1,
0,
0,
)
class TestPatterns:
def test_star_stops_at_a_slash_and_double_star_does_not(self):
excludes = count_loc.Excludes(["src/*.py", "docs/**"])
assert excludes.matches("a.py", "src/a.py")
assert not excludes.matches("a.py", "src/sub/a.py")
assert excludes.matches("x.py", "docs/a/b/x.py")
def test_name_pattern_matches_anywhere(self):
excludes = count_loc.Excludes(["gen_*", "?.py"])
assert excludes.matches("gen_a.py", "deep/er/gen_a.py")
assert excludes.matches("a.py", "x/a.py")
assert not excludes.matches("ab.py", "x/ab.py")
def test_empty_patterns_are_ignored(self):
assert not count_loc.Excludes(["", " / "]).matches("a.py", "a.py")
def write(root: Path, name: str, content: str | bytes) -> None:
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
if isinstance(content, bytes):
path.write_bytes(content)
else:
path.write_text(content, encoding="utf-8")
class TestCount:
def test_counts_files_and_lines_per_language(self, tmp_path):
write(tmp_path, "a.py", "x = 1\n# c\n\n")
write(tmp_path, "pkg/b.py", "y = 2\n")
write(tmp_path, "run.sh", "echo hi\n")
write(tmp_path, "README.md", "# not code\n")
result = count_loc.count(tmp_path)
assert result == {
"files": 3,
"lines": 3,
"comment": 1,
"blank": 1,
"languages": {
"Python": {"files": 2, "lines": 2, "comment": 1, "blank": 1},
"Shell": {"files": 1, "lines": 1, "comment": 0, "blank": 0},
},
}
def test_default_excludes_hidden_vendored_and_shared_script_directories(
self, tmp_path
):
write(tmp_path, "keep.py", "a = 1\n")
for directory in (
".git",
".venv",
"node_modules",
"vendor",
"shared-scripts",
"build",
):
write(tmp_path, f"{directory}/skip.py", "b = 2\n")
assert count_loc.count(tmp_path)["files"] == 1
def test_exclude_patterns_add_to_the_defaults(self, tmp_path):
write(tmp_path, "keep.py", "a = 1\n")
write(tmp_path, "gen/out.py", "b = 2\n")
write(tmp_path, "src/gen_x.py", "c = 3\n")
write(tmp_path, "src/keep2.py", "d = 4\n")
result = count_loc.count(tmp_path, ["gen", "gen_*"])
assert result["files"] == 2
def test_binary_file_with_a_code_extension_is_not_counted(self, tmp_path):
write(tmp_path, "blob.py", b"a = 1\n\x00\x01")
assert count_loc.count(tmp_path)["files"] == 0
def test_invalid_utf8_and_a_byte_order_mark_are_read(self, tmp_path):
write(tmp_path, "a.py", b"\xef\xbb\xbfx = 1\n\xff\xfe = 2\n")
assert count_loc.count(tmp_path)["lines"] == 2
def test_empty_code_file_is_a_file_with_no_lines(self, tmp_path):
write(tmp_path, "empty.py", "")
result = count_loc.count(tmp_path)
assert (result["files"], result["lines"]) == (1, 0)
def test_directory_without_code_files_counts_zero(self, tmp_path):
write(tmp_path, "notes.txt", "hello\n")
result = count_loc.count(tmp_path)
assert result == {
"files": 0,
"lines": 0,
"comment": 0,
"blank": 0,
"languages": {},
}
def test_symbolic_link_is_not_followed(self, tmp_path):
real = tmp_path / "real"
write(real, "a.py", "x = 1\n")
link = tmp_path / "link.py"
try:
link.symlink_to(real / "a.py")
except (OSError, NotImplementedError):
pytest.skip("symbolic links are not available here")
assert count_loc.count(real)["files"] == 1
assert count_loc.count(tmp_path)["files"] == 1 # real/a.py only, not link.py
class TestMain:
def test_prints_json_to_standard_output(self, tmp_path, capsys):
write(tmp_path, "a.py", "x = 1\n")
assert count_loc.main(["--directory", str(tmp_path)]) == 0
out = json.loads(capsys.readouterr().out)
assert out["files"] == 1 and out["lines"] == 1
def test_writes_the_output_file_with_lf_endings(self, tmp_path):
write(tmp_path, "src/a.py", "x = 1\n")
target = tmp_path / "result.json"
assert (
count_loc.main(
["--directory", str(tmp_path / "src"), "--output", str(target)]
)
== 0
)
data = target.read_bytes()
assert b"\r" not in data and json.loads(data)["files"] == 1
def test_exclude_option_is_a_comma_list(self, tmp_path, capsys):
write(tmp_path, "a.py", "x = 1\n")
write(tmp_path, "b.py", "y = 1\n")
write(tmp_path, "c.py", "z = 1\n")
assert (
count_loc.main(["--directory", str(tmp_path), "--exclude", "a.py, b.py"])
== 0
)
assert json.loads(capsys.readouterr().out)["files"] == 1
def test_missing_directory_is_a_configuration_error(self, tmp_path, capsys):
assert count_loc.main(["--directory", str(tmp_path / "nope")]) == 1
assert "::error::[configuration]" in capsys.readouterr().out
+226
View File
@@ -0,0 +1,226 @@
"""Read a Cobertura coverage report and write the code metrics badges (ADR-0010).
Usage:
python tools/badges.py --metrics code-metrics.json --output-dir DIR
[--coverage-report coverage.xml] [--coverage-warn 60] [--coverage-good 80]
Writes `code-files.svg`, `code-lines.svg`, `code-metrics.json` and, when a report
is given, `coverage.svg` into DIR. The same input gives the same bytes.
Exit codes: 0 written, 1 a configuration problem (printed as
`::error::[configuration] ...`, nothing is written).
"""
import argparse
import json
import sys
import xml.etree.ElementTree as ElementTree
from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
from pathlib import Path
BLUE = "#007ec6"
RED = "#e05d44"
YELLOW = "#dfb317"
GREEN = "#44cc11"
LABEL_COLOR = "#555555"
CHAR_WIDTH = 6
PADDING = 10
HEIGHT = 20
class ConfigurationError(Exception):
"""An input is invalid; the message is printed as a `[configuration]` failure."""
def read_coverage(path: Path) -> int:
"""Return the line coverage of a Cobertura XML report as a whole percent, half up."""
if not path.is_file():
raise ConfigurationError(f"coverage-report '{path}' does not exist.")
data = path.read_bytes()
if b"<!ENTITY" in data:
raise ConfigurationError(
f"coverage-report '{path}' declares entities, which are refused."
)
try:
root = ElementTree.fromstring(data)
except ElementTree.ParseError as error:
raise ConfigurationError(
f"coverage-report '{path}' is not well-formed XML: {error}."
) from error
if root.tag != "coverage":
raise ConfigurationError(
f"coverage-report '{path}' is not a Cobertura report (root element '{root.tag}')."
)
covered, valid = root.get("lines-covered"), root.get("lines-valid")
try:
if covered is not None and valid is not None:
covered_lines, valid_lines = int(covered), int(valid)
if valid_lines <= 0:
raise ConfigurationError(
f"coverage-report '{path}' has no valid lines."
)
if not 0 <= covered_lines <= valid_lines:
raise ConfigurationError(
f"coverage-report '{path}' has lines-covered outside 0..lines-valid."
)
fraction = Decimal(covered_lines) / Decimal(valid_lines)
else:
rate = root.get("line-rate")
if rate is None:
raise ConfigurationError(
f"coverage-report '{path}' has neither lines-valid and lines-covered nor line-rate."
)
fraction = Decimal(rate)
if not Decimal(0) <= fraction <= Decimal(1):
raise ConfigurationError(
f"coverage-report '{path}' has a line-rate outside 0..1."
)
except (ValueError, InvalidOperation) as error:
raise ConfigurationError(
f"coverage-report '{path}' has a value that is not a number."
) from error
return int((fraction * 100).quantize(Decimal(1), rounding=ROUND_HALF_UP))
def coverage_color(percent: int, warn: float, good: float) -> str:
"""Return red below warn, yellow from warn to below good, green from good."""
if percent < warn:
return RED
if percent < good:
return YELLOW
return GREEN
def _luminance(color: str) -> float:
channels = [int(color[i : i + 2], 16) / 255 for i in (1, 3, 5)]
linear = [
c / 12.92 if c <= 0.04045 else ((c + 0.055) / 1.055) ** 2.4 for c in channels
]
return 0.2126 * linear[0] + 0.7152 * linear[1] + 0.0722 * linear[2]
def _text_color(background: str) -> str:
"""Black or white, whichever has the higher contrast on the background."""
lum = _luminance(background)
return "#ffffff" if (1.05 / (lum + 0.05)) >= ((lum + 0.05) / 0.05) else "#000000"
def _escape(text: str) -> str:
return (
text.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replace('"', "&quot;")
)
def render(label: str, value: str, color: str) -> str:
"""Return a flat two-part SVG badge; the width follows the number of characters."""
label_width = CHAR_WIDTH * len(label) + PADDING
value_width = CHAR_WIDTH * len(value) + PADDING
total = label_width + value_width
text = _escape(f"{label}: {value}")
label_x = label_width / 2
value_x = label_width + value_width / 2
return (
f'<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="{HEIGHT}" '
f'role="img" aria-label="{text}">\n'
f"<title>{text}</title>\n"
f'<rect width="{label_width}" height="{HEIGHT}" fill="{LABEL_COLOR}"/>\n'
f'<rect x="{label_width}" width="{value_width}" height="{HEIGHT}" fill="{color}"/>\n'
'<g font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11" '
'text-anchor="middle">\n'
f'<text x="{label_x:g}" y="14" fill="{_text_color(LABEL_COLOR)}">{_escape(label)}</text>\n'
f'<text x="{value_x:g}" y="14" fill="{_text_color(color)}">{_escape(value)}</text>\n'
"</g>\n"
"</svg>\n"
)
def _threshold(name: str, text: str) -> float:
try:
value = float(text)
except ValueError as error:
raise ConfigurationError(
f"{name} '{text}' is not a number from 0 to 100."
) from error
if not 0 <= value <= 100:
raise ConfigurationError(f"{name} '{text}' is not a number from 0 to 100.")
return value
def write_badges(
file_count: int,
line_count: int,
metrics_json: str,
output_dir: Path,
coverage: int | None,
warn: float,
good: float,
) -> list[Path]:
"""Write the badge files and the metrics JSON; return the paths in a fixed order."""
output_dir.mkdir(parents=True, exist_ok=True)
files = {
"code-files.svg": render("code files", f"{file_count:,}", BLUE),
"code-lines.svg": render("lines of code", f"{line_count:,}", BLUE),
}
if coverage is not None:
files["coverage.svg"] = render(
"coverage", f"{coverage}%", coverage_color(coverage, warn, good)
)
files["code-metrics.json"] = metrics_json
paths = []
for name, content in files.items():
path = output_dir / name
path.write_text(content, encoding="utf-8", newline="\n")
paths.append(path)
return paths
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--metrics", required=True, help="JSON written by count_loc.py")
parser.add_argument("--output-dir", required=True)
parser.add_argument("--coverage-report", default="")
parser.add_argument("--coverage-warn", default="60")
parser.add_argument("--coverage-good", default="80")
args = parser.parse_args(argv)
try:
warn = _threshold("coverage-warn", args.coverage_warn)
good = _threshold("coverage-good", args.coverage_good)
if warn > good:
raise ConfigurationError(
f"coverage-warn {args.coverage_warn} is above coverage-good {args.coverage_good}."
)
coverage = (
read_coverage(Path(args.coverage_report)) if args.coverage_report else None
)
metrics = json.loads(Path(args.metrics).read_text(encoding="utf-8"))
file_count, line_count = int(metrics["files"]), int(metrics["lines"])
except ConfigurationError as error:
print(f"::error::[configuration] {error}")
return 1
except (OSError, ValueError, KeyError, TypeError) as error:
print(
f"::error::[configuration] metrics file '{args.metrics}' cannot be read: {error}"
)
return 1
if coverage is None:
print("::notice::No coverage-report given; no coverage badge is written.")
metrics_json = json.dumps(metrics, indent=2, sort_keys=True) + "\n"
written = write_badges(
file_count,
line_count,
metrics_json,
Path(args.output_dir),
coverage,
warn,
good,
)
for path in written:
print(f"wrote {path}")
return 0
if __name__ == "__main__":
sys.exit(main())
+375
View File
@@ -0,0 +1,375 @@
"""Count the code files and the lines of code of a directory (ADR-0010).
A code file is a text file whose extension is in LANGUAGES. Every physical line
of it is exactly one of blank, comment or code; the lines of code are the code
lines only. The result is JSON:
{"files": 3, "lines": 120, "comment": 14, "blank": 22,
"languages": {"Python": {"files": 2, "lines": 100, "comment": 10, "blank": 20}}}
Usage:
python tools/count_loc.py [--directory DIR] [--exclude PATTERNS] [--output FILE]
Exit codes: 0 counted, 1 the directory is not a directory (printed as
`::error::[configuration] ...`).
"""
import argparse
import json
import os
import re
import sys
from dataclasses import dataclass
from pathlib import Path
DEFAULT_EXCLUDED_DIRS = frozenset(
{
"node_modules",
"vendor",
"third_party",
"dist",
"build",
"out",
"bin",
"obj",
"target",
"venv",
"__pycache__",
"shared-scripts",
}
)
_C_STYLE_CHAR = re.compile(r"'(?:\\.[^']*|[^'\\])'")
@dataclass(frozen=True)
class Language:
"""The comment and string rules of one language."""
name: str
extensions: tuple[str, ...]
line_comment: str | None = None
block: tuple[str, str] | None = None
nested_block: bool = False
hash_at_word_start: bool = False
triple_quotes: bool = False
backtick_strings: bool = False
single_quote: str = (
"string" # "string", "char" (a literal of one character) or "none"
)
backslash_escapes: bool = True
escapes_in_single_quotes: bool = True
def _c_family(name: str, *extensions: str, single_quote: str = "char") -> Language:
return Language(
name,
extensions,
line_comment="//",
block=("/*", "*/"),
single_quote=single_quote,
)
LANGUAGES: tuple[Language, ...] = (
Language("Python", (".py",), line_comment="#", triple_quotes=True),
Language(
"Shell",
(".sh", ".bash"),
line_comment="#",
hash_at_word_start=True,
escapes_in_single_quotes=False,
),
Language(
"PowerShell",
(".ps1", ".psm1"),
line_comment="#",
block=("<#", "#>"),
hash_at_word_start=True,
backslash_escapes=False,
),
_c_family("C", ".c", ".h"),
_c_family("C++", ".cc", ".cpp", ".cxx", ".hh", ".hpp"),
_c_family("C#", ".cs"),
_c_family("Java", ".java"),
Language(
"JavaScript",
(".js", ".mjs", ".jsx"),
line_comment="//",
block=("/*", "*/"),
backtick_strings=True,
),
Language(
"TypeScript",
(".ts", ".tsx"),
line_comment="//",
block=("/*", "*/"),
backtick_strings=True,
),
Language(
"Go",
(".go",),
line_comment="//",
block=("/*", "*/"),
backtick_strings=True,
single_quote="char",
),
Language(
"Rust",
(".rs",),
line_comment="//",
block=("/*", "*/"),
nested_block=True,
single_quote="char",
),
Language(
"SQL",
(".sql",),
line_comment="--",
block=("/*", "*/"),
backslash_escapes=False,
),
)
_BY_EXTENSION = {ext: language for language in LANGUAGES for ext in language.extensions}
_WORD_START = " \t;|&("
def language_of(path: str) -> Language | None:
"""Return the language of a file name by its extension, or None."""
return _BY_EXTENSION.get(os.path.splitext(path)[1].lower())
@dataclass
class _State:
block_depth: int = 0
triple: str | None = None
backtick: bool = False
def _skip_string(line: str, start: int, quote: str, escapes: bool) -> int:
"""Return the index after the string that opens at start; the end of line if unclosed."""
i = start + 1
while i < len(line):
char = line[i]
if escapes and char == "\\":
i += 2
continue
if char == quote:
return i + 1
i += 1
return len(line)
def _scan_line(line: str, language: Language, state: _State) -> tuple[bool, bool]:
"""Scan one line; return (has_code, has_comment) and update the state."""
has_code = False
has_comment = False
i = 0
size = len(line)
block = language.block
while i < size:
if state.block_depth:
assert block is not None
has_comment = True
end = line.find(block[1], i)
nested = line.find(block[0], i) if language.nested_block else -1
if nested != -1 and (end == -1 or nested < end):
state.block_depth += 1
i = nested + len(block[0])
elif end == -1:
return has_code, has_comment
else:
state.block_depth -= 1
i = end + len(block[1])
continue
if state.triple is not None:
has_code = True
end = line.find(state.triple, i)
if end == -1:
return has_code, has_comment
i = end + len(state.triple)
state.triple = None
continue
if state.backtick:
has_code = True
end = line.find("`", i)
if end == -1:
return has_code, has_comment
i = end + 1
state.backtick = False
continue
char = line[i]
if char in " \t\r\f\v":
i += 1
continue
if block is not None and line.startswith(block[0], i):
state.block_depth = 1
has_comment = True
i += len(block[0])
continue
comment = language.line_comment
if comment is not None and line.startswith(comment, i):
word_start = i == 0 or line[i - 1] in _WORD_START
if word_start or not language.hash_at_word_start:
return has_code, True
if language.triple_quotes and (
line.startswith('"""', i) or line.startswith("'''", i)
):
state.triple = line[i : i + 3]
has_code = True
i += 3
continue
if char == '"':
has_code = True
i = _skip_string(line, i, '"', language.backslash_escapes)
continue
if char == "'" and language.single_quote == "string":
has_code = True
escapes = language.backslash_escapes and language.escapes_in_single_quotes
i = _skip_string(line, i, "'", escapes)
continue
if char == "'" and language.single_quote == "char":
has_code = True
literal = _C_STYLE_CHAR.match(line, i)
i = literal.end() if literal else i + 1
continue
if char == "`" and language.backtick_strings:
has_code = True
end = line.find("`", i + 1)
if end == -1:
state.backtick = True
return has_code, has_comment
i = end + 1
continue
has_code = True
i += 1
return has_code, has_comment
def classify_text(text: str, language: Language) -> tuple[int, int, int]:
"""Return (code, comment, blank) line counts of the text of a file."""
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
if lines and lines[-1] == "":
lines.pop()
state = _State()
code = comment = blank = 0
for line in lines:
has_code, _ = _scan_line(line, language, state)
if line.strip() == "":
blank += 1
elif has_code:
code += 1
else:
comment += 1
return code, comment, blank
def _pattern_regex(pattern: str) -> re.Pattern[str]:
"""Translate an exclude pattern: `*` stops at `/`, `**` crosses it, `?` is one character."""
out = []
i = 0
while i < len(pattern):
char = pattern[i]
if pattern.startswith("**", i):
out.append(".*")
i += 2
continue
out.append(
"[^/]*" if char == "*" else "[^/]" if char == "?" else re.escape(char)
)
i += 1
return re.compile("".join(out) + r"\Z")
class Excludes:
"""The exclude patterns of the caller; a pattern without `/` matches a name anywhere."""
def __init__(self, patterns: list[str]) -> None:
self._names: list[re.Pattern[str]] = []
self._paths: list[re.Pattern[str]] = []
for pattern in patterns:
pattern = pattern.strip().strip("/")
if not pattern:
continue
target = self._paths if "/" in pattern else self._names
target.append(_pattern_regex(pattern))
def matches(self, name: str, relative: str) -> bool:
return any(rx.match(name) for rx in self._names) or any(
rx.match(relative) for rx in self._paths
)
def _is_text(data: bytes) -> bool:
return b"\x00" not in data
def count(directory: Path, patterns: list[str] | None = None) -> dict[str, object]:
"""Count the code files under directory; see the module docstring for the result."""
excludes = Excludes(patterns or [])
totals = {"files": 0, "lines": 0, "comment": 0, "blank": 0}
per_language: dict[str, dict[str, int]] = {}
root = str(directory)
for current, dirs, files in os.walk(root, followlinks=False):
relative_dir = os.path.relpath(current, root).replace(os.sep, "/")
prefix = "" if relative_dir == "." else relative_dir + "/"
dirs[:] = sorted(
name
for name in dirs
if not name.startswith(".")
and name not in DEFAULT_EXCLUDED_DIRS
and not excludes.matches(name, prefix + name)
)
for name in sorted(files):
full = os.path.join(current, name)
language = language_of(name)
if (
language is None
or os.path.islink(full)
or excludes.matches(name, prefix + name)
):
continue
data = Path(full).read_bytes()
if not _is_text(data):
continue
text = data.decode("utf-8", errors="replace").lstrip("")
code, comment, blank = classify_text(text, language)
entry = per_language.setdefault(
language.name, {"files": 0, "lines": 0, "comment": 0, "blank": 0}
)
for target in (entry, totals):
target["files"] += 1
target["lines"] += code
target["comment"] += comment
target["blank"] += blank
return {**totals, "languages": dict(sorted(per_language.items()))}
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument(
"--directory", default=".", help="directory to count (default .)"
)
parser.add_argument("--exclude", default="", help="comma list of exclude patterns")
parser.add_argument(
"--output", help="write the JSON here instead of standard output"
)
args = parser.parse_args(argv)
directory = Path(args.directory)
if not directory.is_dir():
print(
f"::error::[configuration] working-directory '{args.directory}' does not exist."
)
return 1
result = count(directory, [p for p in args.exclude.split(",") if p.strip()])
text = json.dumps(result, indent=2, sort_keys=True) + "\n"
if args.output:
Path(args.output).write_text(text, encoding="utf-8", newline="\n")
else:
sys.stdout.write(text)
return 0
if __name__ == "__main__":
sys.exit(main())