MIL-010 tasks 4 and 5: counter, coverage reader and badge renderer
Task: MIL-010#4 Task: MIL-010#5 Refs #158 Refs #159
This commit is contained in:
+226
@@ -0,0 +1,226 @@
|
||||
"""Read a Cobertura coverage report and write the code metrics badges (ADR-0010).
|
||||
|
||||
Usage:
|
||||
python tools/badges.py --metrics code-metrics.json --output-dir DIR
|
||||
[--coverage-report coverage.xml] [--coverage-warn 60] [--coverage-good 80]
|
||||
|
||||
Writes `code-files.svg`, `code-lines.svg`, `code-metrics.json` and, when a report
|
||||
is given, `coverage.svg` into DIR. The same input gives the same bytes.
|
||||
|
||||
Exit codes: 0 written, 1 a configuration problem (printed as
|
||||
`::error::[configuration] ...`, nothing is written).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import xml.etree.ElementTree as ElementTree
|
||||
from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
|
||||
BLUE = "#007ec6"
|
||||
RED = "#e05d44"
|
||||
YELLOW = "#dfb317"
|
||||
GREEN = "#44cc11"
|
||||
LABEL_COLOR = "#555555"
|
||||
CHAR_WIDTH = 6
|
||||
PADDING = 10
|
||||
HEIGHT = 20
|
||||
|
||||
|
||||
class ConfigurationError(Exception):
|
||||
"""An input is invalid; the message is printed as a `[configuration]` failure."""
|
||||
|
||||
|
||||
def read_coverage(path: Path) -> int:
|
||||
"""Return the line coverage of a Cobertura XML report as a whole percent, half up."""
|
||||
if not path.is_file():
|
||||
raise ConfigurationError(f"coverage-report '{path}' does not exist.")
|
||||
data = path.read_bytes()
|
||||
if b"<!ENTITY" in data:
|
||||
raise ConfigurationError(
|
||||
f"coverage-report '{path}' declares entities, which are refused."
|
||||
)
|
||||
try:
|
||||
root = ElementTree.fromstring(data)
|
||||
except ElementTree.ParseError as error:
|
||||
raise ConfigurationError(
|
||||
f"coverage-report '{path}' is not well-formed XML: {error}."
|
||||
) from error
|
||||
if root.tag != "coverage":
|
||||
raise ConfigurationError(
|
||||
f"coverage-report '{path}' is not a Cobertura report (root element '{root.tag}')."
|
||||
)
|
||||
covered, valid = root.get("lines-covered"), root.get("lines-valid")
|
||||
try:
|
||||
if covered is not None and valid is not None:
|
||||
covered_lines, valid_lines = int(covered), int(valid)
|
||||
if valid_lines <= 0:
|
||||
raise ConfigurationError(
|
||||
f"coverage-report '{path}' has no valid lines."
|
||||
)
|
||||
if not 0 <= covered_lines <= valid_lines:
|
||||
raise ConfigurationError(
|
||||
f"coverage-report '{path}' has lines-covered outside 0..lines-valid."
|
||||
)
|
||||
fraction = Decimal(covered_lines) / Decimal(valid_lines)
|
||||
else:
|
||||
rate = root.get("line-rate")
|
||||
if rate is None:
|
||||
raise ConfigurationError(
|
||||
f"coverage-report '{path}' has neither lines-valid and lines-covered nor line-rate."
|
||||
)
|
||||
fraction = Decimal(rate)
|
||||
if not Decimal(0) <= fraction <= Decimal(1):
|
||||
raise ConfigurationError(
|
||||
f"coverage-report '{path}' has a line-rate outside 0..1."
|
||||
)
|
||||
except (ValueError, InvalidOperation) as error:
|
||||
raise ConfigurationError(
|
||||
f"coverage-report '{path}' has a value that is not a number."
|
||||
) from error
|
||||
return int((fraction * 100).quantize(Decimal(1), rounding=ROUND_HALF_UP))
|
||||
|
||||
|
||||
def coverage_color(percent: int, warn: float, good: float) -> str:
|
||||
"""Return red below warn, yellow from warn to below good, green from good."""
|
||||
if percent < warn:
|
||||
return RED
|
||||
if percent < good:
|
||||
return YELLOW
|
||||
return GREEN
|
||||
|
||||
|
||||
def _luminance(color: str) -> float:
|
||||
channels = [int(color[i : i + 2], 16) / 255 for i in (1, 3, 5)]
|
||||
linear = [
|
||||
c / 12.92 if c <= 0.04045 else ((c + 0.055) / 1.055) ** 2.4 for c in channels
|
||||
]
|
||||
return 0.2126 * linear[0] + 0.7152 * linear[1] + 0.0722 * linear[2]
|
||||
|
||||
|
||||
def _text_color(background: str) -> str:
|
||||
"""Black or white, whichever has the higher contrast on the background."""
|
||||
lum = _luminance(background)
|
||||
return "#ffffff" if (1.05 / (lum + 0.05)) >= ((lum + 0.05) / 0.05) else "#000000"
|
||||
|
||||
|
||||
def _escape(text: str) -> str:
|
||||
return (
|
||||
text.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace('"', """)
|
||||
)
|
||||
|
||||
|
||||
def render(label: str, value: str, color: str) -> str:
|
||||
"""Return a flat two-part SVG badge; the width follows the number of characters."""
|
||||
label_width = CHAR_WIDTH * len(label) + PADDING
|
||||
value_width = CHAR_WIDTH * len(value) + PADDING
|
||||
total = label_width + value_width
|
||||
text = _escape(f"{label}: {value}")
|
||||
label_x = label_width / 2
|
||||
value_x = label_width + value_width / 2
|
||||
return (
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="{HEIGHT}" '
|
||||
f'role="img" aria-label="{text}">\n'
|
||||
f"<title>{text}</title>\n"
|
||||
f'<rect width="{label_width}" height="{HEIGHT}" fill="{LABEL_COLOR}"/>\n'
|
||||
f'<rect x="{label_width}" width="{value_width}" height="{HEIGHT}" fill="{color}"/>\n'
|
||||
'<g font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11" '
|
||||
'text-anchor="middle">\n'
|
||||
f'<text x="{label_x:g}" y="14" fill="{_text_color(LABEL_COLOR)}">{_escape(label)}</text>\n'
|
||||
f'<text x="{value_x:g}" y="14" fill="{_text_color(color)}">{_escape(value)}</text>\n'
|
||||
"</g>\n"
|
||||
"</svg>\n"
|
||||
)
|
||||
|
||||
|
||||
def _threshold(name: str, text: str) -> float:
|
||||
try:
|
||||
value = float(text)
|
||||
except ValueError as error:
|
||||
raise ConfigurationError(
|
||||
f"{name} '{text}' is not a number from 0 to 100."
|
||||
) from error
|
||||
if not 0 <= value <= 100:
|
||||
raise ConfigurationError(f"{name} '{text}' is not a number from 0 to 100.")
|
||||
return value
|
||||
|
||||
|
||||
def write_badges(
|
||||
file_count: int,
|
||||
line_count: int,
|
||||
metrics_json: str,
|
||||
output_dir: Path,
|
||||
coverage: int | None,
|
||||
warn: float,
|
||||
good: float,
|
||||
) -> list[Path]:
|
||||
"""Write the badge files and the metrics JSON; return the paths in a fixed order."""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
files = {
|
||||
"code-files.svg": render("code files", f"{file_count:,}", BLUE),
|
||||
"code-lines.svg": render("lines of code", f"{line_count:,}", BLUE),
|
||||
}
|
||||
if coverage is not None:
|
||||
files["coverage.svg"] = render(
|
||||
"coverage", f"{coverage}%", coverage_color(coverage, warn, good)
|
||||
)
|
||||
files["code-metrics.json"] = metrics_json
|
||||
paths = []
|
||||
for name, content in files.items():
|
||||
path = output_dir / name
|
||||
path.write_text(content, encoding="utf-8", newline="\n")
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--metrics", required=True, help="JSON written by count_loc.py")
|
||||
parser.add_argument("--output-dir", required=True)
|
||||
parser.add_argument("--coverage-report", default="")
|
||||
parser.add_argument("--coverage-warn", default="60")
|
||||
parser.add_argument("--coverage-good", default="80")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
warn = _threshold("coverage-warn", args.coverage_warn)
|
||||
good = _threshold("coverage-good", args.coverage_good)
|
||||
if warn > good:
|
||||
raise ConfigurationError(
|
||||
f"coverage-warn {args.coverage_warn} is above coverage-good {args.coverage_good}."
|
||||
)
|
||||
coverage = (
|
||||
read_coverage(Path(args.coverage_report)) if args.coverage_report else None
|
||||
)
|
||||
metrics = json.loads(Path(args.metrics).read_text(encoding="utf-8"))
|
||||
file_count, line_count = int(metrics["files"]), int(metrics["lines"])
|
||||
except ConfigurationError as error:
|
||||
print(f"::error::[configuration] {error}")
|
||||
return 1
|
||||
except (OSError, ValueError, KeyError, TypeError) as error:
|
||||
print(
|
||||
f"::error::[configuration] metrics file '{args.metrics}' cannot be read: {error}"
|
||||
)
|
||||
return 1
|
||||
if coverage is None:
|
||||
print("::notice::No coverage-report given; no coverage badge is written.")
|
||||
metrics_json = json.dumps(metrics, indent=2, sort_keys=True) + "\n"
|
||||
written = write_badges(
|
||||
file_count,
|
||||
line_count,
|
||||
metrics_json,
|
||||
Path(args.output_dir),
|
||||
coverage,
|
||||
warn,
|
||||
good,
|
||||
)
|
||||
for path in written:
|
||||
print(f"wrote {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,375 @@
|
||||
"""Count the code files and the lines of code of a directory (ADR-0010).
|
||||
|
||||
A code file is a text file whose extension is in LANGUAGES. Every physical line
|
||||
of it is exactly one of blank, comment or code; the lines of code are the code
|
||||
lines only. The result is JSON:
|
||||
|
||||
{"files": 3, "lines": 120, "comment": 14, "blank": 22,
|
||||
"languages": {"Python": {"files": 2, "lines": 100, "comment": 10, "blank": 20}}}
|
||||
|
||||
Usage:
|
||||
python tools/count_loc.py [--directory DIR] [--exclude PATTERNS] [--output FILE]
|
||||
|
||||
Exit codes: 0 counted, 1 the directory is not a directory (printed as
|
||||
`::error::[configuration] ...`).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_EXCLUDED_DIRS = frozenset(
|
||||
{
|
||||
"node_modules",
|
||||
"vendor",
|
||||
"third_party",
|
||||
"dist",
|
||||
"build",
|
||||
"out",
|
||||
"bin",
|
||||
"obj",
|
||||
"target",
|
||||
"venv",
|
||||
"__pycache__",
|
||||
"shared-scripts",
|
||||
}
|
||||
)
|
||||
|
||||
_C_STYLE_CHAR = re.compile(r"'(?:\\.[^']*|[^'\\])'")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Language:
|
||||
"""The comment and string rules of one language."""
|
||||
|
||||
name: str
|
||||
extensions: tuple[str, ...]
|
||||
line_comment: str | None = None
|
||||
block: tuple[str, str] | None = None
|
||||
nested_block: bool = False
|
||||
hash_at_word_start: bool = False
|
||||
triple_quotes: bool = False
|
||||
backtick_strings: bool = False
|
||||
single_quote: str = (
|
||||
"string" # "string", "char" (a literal of one character) or "none"
|
||||
)
|
||||
backslash_escapes: bool = True
|
||||
escapes_in_single_quotes: bool = True
|
||||
|
||||
|
||||
def _c_family(name: str, *extensions: str, single_quote: str = "char") -> Language:
|
||||
return Language(
|
||||
name,
|
||||
extensions,
|
||||
line_comment="//",
|
||||
block=("/*", "*/"),
|
||||
single_quote=single_quote,
|
||||
)
|
||||
|
||||
|
||||
LANGUAGES: tuple[Language, ...] = (
|
||||
Language("Python", (".py",), line_comment="#", triple_quotes=True),
|
||||
Language(
|
||||
"Shell",
|
||||
(".sh", ".bash"),
|
||||
line_comment="#",
|
||||
hash_at_word_start=True,
|
||||
escapes_in_single_quotes=False,
|
||||
),
|
||||
Language(
|
||||
"PowerShell",
|
||||
(".ps1", ".psm1"),
|
||||
line_comment="#",
|
||||
block=("<#", "#>"),
|
||||
hash_at_word_start=True,
|
||||
backslash_escapes=False,
|
||||
),
|
||||
_c_family("C", ".c", ".h"),
|
||||
_c_family("C++", ".cc", ".cpp", ".cxx", ".hh", ".hpp"),
|
||||
_c_family("C#", ".cs"),
|
||||
_c_family("Java", ".java"),
|
||||
Language(
|
||||
"JavaScript",
|
||||
(".js", ".mjs", ".jsx"),
|
||||
line_comment="//",
|
||||
block=("/*", "*/"),
|
||||
backtick_strings=True,
|
||||
),
|
||||
Language(
|
||||
"TypeScript",
|
||||
(".ts", ".tsx"),
|
||||
line_comment="//",
|
||||
block=("/*", "*/"),
|
||||
backtick_strings=True,
|
||||
),
|
||||
Language(
|
||||
"Go",
|
||||
(".go",),
|
||||
line_comment="//",
|
||||
block=("/*", "*/"),
|
||||
backtick_strings=True,
|
||||
single_quote="char",
|
||||
),
|
||||
Language(
|
||||
"Rust",
|
||||
(".rs",),
|
||||
line_comment="//",
|
||||
block=("/*", "*/"),
|
||||
nested_block=True,
|
||||
single_quote="char",
|
||||
),
|
||||
Language(
|
||||
"SQL",
|
||||
(".sql",),
|
||||
line_comment="--",
|
||||
block=("/*", "*/"),
|
||||
backslash_escapes=False,
|
||||
),
|
||||
)
|
||||
|
||||
_BY_EXTENSION = {ext: language for language in LANGUAGES for ext in language.extensions}
|
||||
_WORD_START = " \t;|&("
|
||||
|
||||
|
||||
def language_of(path: str) -> Language | None:
|
||||
"""Return the language of a file name by its extension, or None."""
|
||||
return _BY_EXTENSION.get(os.path.splitext(path)[1].lower())
|
||||
|
||||
|
||||
@dataclass
|
||||
class _State:
|
||||
block_depth: int = 0
|
||||
triple: str | None = None
|
||||
backtick: bool = False
|
||||
|
||||
|
||||
def _skip_string(line: str, start: int, quote: str, escapes: bool) -> int:
|
||||
"""Return the index after the string that opens at start; the end of line if unclosed."""
|
||||
i = start + 1
|
||||
while i < len(line):
|
||||
char = line[i]
|
||||
if escapes and char == "\\":
|
||||
i += 2
|
||||
continue
|
||||
if char == quote:
|
||||
return i + 1
|
||||
i += 1
|
||||
return len(line)
|
||||
|
||||
|
||||
def _scan_line(line: str, language: Language, state: _State) -> tuple[bool, bool]:
|
||||
"""Scan one line; return (has_code, has_comment) and update the state."""
|
||||
has_code = False
|
||||
has_comment = False
|
||||
i = 0
|
||||
size = len(line)
|
||||
block = language.block
|
||||
while i < size:
|
||||
if state.block_depth:
|
||||
assert block is not None
|
||||
has_comment = True
|
||||
end = line.find(block[1], i)
|
||||
nested = line.find(block[0], i) if language.nested_block else -1
|
||||
if nested != -1 and (end == -1 or nested < end):
|
||||
state.block_depth += 1
|
||||
i = nested + len(block[0])
|
||||
elif end == -1:
|
||||
return has_code, has_comment
|
||||
else:
|
||||
state.block_depth -= 1
|
||||
i = end + len(block[1])
|
||||
continue
|
||||
if state.triple is not None:
|
||||
has_code = True
|
||||
end = line.find(state.triple, i)
|
||||
if end == -1:
|
||||
return has_code, has_comment
|
||||
i = end + len(state.triple)
|
||||
state.triple = None
|
||||
continue
|
||||
if state.backtick:
|
||||
has_code = True
|
||||
end = line.find("`", i)
|
||||
if end == -1:
|
||||
return has_code, has_comment
|
||||
i = end + 1
|
||||
state.backtick = False
|
||||
continue
|
||||
char = line[i]
|
||||
if char in " \t\r\f\v":
|
||||
i += 1
|
||||
continue
|
||||
if block is not None and line.startswith(block[0], i):
|
||||
state.block_depth = 1
|
||||
has_comment = True
|
||||
i += len(block[0])
|
||||
continue
|
||||
comment = language.line_comment
|
||||
if comment is not None and line.startswith(comment, i):
|
||||
word_start = i == 0 or line[i - 1] in _WORD_START
|
||||
if word_start or not language.hash_at_word_start:
|
||||
return has_code, True
|
||||
if language.triple_quotes and (
|
||||
line.startswith('"""', i) or line.startswith("'''", i)
|
||||
):
|
||||
state.triple = line[i : i + 3]
|
||||
has_code = True
|
||||
i += 3
|
||||
continue
|
||||
if char == '"':
|
||||
has_code = True
|
||||
i = _skip_string(line, i, '"', language.backslash_escapes)
|
||||
continue
|
||||
if char == "'" and language.single_quote == "string":
|
||||
has_code = True
|
||||
escapes = language.backslash_escapes and language.escapes_in_single_quotes
|
||||
i = _skip_string(line, i, "'", escapes)
|
||||
continue
|
||||
if char == "'" and language.single_quote == "char":
|
||||
has_code = True
|
||||
literal = _C_STYLE_CHAR.match(line, i)
|
||||
i = literal.end() if literal else i + 1
|
||||
continue
|
||||
if char == "`" and language.backtick_strings:
|
||||
has_code = True
|
||||
end = line.find("`", i + 1)
|
||||
if end == -1:
|
||||
state.backtick = True
|
||||
return has_code, has_comment
|
||||
i = end + 1
|
||||
continue
|
||||
has_code = True
|
||||
i += 1
|
||||
return has_code, has_comment
|
||||
|
||||
|
||||
def classify_text(text: str, language: Language) -> tuple[int, int, int]:
|
||||
"""Return (code, comment, blank) line counts of the text of a file."""
|
||||
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
|
||||
if lines and lines[-1] == "":
|
||||
lines.pop()
|
||||
state = _State()
|
||||
code = comment = blank = 0
|
||||
for line in lines:
|
||||
has_code, _ = _scan_line(line, language, state)
|
||||
if line.strip() == "":
|
||||
blank += 1
|
||||
elif has_code:
|
||||
code += 1
|
||||
else:
|
||||
comment += 1
|
||||
return code, comment, blank
|
||||
|
||||
|
||||
def _pattern_regex(pattern: str) -> re.Pattern[str]:
|
||||
"""Translate an exclude pattern: `*` stops at `/`, `**` crosses it, `?` is one character."""
|
||||
out = []
|
||||
i = 0
|
||||
while i < len(pattern):
|
||||
char = pattern[i]
|
||||
if pattern.startswith("**", i):
|
||||
out.append(".*")
|
||||
i += 2
|
||||
continue
|
||||
out.append(
|
||||
"[^/]*" if char == "*" else "[^/]" if char == "?" else re.escape(char)
|
||||
)
|
||||
i += 1
|
||||
return re.compile("".join(out) + r"\Z")
|
||||
|
||||
|
||||
class Excludes:
|
||||
"""The exclude patterns of the caller; a pattern without `/` matches a name anywhere."""
|
||||
|
||||
def __init__(self, patterns: list[str]) -> None:
|
||||
self._names: list[re.Pattern[str]] = []
|
||||
self._paths: list[re.Pattern[str]] = []
|
||||
for pattern in patterns:
|
||||
pattern = pattern.strip().strip("/")
|
||||
if not pattern:
|
||||
continue
|
||||
target = self._paths if "/" in pattern else self._names
|
||||
target.append(_pattern_regex(pattern))
|
||||
|
||||
def matches(self, name: str, relative: str) -> bool:
|
||||
return any(rx.match(name) for rx in self._names) or any(
|
||||
rx.match(relative) for rx in self._paths
|
||||
)
|
||||
|
||||
|
||||
def _is_text(data: bytes) -> bool:
|
||||
return b"\x00" not in data
|
||||
|
||||
|
||||
def count(directory: Path, patterns: list[str] | None = None) -> dict[str, object]:
|
||||
"""Count the code files under directory; see the module docstring for the result."""
|
||||
excludes = Excludes(patterns or [])
|
||||
totals = {"files": 0, "lines": 0, "comment": 0, "blank": 0}
|
||||
per_language: dict[str, dict[str, int]] = {}
|
||||
root = str(directory)
|
||||
for current, dirs, files in os.walk(root, followlinks=False):
|
||||
relative_dir = os.path.relpath(current, root).replace(os.sep, "/")
|
||||
prefix = "" if relative_dir == "." else relative_dir + "/"
|
||||
dirs[:] = sorted(
|
||||
name
|
||||
for name in dirs
|
||||
if not name.startswith(".")
|
||||
and name not in DEFAULT_EXCLUDED_DIRS
|
||||
and not excludes.matches(name, prefix + name)
|
||||
)
|
||||
for name in sorted(files):
|
||||
full = os.path.join(current, name)
|
||||
language = language_of(name)
|
||||
if (
|
||||
language is None
|
||||
or os.path.islink(full)
|
||||
or excludes.matches(name, prefix + name)
|
||||
):
|
||||
continue
|
||||
data = Path(full).read_bytes()
|
||||
if not _is_text(data):
|
||||
continue
|
||||
text = data.decode("utf-8", errors="replace").lstrip("")
|
||||
code, comment, blank = classify_text(text, language)
|
||||
entry = per_language.setdefault(
|
||||
language.name, {"files": 0, "lines": 0, "comment": 0, "blank": 0}
|
||||
)
|
||||
for target in (entry, totals):
|
||||
target["files"] += 1
|
||||
target["lines"] += code
|
||||
target["comment"] += comment
|
||||
target["blank"] += blank
|
||||
return {**totals, "languages": dict(sorted(per_language.items()))}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument(
|
||||
"--directory", default=".", help="directory to count (default .)"
|
||||
)
|
||||
parser.add_argument("--exclude", default="", help="comma list of exclude patterns")
|
||||
parser.add_argument(
|
||||
"--output", help="write the JSON here instead of standard output"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
directory = Path(args.directory)
|
||||
if not directory.is_dir():
|
||||
print(
|
||||
f"::error::[configuration] working-directory '{args.directory}' does not exist."
|
||||
)
|
||||
return 1
|
||||
result = count(directory, [p for p in args.exclude.split(",") if p.strip()])
|
||||
text = json.dumps(result, indent=2, sort_keys=True) + "\n"
|
||||
if args.output:
|
||||
Path(args.output).write_text(text, encoding="utf-8", newline="\n")
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user