github-action/tools/count_loc.py

"""Count the code files and the lines of code of a directory (ADR-0010).

A code file is a text file whose extension is in LANGUAGES. Every physical line
of it is exactly one of blank, comment or code; the lines of code are the code
lines only. The result is JSON:

    {"files": 3, "lines": 120, "comment": 14, "blank": 22,
     "languages": {"Python": {"files": 2, "lines": 100, "comment": 10, "blank": 20}}}

Usage:
    python tools/count_loc.py [--directory DIR] [--exclude PATTERNS] [--output FILE]

Exit codes: 0 counted, 1 the directory is not a directory (printed as
`::error::[configuration] ...`).
"""

import argparse
import json
import os
import re
import sys
from dataclasses import dataclass
from pathlib import Path

DEFAULT_EXCLUDED_DIRS = frozenset(
    {
        "node_modules",
        "vendor",
        "third_party",
        "dist",
        "build",
        "out",
        "bin",
        "obj",
        "target",
        "venv",
        "__pycache__",
        "shared-scripts",
    }
)

_C_STYLE_CHAR = re.compile(r"'(?:\\.[^']*|[^'\\])'")


@dataclass(frozen=True)
class Language:
    """The comment and string rules of one language."""

    name: str
    extensions: tuple[str, ...]
    line_comment: str | None = None
    block: tuple[str, str] | None = None
    nested_block: bool = False
    hash_at_word_start: bool = False
    triple_quotes: bool = False
    backtick_strings: bool = False
    single_quote: str = (
        "string"  # "string", "char" (a literal of one character) or "none"
    )
    backslash_escapes: bool = True
    escapes_in_single_quotes: bool = True


def _c_family(name: str, *extensions: str, single_quote: str = "char") -> Language:
    return Language(
        name,
        extensions,
        line_comment="//",
        block=("/*", "*/"),
        single_quote=single_quote,
    )


LANGUAGES: tuple[Language, ...] = (
    Language("Python", (".py",), line_comment="#", triple_quotes=True),
    Language(
        "Shell",
        (".sh", ".bash"),
        line_comment="#",
        hash_at_word_start=True,
        escapes_in_single_quotes=False,
    ),
    Language(
        "PowerShell",
        (".ps1", ".psm1"),
        line_comment="#",
        block=("<#", "#>"),
        hash_at_word_start=True,
        backslash_escapes=False,
    ),
    _c_family("C", ".c", ".h"),
    _c_family("C++", ".cc", ".cpp", ".cxx", ".hh", ".hpp"),
    _c_family("C#", ".cs"),
    _c_family("Java", ".java"),
    Language(
        "JavaScript",
        (".js", ".mjs", ".jsx"),
        line_comment="//",
        block=("/*", "*/"),
        backtick_strings=True,
    ),
    Language(
        "TypeScript",
        (".ts", ".tsx"),
        line_comment="//",
        block=("/*", "*/"),
        backtick_strings=True,
    ),
    Language(
        "Go",
        (".go",),
        line_comment="//",
        block=("/*", "*/"),
        backtick_strings=True,
        single_quote="char",
    ),
    Language(
        "Rust",
        (".rs",),
        line_comment="//",
        block=("/*", "*/"),
        nested_block=True,
        single_quote="char",
    ),
    Language(
        "SQL",
        (".sql",),
        line_comment="--",
        block=("/*", "*/"),
        backslash_escapes=False,
    ),
)

_BY_EXTENSION = {ext: language for language in LANGUAGES for ext in language.extensions}
_WORD_START = " \t;|&("


def language_of(path: str) -> Language | None:
    """Return the language of a file name by its extension, or None."""
    return _BY_EXTENSION.get(os.path.splitext(path)[1].lower())


@dataclass
class _State:
    block_depth: int = 0
    triple: str | None = None
    backtick: bool = False


def _skip_string(line: str, start: int, quote: str, escapes: bool) -> int:
    """Return the index after the string that opens at start; the end of line if unclosed."""
    i = start + 1
    while i < len(line):
        char = line[i]
        if escapes and char == "\\":
            i += 2
            continue
        if char == quote:
            return i + 1
        i += 1
    return len(line)


def _scan_line(line: str, language: Language, state: _State) -> tuple[bool, bool]:
    """Scan one line; return (has_code, has_comment) and update the state."""
    has_code = False
    has_comment = False
    i = 0
    size = len(line)
    block = language.block
    while i < size:
        if state.block_depth:
            assert block is not None
            has_comment = True
            end = line.find(block[1], i)
            nested = line.find(block[0], i) if language.nested_block else -1
            if nested != -1 and (end == -1 or nested < end):
                state.block_depth += 1
                i = nested + len(block[0])
            elif end == -1:
                return has_code, has_comment
            else:
                state.block_depth -= 1
                i = end + len(block[1])
            continue
        if state.triple is not None:
            has_code = True
            end = line.find(state.triple, i)
            if end == -1:
                return has_code, has_comment
            i = end + len(state.triple)
            state.triple = None
            continue
        if state.backtick:
            has_code = True
            end = line.find("`", i)
            if end == -1:
                return has_code, has_comment
            i = end + 1
            state.backtick = False
            continue
        char = line[i]
        if char in " \t\r\f\v":
            i += 1
            continue
        if block is not None and line.startswith(block[0], i):
            state.block_depth = 1
            has_comment = True
            i += len(block[0])
            continue
        comment = language.line_comment
        if comment is not None and line.startswith(comment, i):
            word_start = i == 0 or line[i - 1] in _WORD_START
            if word_start or not language.hash_at_word_start:
                return has_code, True
        if language.triple_quotes and (
            line.startswith('"""', i) or line.startswith("'''", i)
        ):
            state.triple = line[i : i + 3]
            has_code = True
            i += 3
            continue
        if char == '"':
            has_code = True
            i = _skip_string(line, i, '"', language.backslash_escapes)
            continue
        if char == "'" and language.single_quote == "string":
            has_code = True
            escapes = language.backslash_escapes and language.escapes_in_single_quotes
            i = _skip_string(line, i, "'", escapes)
            continue
        if char == "'" and language.single_quote == "char":
            has_code = True
            literal = _C_STYLE_CHAR.match(line, i)
            i = literal.end() if literal else i + 1
            continue
        if char == "`" and language.backtick_strings:
            has_code = True
            end = line.find("`", i + 1)
            if end == -1:
                state.backtick = True
                return has_code, has_comment
            i = end + 1
            continue
        has_code = True
        i += 1
    return has_code, has_comment


def classify_text(text: str, language: Language) -> tuple[int, int, int]:
    """Return (code, comment, blank) line counts of the text of a file."""
    lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
    if lines and lines[-1] == "":
        lines.pop()
    state = _State()
    code = comment = blank = 0
    for line in lines:
        has_code, _ = _scan_line(line, language, state)
        if line.strip() == "":
            blank += 1
        elif has_code:
            code += 1
        else:
            comment += 1
    return code, comment, blank


def _pattern_regex(pattern: str) -> re.Pattern[str]:
    """Translate an exclude pattern: `*` stops at `/`, `**` crosses it, `?` is one character."""
    out = []
    i = 0
    while i < len(pattern):
        char = pattern[i]
        if pattern.startswith("**", i):
            out.append(".*")
            i += 2
            continue
        out.append(
            "[^/]*" if char == "*" else "[^/]" if char == "?" else re.escape(char)
        )
        i += 1
    return re.compile("".join(out) + r"\Z")


class Excludes:
    """The exclude patterns of the caller; a pattern without `/` matches a name anywhere."""

    def __init__(self, patterns: list[str]) -> None:
        self._names: list[re.Pattern[str]] = []
        self._paths: list[re.Pattern[str]] = []
        for pattern in patterns:
            pattern = pattern.strip().strip("/")
            if not pattern:
                continue
            target = self._paths if "/" in pattern else self._names
            target.append(_pattern_regex(pattern))

    def matches(self, name: str, relative: str) -> bool:
        return any(rx.match(name) for rx in self._names) or any(
            rx.match(relative) for rx in self._paths
        )


def _is_text(data: bytes) -> bool:
    return b"\x00" not in data


def count(directory: Path, patterns: list[str] | None = None) -> dict[str, object]:
    """Count the code files under directory; see the module docstring for the result."""
    excludes = Excludes(patterns or [])
    totals = {"files": 0, "lines": 0, "comment": 0, "blank": 0}
    per_language: dict[str, dict[str, int]] = {}
    root = str(directory)
    for current, dirs, files in os.walk(root, followlinks=False):
        relative_dir = os.path.relpath(current, root).replace(os.sep, "/")
        prefix = "" if relative_dir == "." else relative_dir + "/"
        dirs[:] = sorted(
            name
            for name in dirs
            if not name.startswith(".")
            and name not in DEFAULT_EXCLUDED_DIRS
            and not excludes.matches(name, prefix + name)
        )
        for name in sorted(files):
            full = os.path.join(current, name)
            language = language_of(name)
            if (
                language is None
                or os.path.islink(full)
                or excludes.matches(name, prefix + name)
            ):
                continue
            data = Path(full).read_bytes()
            if not _is_text(data):
                continue
            text = data.decode("utf-8", errors="replace").lstrip("")
            code, comment, blank = classify_text(text, language)
            entry = per_language.setdefault(
                language.name, {"files": 0, "lines": 0, "comment": 0, "blank": 0}
            )
            for target in (entry, totals):
                target["files"] += 1
                target["lines"] += code
                target["comment"] += comment
                target["blank"] += blank
    return {**totals, "languages": dict(sorted(per_language.items()))}


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument(
        "--directory", default=".", help="directory to count (default .)"
    )
    parser.add_argument("--exclude", default="", help="comma list of exclude patterns")
    parser.add_argument(
        "--output", help="write the JSON here instead of standard output"
    )
    args = parser.parse_args(argv)
    directory = Path(args.directory)
    if not directory.is_dir():
        print(
            f"::error::[configuration] working-directory '{args.directory}' does not exist."
        )
        return 1
    result = count(directory, [p for p in args.exclude.split(",") if p.strip()])
    text = json.dumps(result, indent=2, sort_keys=True) + "\n"
    if args.output:
        Path(args.output).write_text(text, encoding="utf-8", newline="\n")
    else:
        sys.stdout.write(text)
    return 0


if __name__ == "__main__":
    sys.exit(main())