From b8e63d0a29d6aa335713b35a639e321f9e40baf4 Mon Sep 17 00:00:00 2001 From: Jens Tirsvad Nielsen Date: Mon, 19 Oct 2026 12:34:00 +0800 Subject: [PATCH] Add the comment extractor and the language detector of the English-comments check tools/english_comments.py reads the comments and docstrings of the languages of ADR-0010 (the scanner of count_loc.py now returns the comment text), joins the lines of a block, removes everything that is not a word and judges the block with langid (ADR-0012). It checks the lines a branch adds, or every tracked file, honours an ignore marker and an allowed-words file, and prints each finding as an annotation. The rates of the corpus are tests. Refs #199 Refs #200 Refs #203 Task: MIL-013#3 Task: MIL-013#4 --- .../adr/adr-0012-english-comment-detection.md | 4 +- tests/requirements.txt | 2 + tests/test_english_comments.py | 274 +++++++++++++ tools/count_loc.py | 18 +- tools/english_comments.py | 387 ++++++++++++++++++ 5 files changed, 682 insertions(+), 3 deletions(-) create mode 100644 tests/test_english_comments.py create mode 100644 tools/english_comments.py diff --git a/docs/adr/adr-0012-english-comment-detection.md b/docs/adr/adr-0012-english-comment-detection.md index bf1bd6f..dca9c09 100644 --- a/docs/adr/adr-0012-english-comment-detection.md +++ b/docs/adr/adr-0012-english-comment-detection.md @@ -50,7 +50,7 @@ The two English comments `langid` flagged without a threshold are lines that are A comment is judged **not English** when `langid` 1.1.6, restricted to `en da no sv de fr ms id nl`, with normalised probabilities, ranks a language other than English first with a probability of at least **0.6**, after this pre-filter, and only when the filter leaves **at least three words**: -- remove URLs and tokens that contain `/`, `\`, `::`, `_` or a digit, and camelCase tokens; split the rest on `-`; +- drop every token that is not a word: a token with a code character (`/ \ { } [ ] " ` = < > | $ @ # % & + ^ ~ *`), an underscore, a digit or `::` (a path, a URL, a JSON key, an identifier); a token that starts with `-` (an option), contains `.` (a file name), is written in capitals (an acronym or a constant) or is camelCase; a leading label such as `Usage:` or `Note:`; and the words of the allowed-words file. Split what is left on `-`; - keep a token only if it is letters, with at most one inner apostrophe (letters include `æ ø å ä ö ü` and the other letters of Latin scripts); - a comment with fewer than three words left is **not judged** (it passes), and so is a comment that holds only identifiers, URLs or numbers. @@ -58,6 +58,8 @@ On the corpus this gives 0 false positives in 300 English comments, Danish 44/45 The versions are pinned (`langid==1.1.6`, `numpy` at a pinned version) and installed in the job's virtual environment like the other tools ([ADR-0005]). Nothing is downloaded at run time and nothing is sent to a service. +The rules of the pre-filter came from the build, not from the first measurement. A scan of this repository with the first version of the filter found two false positives that the corpus did not contain: a docstring with a JSON example (`{"files": 3, "lines": 120, ...}`, judged French with probability 1.0) and `Usage:` lines with options and a file name (judged French with 0.6 and 0.88). The code-character, option, file-name, acronym and label rules above remove them; the corpus rates stayed the same, and the repository (31 source files) now gives no finding. Tests of the check keep both cases (the JSON docstring and the `Usage:` lines), so a change to the filter is measured against them. + ## Consequences **Positive:** diff --git a/tests/requirements.txt b/tests/requirements.txt index cfb4328..68771a7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,3 +3,5 @@ PyYAML>=6,<7 coverage>=7,<8 shellcheck-py==0.10.0.1 shfmt-py==3.11.0.2 +langid==1.1.6 +numpy==2.1.3 diff --git a/tests/test_english_comments.py b/tests/test_english_comments.py new file mode 100644 index 0000000..6129183 --- /dev/null +++ b/tests/test_english_comments.py @@ -0,0 +1,274 @@ +"""Offline tests for tools/english_comments.py (MIL-013, ADR-0012). + +The detector is measured on the corpus in tests/fixtures/comments/; the rates +of criterion 1 of MIL-013 are tests, so a change that lowers them fails. +""" + +import csv +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from workflow_source import REPO_ROOT, load_script + +ec = load_script(REPO_ROOT / "tools" / "english_comments.py", "english_comments") +CORPUS = REPO_ROOT / "tests" / "fixtures" / "comments" +GIT = shutil.which("git") + + +def corpus() -> list[tuple[str, str]]: + rows: list[tuple[str, str]] = [] + for name in ("english.tsv", "foreign.tsv"): + with (CORPUS / name).open(encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle, delimiter="\t", quoting=csv.QUOTE_NONE) + rows += [(row["lang"], row["text"]) for row in reader] + return rows + + +@pytest.fixture(scope="module") +def results(): + found: dict[str, list[int]] = {} + for language, text in corpus(): + counts = found.setdefault(language, [0, 0]) + counts[1] += 1 + counts[0] += ec.judge(text) is not None + return found + + +class TestCorpusRates: + """Criterion 1 of MIL-013: recall of at least 90 % for Danish and Malay, and at most + one false positive per 200 English comments.""" + + def test_the_corpus_has_the_size_the_milestone_asks_for(self, results): + assert results["en"][1] >= 200 + assert results["da"][1] >= 40 + assert results["ms"][1] >= 40 + assert sum(v[1] for k, v in results.items() if k not in ("en", "da", "ms")) >= 40 + + def test_english_comments_are_not_flagged(self, results): + flagged, total = results["en"] + assert flagged <= total // 200 + + @pytest.mark.parametrize("language", ["da", "ms"]) + def test_danish_and_malay_are_found(self, results, language): + flagged, total = results[language] + assert flagged / total >= 0.90 + + def test_other_languages_are_found(self, results): + for language in ("de", "sv", "no", "fr"): + flagged, total = results[language] + assert flagged / total >= 0.80 + + +class TestWords: + def test_paths_options_identifiers_and_numbers_are_not_words(self): + text = "Run tools/badges.py --metrics code-metrics.json with snake_case camelCase v2 DIR" + assert ec.words(text) == ["Run", "with"] + + def test_json_and_code_characters_are_not_words(self): + assert ec.words('{"files": 3, "lines": 120} x = 1 ') == [] + + def test_a_label_and_an_acronym_are_dropped(self): + assert ec.words("Usage: lint the JSON files") == ["lint", "the", "files"] + + def test_a_hyphenated_word_is_split_and_allowed_words_are_removed(self): + assert ec.words("read-only access", frozenset({"access"})) == ["read", "only"] + + def test_a_hidden_file_is_not_a_word_but_an_ellipsis_is_kept(self): + assert ec.words("needs a .clang-format file...") == ["needs", "file"] + + def test_quotes_inside_a_word_are_kept_once(self): + assert ec.words("don't can't won''t") == ["don't", "can't"] + + +class TestExtract: + def test_python_comments_and_docstrings(self): + source = '"""Module line one.\n\nSecond paragraph."""\n\n# a comment\nx = 1 # trailing\n\ndef f():\n """Doc of f."""\n' + found = dict(ec.extract("a.py", source)) + assert found[1] == "Module line one." + assert found[5] == "a comment" + assert found[6] == "trailing" + assert found[9] == "Doc of f." + + def test_python_that_does_not_parse_falls_back_to_the_scanner(self): + assert dict(ec.extract("a.py", "def (:\n# still a comment\n")) == {2: "still a comment"} + + def test_shell_and_sql_line_comments(self): + assert ec.extract("a.sh", "#!/bin/sh\n# real comment\necho '# not'\n") == [(2, "real comment")] + assert ec.extract("a.sql", "SELECT 1; -- one row\n") == [(1, "one row")] + + def test_c_block_and_line_comments_lose_their_decoration(self): + source = "/**\n * First line\n * second line\n */\nint x; // trailing\n/// doc\n" + assert ec.extract("a.c", source) == [(2, "First line"), (3, "second line"), (5, "trailing"), (6, "doc")] + + def test_a_comment_inside_a_string_is_not_a_comment(self): + assert ec.extract("a.c", 'char *s = "// not a comment";\n') == [] + + def test_tool_pragmas_are_not_prose(self): + source = "x = 1 # noqa: E501\n# type: ignore\n# pylint: disable=x\n# a real comment\n" + assert ec.extract("a.py", source) == [(4, "a real comment")] + + def test_a_file_of_an_unknown_language_has_no_comments(self): + assert ec.extract("notes.md", "# Heading\n") == [] + + def test_consecutive_lines_are_one_block(self): + lines = [(1, "a"), (2, "b"), (4, "c"), (5, "d"), (5, "e")] + assert ec.blocks(lines) == [(1, 2, "a b"), (4, 5, "c d e")] + + +class TestCheckText: + DANISH = "# Denne funktion beregner totalen for kurven og gemmer resultatet\n" + + def test_a_danish_comment_is_a_finding_with_its_line(self): + findings = ec.check_text("a.py", "x = 1\n" + self.DANISH) + assert [(f.path, f.line, f.language) for f in findings] == [("a.py", 2, "da")] + + def test_an_english_comment_is_not(self): + assert ec.check_text("a.py", "# This function adds the totals of the basket together\n") == [] + + def test_only_the_blocks_on_the_given_lines_are_checked(self): + text = self.DANISH + "x = 1\n" + self.DANISH + assert [f.line for f in ec.check_text("a.py", text, {3})] == [3] + assert ec.check_text("a.py", text, {2}) == [] + + def test_the_marker_skips_the_block(self): + text = self.DANISH.rstrip() + " # english-comments: ignore\n" + assert ec.check_text("a.py", text) == [] + + def test_allowed_words_are_not_judged(self): + allowed = frozenset(self.DANISH.lower().replace("#", "").replace(",", "").split()) + assert ec.check_text("a.py", self.DANISH, allowed=allowed) == [] + + def test_a_short_comment_is_not_judged(self): + assert ec.check_text("a.py", "# ret fejlen\n") == [] + + def test_json_in_a_docstring_is_not_french(self): + source = '"""Count.\n\n {"files": 3, "lines": 120, "comment": 14, "blank": 22,\n "languages": {"Python": {"files": 2, "lines": 100}}}\n"""\n' + assert ec.check_text("a.py", source) == [] + + def test_usage_lines_with_options_are_not_judged(self): + text = "# Usage: lint-action-scripts.sh [] (default: actions)\n" + assert ec.check_text("a.sh", text) == [] + + def test_describe_names_the_language_and_cuts_a_long_text(self): + finding = ec.Finding("a.py", 1, 1, "da", 0.93, "x " * 100) + message = ec.describe(finding) + assert message.startswith('looks Danish (0.93): "') and message.endswith('..."') + + +class TestAllowedWords: + def test_words_one_per_line_with_comments(self, tmp_path): + file = tmp_path / "allowed.txt" + file.write_text("Kubernetes\n# a comment\nSkat # a Danish tax name\n\n", encoding="utf-8") + assert ec.read_allowed_words(str(file)) == {"kubernetes", "skat"} + + def test_no_file_named_is_no_words(self): + assert ec.read_allowed_words(None) == frozenset() + + def test_an_unreadable_file_is_an_error(self, tmp_path): + with pytest.raises(ec.CheckError): + ec.read_allowed_words(str(tmp_path / "missing.txt")) + + +def run_git(directory: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=directory, check=True, capture_output=True) + + +@pytest.fixture +def repo(tmp_path): + """A repository with a base commit on main and a branch that adds comments.""" + if GIT is None: + pytest.skip("git is needed") + run_git(tmp_path, "init", "-q", "-b", "main") + run_git(tmp_path, "config", "user.email", "t@example.org") + run_git(tmp_path, "config", "user.name", "t") + run_git(tmp_path, "config", "commit.gpgsign", "false") + (tmp_path / "old.py").write_bytes(b"# Denne gamle funktion beregner totalen for kurven\nx = 1\n") + run_git(tmp_path, "add", ".") + run_git(tmp_path, "commit", "-q", "-m", "base") + run_git(tmp_path, "checkout", "-q", "-b", "feature") + return tmp_path + + +def commit_file(repo: Path, name: str, content: bytes) -> None: + (repo / name).write_bytes(content) + run_git(repo, "add", ".") + run_git(repo, "commit", "-q", "-m", name) + + +class TestRepository: + def test_only_the_comments_a_branch_adds_are_checked(self, repo, capsys): + commit_file(repo, "new.py", b"# Denne funktion beregner totalen for kurven\n") + code = ec.main(["check", "--directory", str(repo), "--base", "main", "--output", str(repo / "f.json")]) + out = capsys.readouterr().out + assert code == 1 + assert "::error file=new.py,line=1::[english-comments] looks Danish" in out + assert "old.py" not in out # the old Danish comment is not this branch's + saved = json.loads((repo / "f.json").read_text(encoding="utf-8")) + assert [(f["path"], f["line"], f["language"]) for f in saved] == [("new.py", 1, "da")] + + def test_an_added_line_in_an_old_file_is_checked_and_the_old_ones_are_not(self, repo): + commit_file(repo, "old.py", b"# Denne gamle funktion beregner totalen for kurven\nx = 1\n# Hold the lock while the file is written\n") + findings, files = ec.check_repository(repo, "main", False, [], frozenset(), 0.6, 3) + assert findings == [] and files == 1 + + def test_a_clean_branch_exits_zero(self, repo, capsys): + commit_file(repo, "new.py", b"# This function adds the totals of the basket together\n") + assert ec.main(["check", "--directory", str(repo), "--base", "main"]) == 0 + assert "All comments are in English: 1 file(s) checked." in capsys.readouterr().out + + def test_scope_all_checks_every_tracked_file(self, repo, capsys): + assert ec.main(["check", "--directory", str(repo), "--scope", "all"]) == 1 + assert "old.py:1" in capsys.readouterr().out + + def test_excluded_binary_unknown_and_large_files_are_skipped(self, repo, monkeypatch): + danish = b"# Denne funktion beregner totalen for kurven og gemmer resultatet\n" + (repo / "vendor").mkdir() + commit_file(repo, "vendor/skip.py", danish) + commit_file(repo, "skipme.py", danish) + commit_file(repo, "bin.py", danish + b"\x00") + commit_file(repo, "notes.md", b"# Denne funktion beregner totalen for kurven\n") + monkeypatch.setattr(ec, "MAX_FILE_BYTES", 10) + commit_file(repo, "big.py", danish) + findings, files = ec.check_repository(repo, "main", False, ["skipme.py"], frozenset(), 0.6, 3) + assert (findings, files) == ([], 0) + + def test_a_deleted_file_in_the_selection_is_skipped(self, repo): + commit_file(repo, "gone.py", b"x = 1\n") + (repo / "gone.py").unlink() + findings, files = ec.check_repository(repo, "main", False, [], frozenset(), 0.6, 3) + assert findings == [] + + def test_changed_lines_reads_every_hunk(self, repo): + commit_file(repo, "h.py", b"a\nb\nc\nd\n") + commit_file(repo, "h.py", b"a\nB\nc\nD\nE\n") + assert ec.changed_lines(repo, "main")["h.py"] == {1, 2, 3, 4, 5} + + +class TestMain: + def test_a_missing_base_is_a_configuration_error(self, repo, capsys): + assert ec.main(["check", "--directory", str(repo)]) == 2 + assert "::error::[configuration] give --base" in capsys.readouterr().out + + def test_an_unknown_base_is_a_configuration_error(self, repo, capsys): + assert ec.main(["check", "--directory", str(repo), "--base", "nope"]) == 2 + assert "failed: fatal" in capsys.readouterr().out + + def test_a_missing_directory_is_a_configuration_error(self, tmp_path, capsys): + assert ec.main(["check", "--directory", str(tmp_path / "x"), "--scope", "all"]) == 2 + assert "does not exist" in capsys.readouterr().out + + @pytest.mark.parametrize("flags", [["--threshold", "0"], ["--threshold", "1.5"], ["--min-words", "0"]]) + def test_bad_numbers_are_a_configuration_error(self, tmp_path, capsys, flags): + assert ec.main(["check", "--directory", str(tmp_path), "--scope", "all", *flags]) == 2 + assert "threshold" in capsys.readouterr().out + + def test_langid_missing_is_a_configuration_error(self, repo, monkeypatch, capsys): + commit_file(repo, "new.py", b"# Denne funktion beregner totalen for kurven og gemmer resultatet\n") + monkeypatch.setattr(ec, "_identifier", None) + monkeypatch.setitem(__import__("sys").modules, "langid.langid", None) + assert ec.main(["check", "--directory", str(repo), "--base", "main"]) == 2 + assert "langid is not installed" in capsys.readouterr().out diff --git a/tools/count_loc.py b/tools/count_loc.py index 4472900..d6ea5d4 100644 --- a/tools/count_loc.py +++ b/tools/count_loc.py @@ -161,8 +161,17 @@ def _skip_string(line: str, start: int, quote: str, escapes: bool) -> int: return len(line) -def _scan_line(line: str, language: Language, state: _State) -> tuple[bool, bool]: - """Scan one line; return (has_code, has_comment) and update the state.""" +def _scan_line( + line: str, + language: Language, + state: _State, + comments: list[str] | None = None, +) -> tuple[bool, bool]: + """Scan one line; return (has_code, has_comment) and update the state. + + When ``comments`` is given, the text of every comment on the line is + appended to it (the English-comments check, ADR-0012, reads that text). + """ has_code = False has_comment = False i = 0 @@ -174,6 +183,9 @@ def _scan_line(line: str, language: Language, state: _State) -> tuple[bool, bool has_comment = True end = line.find(block[1], i) nested = line.find(block[0], i) if language.nested_block else -1 + if comments is not None: + stop = nested if nested != -1 and (end == -1 or nested < end) else end + comments.append(line[i : size if stop == -1 else stop]) if nested != -1 and (end == -1 or nested < end): state.block_depth += 1 i = nested + len(block[0]) @@ -212,6 +224,8 @@ def _scan_line(line: str, language: Language, state: _State) -> tuple[bool, bool if comment is not None and line.startswith(comment, i): word_start = i == 0 or line[i - 1] in _WORD_START if word_start or not language.hash_at_word_start: + if comments is not None: + comments.append(line[i + len(comment) :]) return has_code, True if language.triple_quotes and ( line.startswith('"""', i) or line.startswith("'''", i) diff --git a/tools/english_comments.py b/tools/english_comments.py new file mode 100644 index 0000000..4877cf0 --- /dev/null +++ b/tools/english_comments.py @@ -0,0 +1,387 @@ +"""Find comments in source code that are not written in English (MIL-013, ADR-0012). + +Run by .gitea/workflows/english-comments.yml. It reads the comments and docstrings +of the source files of the languages of ADR-0010, joins the lines of one comment +block, and judges the block with ``langid`` (restricted to a few candidate +languages) after removing everything that is not a word: paths, options, +identifiers, numbers and the words of the allowed-words file. A block with fewer +than ``--min-words`` words left is not judged. + +Usage: + python tools/english_comments.py check [--directory DIR] [--scope changed|all] + [--base REF] [--exclude PATTERNS] [--allowed-words FILE] + [--threshold P] [--min-words N] [--output FILE] + +With ``--scope changed`` (the default) only the comments on lines that the +current branch added since ``--base`` (a git ref, for example ``origin/main``) +are checked, so a pull request is not blamed for a comment it did not write; +``--scope all`` checks every tracked file. A comment block that contains ``english-comments: ignore`` is +skipped (a quotation, a name). + +Every finding is printed as ``::error file=PATH,line=N::...`` and as ``PATH:N: +looks Danish (0.93): "text"``; ``--output`` writes them as JSON for the +reporting step. Exit codes: 0 no finding, 1 at least one finding, 2 a setting +or a missing tool (printed as ``::error::[configuration] ...``). +""" + +import argparse +import ast +import importlib.util +import io +import json +import re +import subprocess +import sys +import tokenize +from dataclasses import asdict, dataclass +from pathlib import Path + +DEFAULT_THRESHOLD = 0.6 +DEFAULT_MIN_WORDS = 3 +MAX_FILE_BYTES = 1_000_000 +CANDIDATES = ["en", "da", "no", "sv", "de", "fr", "ms", "id", "nl"] +LANGUAGE_NAMES = { + "da": "Danish", + "no": "Norwegian", + "sv": "Swedish", + "de": "German", + "fr": "French", + "ms": "Malay", + "id": "Indonesian", + "nl": "Dutch", +} +IGNORE_MARKER = re.compile(r"english-comments:\s*ignore", re.IGNORECASE) +# comments that configure a tool are not prose +_PRAGMA = re.compile( + r"^(noqa|type:|pragma|shellcheck|fmt:|pylint|ruff:|mypy:|pyright:|eslint|prettier|" + r"-\*-|vim:|coding[:=]|nosec|isort:|nolint|clang-format|cppcheck|lint:)", + re.IGNORECASE, +) +_WORD = re.compile(r"^[^\W\d_]+(?:'[^\W\d_]+)?$") +_NOT_WORD_CHARS = re.compile(r"[\d_/\{}\[\]\"`=<>|$@#%&+^~*]") +_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + + +class CheckError(RuntimeError): + """A setting or tool problem (exit code 2).""" + + +def _load_sibling(name: str): + path = Path(__file__).resolve().parent / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +count_loc = _load_sibling("count_loc") + + +@dataclass(frozen=True) +class Finding: + """A comment block that is probably not English.""" + + path: str + line: int + end_line: int + language: str + probability: float + text: str + + +def _clean(text: str) -> str: + """Remove comment decoration (``*``, ``/``, ``#``, ``-``, ``=``) from both ends.""" + text = re.sub(r"^[/*!#<>=\-\s]+", "", text.strip()) + return re.sub(r"[*/\s]+$", "", text).strip() + + +def _python_comments(source: str) -> list[tuple[int, str]]: + """Comments (tokenizer) and docstring lines (ast) of a Python file.""" + found: list[tuple[int, str]] = [] + for token in tokenize.generate_tokens(io.StringIO(source).readline): + if token.type == tokenize.COMMENT: + found.append((token.start[0], _clean(token.string))) + for node in ast.walk(ast.parse(source)): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + first = node.body[0] if node.body else None + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + for offset, line in enumerate(first.value.value.splitlines()): + found.append((first.value.lineno + offset, _clean(line))) + return found + + +def _scanned_comments(text: str, language) -> list[tuple[int, str]]: + """Comments of any other language, from the scanner of the code counter.""" + state = count_loc._State() + found: list[tuple[int, str]] = [] + lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + for number, line in enumerate(lines, start=1): + pieces: list[str] = [] + count_loc._scan_line(line, language, state, pieces) + if number == 1 and pieces and pieces[0].startswith("!"): + pieces = pieces[1:] # a shebang line is not a comment + joined = " ".join(_clean(piece) for piece in pieces).strip() + if joined: + found.append((number, joined)) + return found + + +def extract(path: str, text: str) -> list[tuple[int, str]]: + """Return (line, text) for every comment line of a source file, sorted by line. + + A file of a language the code counter does not know has no comments here. + """ + language = count_loc.language_of(path) + if language is None: + return [] + if language.name == "Python": + try: + found = _python_comments(text) + except (SyntaxError, tokenize.TokenError): + found = _scanned_comments(text, language) + else: + found = _scanned_comments(text, language) + return sorted((n, t) for n, t in found if t and not _PRAGMA.match(t)) + + +def blocks(lines: list[tuple[int, str]]) -> list[tuple[int, int, str]]: + """Join comment lines on consecutive line numbers: (first line, last line, text).""" + joined: list[tuple[int, int, str]] = [] + for number, text in lines: + if joined and joined[-1][1] in (number, number - 1): + first, _, previous = joined[-1] + joined[-1] = (first, number, f"{previous} {text}") + else: + joined.append((number, number, text)) + return joined + + +def words(text: str, allowed: frozenset[str] = frozenset()) -> list[str]: + """The words of a comment: no URLs, paths, options, identifiers or allowed words.""" + out: list[str] = [] + tokens = text.split() + if tokens and tokens[0].endswith(":") and tokens[0][:-1].isalpha(): + tokens = tokens[1:] # a label such as "Usage:" or "Note:" is not prose + for token in tokens: + if len(token) > 1 and token[0] == "." and token[1].isalpha(): + continue # a hidden file such as .clang-format + token = token.strip("().,;:!?") + # a token with code characters (quotes, braces, `=`, `/`, `_`, digits) is not a word + if not token or _NOT_WORD_CHARS.search(token) or "::" in token: + continue + if token.startswith("-") or "." in token or (len(token) > 1 and token.isupper()): + continue # an option, a file name, an acronym or a constant + if re.search(r"[a-z][A-Z]", token): + continue + for part in token.split("-"): + if len(part) > 1 and _WORD.match(part) and part.lower() not in allowed: + out.append(part) + return out + + +_identifier = None + + +def _identifier_model(): + global _identifier + if _identifier is None: + try: + from langid.langid import LanguageIdentifier, model + except ImportError: + raise CheckError("langid is not installed (pip install langid==1.1.6).") from None + _identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True) + _identifier.set_languages(CANDIDATES) + return _identifier + + +def judge( + text: str, + allowed: frozenset[str] = frozenset(), + threshold: float = DEFAULT_THRESHOLD, + min_words: int = DEFAULT_MIN_WORDS, +) -> tuple[str, float] | None: + """Return (language code, probability) when the text is not English, else None.""" + kept = words(text, allowed) + if len(kept) < min_words: + return None + code, probability = _identifier_model().classify(" ".join(kept)) + if code != "en" and probability >= threshold: + return code, float(probability) + return None + + +def check_text( + path: str, + text: str, + lines: set[int] | None = None, + allowed: frozenset[str] = frozenset(), + threshold: float = DEFAULT_THRESHOLD, + min_words: int = DEFAULT_MIN_WORDS, +) -> list[Finding]: + """Findings of one file; with ``lines`` only the blocks that touch those line numbers.""" + findings: list[Finding] = [] + for first, last, body in blocks(extract(path, text)): + if IGNORE_MARKER.search(body): + continue + if lines is not None and not any(first <= n <= last for n in lines): + continue + verdict = judge(body, allowed, threshold, min_words) + if verdict: + findings.append(Finding(path, first, last, verdict[0], round(verdict[1], 2), body)) + return findings + + +def read_allowed_words(path: str | None) -> frozenset[str]: + """Words, one per line, ``#`` starts a comment; the file may be missing only when not named.""" + if not path: + return frozenset() + try: + content = Path(path).read_text(encoding="utf-8") + except OSError: + raise CheckError(f"allowed-words file '{path}' cannot be read.") from None + return frozenset( + line.split("#", 1)[0].strip().lower() + for line in content.splitlines() + if line.split("#", 1)[0].strip() + ) + + +def _git(directory: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-c", "core.quotepath=off", *args], + cwd=directory, + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + if result.returncode != 0: + raise CheckError(f"git {' '.join(args[:2])} failed: {result.stderr.strip()[:200]}") + return result.stdout + + +def changed_lines(directory: Path, base: str) -> dict[str, set[int]]: + """Added or changed line numbers per file since the merge base with ``base``.""" + out = _git(directory, "diff", "-U0", "--no-color", "--diff-filter=ACMR", f"{base}...HEAD") + result: dict[str, set[int]] = {} + current: set[int] | None = None + for line in out.splitlines(): + if line.startswith("+++ "): + name = line[4:] + current = result.setdefault(name[2:], set()) if name.startswith("b/") else None + continue + hunk = _HUNK.match(line) + if hunk and current is not None: + start = int(hunk.group(1)) + size = int(hunk.group(2)) if hunk.group(2) is not None else 1 + current.update(range(start, start + size)) + return result + + +def tracked_files(directory: Path) -> list[str]: + return [name for name in _git(directory, "ls-files").splitlines() if name] + + +def _excluded(path: str, excludes) -> bool: + parts = path.split("/") + if any(part in count_loc.DEFAULT_EXCLUDED_DIRS for part in parts[:-1]): + return True + return excludes.matches(parts[-1], path) + + +def check_repository( + directory: Path, + base: str | None, + check_all: bool, + patterns: list[str], + allowed: frozenset[str], + threshold: float, + min_words: int, +) -> tuple[list[Finding], int]: + """Return (findings, number of files read).""" + excludes = count_loc.Excludes(patterns) + if check_all: + selection: dict[str, set[int] | None] = {name: None for name in tracked_files(directory)} + else: + if not base: + raise CheckError("give --base (a git ref such as origin/main) or --all.") + selection = dict(changed_lines(directory, base)) + findings: list[Finding] = [] + read = 0 + for name in sorted(selection): + if count_loc.language_of(name) is None or _excluded(name, excludes): + continue + target = directory / name + if not target.is_file() or target.stat().st_size > MAX_FILE_BYTES: + continue + data = target.read_bytes() + if not count_loc._is_text(data): + continue + read += 1 + text = data.decode("utf-8", errors="replace") + findings += check_text(name, text, selection[name], allowed, threshold, min_words) + return findings, read + + +def describe(finding: Finding) -> str: + name = LANGUAGE_NAMES.get(finding.language, finding.language) + shown = finding.text if len(finding.text) <= 120 else finding.text[:117] + "..." + return f'looks {name} ({finding.probability}): "{shown}"' + + +def _split(value: str) -> list[str]: + return [item.strip() for item in re.split(r"[,\n]", value) if item.strip()] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Check that source code comments are in English.") + parser.add_argument("command", choices=["check"]) + parser.add_argument("--directory", default=".") + parser.add_argument("--base", default="") + parser.add_argument("--scope", choices=["changed", "all"], default="changed") + parser.add_argument("--exclude", default="") + parser.add_argument("--allowed-words", default="") + parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD) + parser.add_argument("--min-words", type=int, default=DEFAULT_MIN_WORDS) + parser.add_argument("--output", default="") + args = parser.parse_args(argv) + try: + if not 0.0 < args.threshold <= 1.0 or args.min_words < 1: + raise CheckError("threshold must be above 0 and at most 1, and min-words at least 1.") + directory = Path(args.directory) + if not directory.is_dir(): + raise CheckError(f"directory '{args.directory}' does not exist.") + allowed = read_allowed_words(args.allowed_words or None) + findings, files = check_repository( + directory, + args.base or None, + args.scope == "all", + _split(args.exclude), + allowed, + args.threshold, + args.min_words, + ) + except CheckError as error: + print(f"::error::[configuration] {error}") + return 2 + for finding in findings: + message = describe(finding) + print(f"::error file={finding.path},line={finding.line}::[english-comments] {message}") + print(f"{finding.path}:{finding.line}: {message}") + if args.output: + Path(args.output).write_text( + json.dumps([asdict(f) for f in findings], indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + summary = f"{len(findings)} comment(s) not in English in {files} file(s) checked." + print(summary if findings else f"All comments are in English: {files} file(s) checked.") + return 1 if findings else 0 + + +if __name__ == "__main__": + sys.exit(main())