MIL-001: repository setup and tooling #35
@@ -0,0 +1,23 @@
|
|||||||
|
# English comments: fails a pull request that adds a comment or docstring that
|
||||||
|
# is not English (the project's language rule for source code).
|
||||||
|
#
|
||||||
|
# Triggers: every pull request. Calls the reusable workflow of
|
||||||
|
# TirSystem/git-workflows at the major tag v2 (same organisation). It changes
|
||||||
|
# no file, needs no secret and writes one comment on the pull request.
|
||||||
|
# Implements MIL-001. Verified on the Gitea instance git.tirsystem.com.
|
||||||
|
name: English comments
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: english-comments-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
english-comments:
|
||||||
|
name: Comments are English
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
pull-requests: write
|
||||||
|
uses: TirSystem/git-workflows/.gitea/workflows/english-comments.yml@v2
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Quality checks (ruff, mypy) for the Python files of this repository.
|
||||||
|
#
|
||||||
|
# Triggers: every pull request and every push to main.
|
||||||
|
# Calls the reusable workflow of TirSystem/git-workflows at the major tag v2
|
||||||
|
# (same organisation, the tag moves with compatible releases). It changes no
|
||||||
|
# file, needs no secret and writes nothing outside the repository.
|
||||||
|
# Implements MIL-001 (Go/No-Go criterion 5: CI evidence). Verified on the
|
||||||
|
# Gitea instance git.tirsystem.com.
|
||||||
|
name: Quality
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: quality-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
quality:
|
||||||
|
name: Quality checks
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
uses: TirSystem/git-workflows/.gitea/workflows/quality-checks.yml@v2
|
||||||
|
with:
|
||||||
|
languages: python
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
# Publish accepted milestones to the git host.
|
||||||
|
#
|
||||||
|
# Triggers: a push to main that changes docs/milestones/ or docs/sqa/reviews/,
|
||||||
|
# and a manual run (dry run by default). It runs the framework's
|
||||||
|
# ci-publish-milestones.sh, which creates or updates the Milestone and the
|
||||||
|
# Issues of every milestone that is Accepted with a Go review. Nothing is closed
|
||||||
|
# or deleted. Secret: TOKEN_FOR_GITEA_SYNC (a Gitea token with read and write on
|
||||||
|
# issues; S01 creates it in the repository settings). Changes outside the
|
||||||
|
# repository: Milestones and Issues on the git host. Copied from
|
||||||
|
# framework/templates/ci/sync-milestones.yml; implements MIL-001.
|
||||||
|
|
||||||
|
name: Publish accepted milestones
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
paths:
|
||||||
|
- "docs/milestones/**"
|
||||||
|
- "docs/sqa/reviews/**"
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
dry_run:
|
||||||
|
description: "Preview only: print the plan and create nothing"
|
||||||
|
type: boolean
|
||||||
|
default: true
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: publish-accepted-milestones
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish:
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
steps:
|
||||||
|
- name: Check out the project and its framework submodule
|
||||||
|
uses: actions/checkout@v4.2.2
|
||||||
|
with:
|
||||||
|
submodules: recursive
|
||||||
|
token: ${{ secrets.TOKEN_FOR_GITEA_SYNC }}
|
||||||
|
|
||||||
|
- name: Make sure curl and jq are available
|
||||||
|
run: bash framework/scripts/ci-publish-milestones.sh ensure-tools
|
||||||
|
|
||||||
|
- name: Publish accepted milestones
|
||||||
|
env:
|
||||||
|
# Gitea reads GITEA_TOKEN, GitHub's gh reads GH_TOKEN; both come from
|
||||||
|
# the one secret named above.
|
||||||
|
GITEA_TOKEN: ${{ secrets.TOKEN_FOR_GITEA_SYNC }}
|
||||||
|
GH_TOKEN: ${{ secrets.TOKEN_FOR_GITEA_SYNC }}
|
||||||
|
# Empty on a push (a real run); "true" or "false" on a manual run.
|
||||||
|
DRY_RUN: ${{ inputs.dry_run }}
|
||||||
|
run: bash framework/scripts/ci-publish-milestones.sh publish
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
# Tests and code metrics.
|
||||||
|
#
|
||||||
|
# Triggers: every pull request (tests only) and every push to main (tests, then
|
||||||
|
# size and coverage badges). The tests run offline. Needs no secret. The metrics
|
||||||
|
# job commits three SVG badges to the badges branch of this repository through
|
||||||
|
# the reusable workflow of TirSystem/git-workflows at the major tag v2 (same
|
||||||
|
# organisation). Implements MIL-001. Verified on git.tirsystem.com.
|
||||||
|
name: Tests and code metrics
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: tests-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
tests:
|
||||||
|
name: Run pytest with coverage
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- name: Check out the repository
|
||||||
|
uses: actions/checkout@v4.2.2
|
||||||
|
- name: Set up Python 3.13
|
||||||
|
uses: actions/setup-python@v5.3.0
|
||||||
|
with:
|
||||||
|
python-version: "3.13"
|
||||||
|
- name: Install the project and its test tools
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
python -m pip install -e ".[dev]"
|
||||||
|
- name: Run the tests offline
|
||||||
|
run: python -m pytest --cov=scrape_movies --cov-report=xml
|
||||||
|
- name: Keep the coverage report
|
||||||
|
uses: actions/upload-artifact@v4.6.2
|
||||||
|
with:
|
||||||
|
name: coverage
|
||||||
|
path: coverage.xml
|
||||||
|
|
||||||
|
metrics:
|
||||||
|
name: Code metrics badges
|
||||||
|
if: ${{ github.event_name == 'push' }}
|
||||||
|
needs: tests
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
uses: TirSystem/git-workflows/.gitea/workflows/code-metrics.yml@v2
|
||||||
|
with:
|
||||||
|
coverage-artifact: coverage
|
||||||
|
coverage-report: coverage.xml
|
||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
# Secrets: personal tokens are never part of the project
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
|
||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*.egg-info/
|
||||||
|
build/
|
||||||
|
dist/
|
||||||
|
.eggs/
|
||||||
|
|
||||||
|
# Virtual environments
|
||||||
|
.venv/
|
||||||
|
venv/
|
||||||
|
env/
|
||||||
|
|
||||||
|
# Test, type and lint caches
|
||||||
|
.pytest_cache/
|
||||||
|
.mypy_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
.coverage
|
||||||
|
coverage.xml
|
||||||
|
htmlcov/
|
||||||
|
|
||||||
|
# Doxygen output
|
||||||
|
docs/api/
|
||||||
|
|
||||||
|
# Editors and operating systems
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# AI prompt log (public repository)
|
||||||
|
/docs/ai-prompts/
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
extends: relaxed
|
||||||
|
rules:
|
||||||
|
line-length:
|
||||||
|
max: 120
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# Doxygen configuration. Run `doxygen Doxyfile` from the repository root;
|
||||||
|
# the HTML pages are written to docs/api/html (ignored by git).
|
||||||
|
PROJECT_NAME = "scrape-movies"
|
||||||
|
PROJECT_BRIEF = "Day 45 of 100 Days of Code: web scraping with BeautifulSoup"
|
||||||
|
OUTPUT_DIRECTORY = docs/api
|
||||||
|
INPUT = src README.md
|
||||||
|
FILE_PATTERNS = *.py *.md
|
||||||
|
RECURSIVE = YES
|
||||||
|
USE_MDFILE_AS_MAINPAGE = README.md
|
||||||
|
OPTIMIZE_OUTPUT_JAVA = YES
|
||||||
|
EXTRACT_ALL = YES
|
||||||
|
AUTOLINK_SUPPORT = YES
|
||||||
|
JAVADOC_AUTOBRIEF = YES
|
||||||
|
GENERATE_HTML = YES
|
||||||
|
GENERATE_LATEX = NO
|
||||||
|
QUIET = YES
|
||||||
@@ -1,2 +1,108 @@
|
|||||||
# 045-scrape-movies
|
# 🚀 scrape-movies
|
||||||
|
|
||||||
|
Day 45 of Udemy's "100 Days of Code: The Complete Python Pro Bootcamp": web scraping with Requests and BeautifulSoup. It parses a local HTML page, scrapes the Hacker News front page and writes Empire's 100 greatest movies to `movies.txt`. It is for fellow coursists and anyone looking for a small, tested scraping example.
|
||||||
|
|
||||||
|
## 📚 Table of Contents
|
||||||
|
|
||||||
|
- [Overview](#-overview)
|
||||||
|
- [Requirements](#-requirements)
|
||||||
|
- [Setup](#-setup)
|
||||||
|
- [Run](#-run)
|
||||||
|
- [Tests](#-tests)
|
||||||
|
- [License](#-license)
|
||||||
|
- [Links](#-links)
|
||||||
|
- [Appendix: Project documents](#-appendix-project-documents)
|
||||||
|
|
||||||
|
## 🧭 Overview
|
||||||
|
|
||||||
|
| Part | What it is for |
|
||||||
|
| --- | --- |
|
||||||
|
| `src/scrape_movies/` | The code in clean-architecture layers (`domain`, `application`, `infrastructure`, `presentation`) plus `constants.py`; the lecture steps `soup_basics`, `hacker_news` and the movies scraper follow milestone by milestone |
|
||||||
|
| `tests/` | pytest tests that run offline against saved pages in `tests/fixtures/` |
|
||||||
|
| `docs/` | Project documents (business case, plan, milestones, reviews) and the wiki in English, Danish and Bahasa Malaysia |
|
||||||
|
| `Doxyfile` | Generates the API pages from the Doxygen comments in the source |
|
||||||
|
|
||||||
|
The only runtime dependencies are `requests` and `beautifulsoup4`.
|
||||||
|
|
||||||
|
## 📋 Requirements
|
||||||
|
|
||||||
|
- Python 3.13 or newer (the `venv` module comes with it).
|
||||||
|
- `requests` and `beautifulsoup4` (installed by the setup below).
|
||||||
|
- Optional: [Doxygen](https://www.doxygen.nl/) to build the API pages.
|
||||||
|
|
||||||
|
## 🛠️ Setup
|
||||||
|
|
||||||
|
Create a local virtual environment named `.venv`, update `pip`, then install the project with its test tools.
|
||||||
|
|
||||||
|
Windows (PowerShell):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
python -m venv .venv
|
||||||
|
.venv\Scripts\Activate.ps1
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
python -m pip install -e ".[dev]"
|
||||||
|
```
|
||||||
|
|
||||||
|
Linux and macOS:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 -m venv .venv
|
||||||
|
source .venv/bin/activate
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
python -m pip install -e ".[dev]"
|
||||||
|
```
|
||||||
|
|
||||||
|
Leave out `[dev]` to install only the runtime dependencies. Personal tokens belong in a git-ignored `.env`; the project never reads it.
|
||||||
|
|
||||||
|
## ▶️ Run
|
||||||
|
|
||||||
|
The lecture steps are added milestone by milestone; each gets a section with its command, its output and a screenshot here.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m scrape_movies # available from the movies milestone
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🧪 Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m pytest
|
||||||
|
```
|
||||||
|
|
||||||
|
The tests use saved pages and never touch the network. Lint and types: `ruff check .`, `ruff format --check .` and `mypy src tests`. API pages: `doxygen Doxyfile`.
|
||||||
|
|
||||||
|
## 📄 License
|
||||||
|
|
||||||
|
See [LICENSE](./LICENSE).
|
||||||
|
|
||||||
|
## 🔗 Links
|
||||||
|
|
||||||
|
- [Repository][repository]
|
||||||
|
- [Documentation][documentation]
|
||||||
|
- [Issue tracker][issues]
|
||||||
|
|
||||||
|
## 📎 Appendix: Project documents
|
||||||
|
|
||||||
|
| Document | What it tells you |
|
||||||
|
| --- | --- |
|
||||||
|
| [AGENTS.md][agents] | The rules AI agents follow in this repository |
|
||||||
|
| [Business Case][BC-001] | Why the project exists, its scope and success criteria |
|
||||||
|
| [Stakeholder Analysis][SA-001] | Who is involved; the stakeholder IDs used for owners and reviewers |
|
||||||
|
| [Project Plan][PP-001] | The gateways and their schedule |
|
||||||
|
| [Milestones][milestones] | One document per gateway: Go/No-Go criteria and tasks |
|
||||||
|
| [Artifact registry][registry] | Where each artifact type lives, the next version, the PO language and domain |
|
||||||
|
| [Framework README][framework-readme] | The framework this project uses |
|
||||||
|
| [Framework changelog][framework-changelog] | What changed in each framework release |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
[repository]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies
|
||||||
|
[documentation]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/wiki
|
||||||
|
[issues]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/issues
|
||||||
|
[agents]: ./AGENTS.md
|
||||||
|
[BC-001]: ./docs/business-case.md
|
||||||
|
[SA-001]: ./docs/stakeholder-analysis.md
|
||||||
|
[PP-001]: ./docs/project-plan.md
|
||||||
|
[milestones]: ./docs/milestones/
|
||||||
|
[registry]: ./docs/artifact-registry.md
|
||||||
|
[framework-readme]: ./framework/README.md
|
||||||
|
[framework-changelog]: ./framework/CHANGELOG.md
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ document of a type. `Primary File` may contain a glob (e.g.
|
|||||||
| SA | Stakeholder Analysis | docs/stakeholder-analysis.md | 002 |
|
| SA | Stakeholder Analysis | docs/stakeholder-analysis.md | 002 |
|
||||||
| PP | Project Plan | docs/project-plan.md | 002 |
|
| PP | Project Plan | docs/project-plan.md | 002 |
|
||||||
| MIL | Milestone / Gateway | docs/milestones/*.md | 006 |
|
| MIL | Milestone / Gateway | docs/milestones/*.md | 006 |
|
||||||
| RC | SQA Review Record | docs/sqa/reviews/rc-*.md | 009 |
|
| RC | SQA Review Record | docs/sqa/reviews/rc-*.md | 012 |
|
||||||
|
|
||||||
## Languages
|
## Languages
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
# Review of MIL-001 Python code
|
||||||
|
|
||||||
|
## Metadata
|
||||||
|
| Key | Value |
|
||||||
|
| --- | --- |
|
||||||
|
| ID | RC-009 |
|
||||||
|
| CrossReference | [MIL-001], [QC-PY-001] |
|
||||||
|
|
||||||
|
## Version History
|
||||||
|
| Date | Status | Author | Reviewer | Change | Commit |
|
||||||
|
| --- | --- | --- | --- | --- | --- |
|
||||||
|
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [44a3b7b] |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Artifact Under Review
|
||||||
|
|
||||||
|
- Instance reviewed: [MIL-001]
|
||||||
|
- Checklist used: [QC-PY-001]
|
||||||
|
- Scope: full review. Code reviewed: pyproject.toml, src/, tests/ of the MIL-001 branch.
|
||||||
|
- Language and domain: n/a (technical type)
|
||||||
|
- Language reviewer: none
|
||||||
|
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||||
|
|
||||||
|
## Checklist Results
|
||||||
|
|
||||||
|
| # | Criterion | Status | Evidence/Notes |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| 1 | Packages, modules, functions, variables, classes and constants follow PEP 8 casing (`snake_case`, `PascalCase`, `UPPER_SNAKE`) | Pass | `constants.py` uses UPPER_SNAKE; packages are snake_case. |
|
||||||
|
| 2 | Names state purpose in the domain's language; no unexplained abbreviations, no single-letter names outside tiny scopes | Pass | names state purpose (`REQUEST_TIMEOUT_SECONDS`, `EMPIRE_WAYBACK_URL`). |
|
||||||
|
| 3 | Code is produced by the project's formatter and passes its linter with no unexplained suppressions | Pass | `ruff check` and `ruff format --check` clean; no `noqa`; only D203/D213 are configured off because they conflict with D211/D212. |
|
||||||
|
| 4 | Every function and method signature is type-annotated, including `-> None` | Pass | the one test function is annotated; `mypy --strict` has no error. |
|
||||||
|
| 5 | No bare `except:`, no swallowed exceptions; specific exceptions are raised and the cause is kept (`raise ... from`) | N-A | no exception handling exists yet. |
|
||||||
|
| 6 | No mutable default arguments and no shadowed builtins | Pass | none present. |
|
||||||
|
| 7 | Files, locks and connections are managed with context managers | N-A | no file or connection is opened yet. |
|
||||||
|
| 8 | Public modules, classes and functions have docstrings that say what, not how | Pass | every module has a Doxygen docstring. |
|
||||||
|
| 9 | Logging uses `logging`, not `print`; no secrets or personal data in log output | N-A | no logging exists yet. |
|
||||||
|
| 10 | Classes and operations trace to the Design Class Diagram they implement; deviations are recorded | N-A | no Design Class Diagram; every task of MIL-001 is a plain technical task (Needs Use Case: No). |
|
||||||
|
| 11 | Tests exist for new behaviour, are named for the behaviour, and do not depend on order or the network | Pass | `tests/unit/test_constants.py` is offline and independent of order. |
|
||||||
|
| 12 | Type checker runs in strict mode without errors; `Any` is justified in a comment | Pass | `mypy --strict` over `src` and `tests`. |
|
||||||
|
| 13 | Dependencies are declared and pinned in the project's dependency file, none unused | Fail | dependencies are declared with lower bounds, not pinned; this is a library-style project without a lock file. |
|
||||||
|
| 14 | Each piece of knowledge (a business rule, constant, format, validation or query) is defined in one place; code is merged only where it expresses the same knowledge, not where it merely looks alike | Pass | every constant is defined once in `constants.py`. |
|
||||||
|
| 15 | Imports point inward: business-rule modules import no framework, database, UI or delivery-mechanism code, and there are no import cycles | Pass | the layer packages hold no imports yet; no cycle is possible. |
|
||||||
|
| 16 | SOLID holds: a class or module has one reason to change; new behaviour is added by extension, not by editing a type test in several places; subclasses honour the contract of their base; protocols are narrow; high-level code depends on protocols or abstract types and receives its implementations | Pass | no class or branching on type exists yet. |
|
||||||
|
| 17 | Source is laid out in the layers and folders of the clean architecture folder structure (`domain`, `application`, `infrastructure`, `presentation` packages under `src/`, and `unit`, `integration` and `e2e` under `tests/`), in `snake_case`, and imports between the layers point inward | Pass | `domain`, `application`, `infrastructure` and `presentation` under `src/scrape_movies/`; `unit`, `integration`, `e2e` under `tests/`. |
|
||||||
|
| 18 | File and folder names are English, contain no space and no character outside letters, digits, `-`, `_` and `.`, and follow the casing of the language for files | Pass | all names are English lowercase with `_` and no space. |
|
||||||
|
|
||||||
|
## Overall Verdict
|
||||||
|
|
||||||
|
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||||
|
|
||||||
|
## Action Items
|
||||||
|
|
||||||
|
| Action | Owner | Due |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Decide whether to add a lock file for exact versions (optional criterion 13) | S01 | 2026-11-14 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
[MIL-001]: ../../milestones/mil-001-repository-setup.md
|
||||||
|
[QC-PY-001]: ../../../framework/qc/code/qc-programming-python.md
|
||||||
|
[44a3b7b]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/44a3b7be335b657bdd91a533e7582743f9448b1b
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
# Review of MIL-001 workflows
|
||||||
|
|
||||||
|
## Metadata
|
||||||
|
| Key | Value |
|
||||||
|
| --- | --- |
|
||||||
|
| ID | RC-010 |
|
||||||
|
| CrossReference | [MIL-001], [QC-WF-001] |
|
||||||
|
|
||||||
|
## Version History
|
||||||
|
| Date | Status | Author | Reviewer | Change | Commit |
|
||||||
|
| --- | --- | --- | --- | --- | --- |
|
||||||
|
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [44a3b7b] |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Artifact Under Review
|
||||||
|
|
||||||
|
- Instance reviewed: [MIL-001]
|
||||||
|
- Checklist used: [QC-WF-001]
|
||||||
|
- Scope: full review. Code reviewed: .gitea/workflows/*.yml.
|
||||||
|
- Language and domain: n/a (technical type)
|
||||||
|
- Language reviewer: none
|
||||||
|
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||||
|
|
||||||
|
## Checklist Results
|
||||||
|
|
||||||
|
| # | Criterion | Status | Evidence/Notes |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| 1 | The file meets the YAML checklist ([QC-YML-001]) | Pass | see RC-011 for QC-YML-001. |
|
||||||
|
| 2 | The file is in `.gitea/workflows/` or `.github/workflows/` with the extension `.yml` or `.yaml`, and the same workflow does not exist in both directories | Pass | all five files are `.yml` under `.gitea/workflows/`; none under `.github/`. |
|
||||||
|
| 3 | The file declares `name`, `on` and `jobs`; `on` lists explicit events and, where it matters, `branches` or `paths` | Pass | each declares `name`, `on` and `jobs`, with branches or paths where they matter. |
|
||||||
|
| 4 | A header comment states the purpose, the triggers, the inputs, the secrets it needs, what it changes outside the repository, and the decision or milestone it implements | Pass | every file opens with a header naming purpose, triggers, secrets, outside effects and MIL-001. |
|
||||||
|
| 5 | Every job and step has a descriptive `name`, unique within its job | Pass | every job and step has a unique `name`. |
|
||||||
|
| 6 | The file passes `actionlint` (or the project's workflow validation test) and the tool version is stated | Pass | actionlint 1.7.12 reports no finding; `gitea.*` contexts were replaced by `github.*` to satisfy it. |
|
||||||
|
| 7 | Every job declares `permissions` explicitly with the least access that works (`contents: read` or none by default); write access only on the job that needs it | Pass | every job sets `permissions`; write only on the metrics job. |
|
||||||
|
| 8 | Every action used is pinned to a full commit SHA or an exact release tag, never to a branch; a moving major tag (`@v1`) is accepted only for an action published by the same organisation under a stated release policy | Pass | actions are at exact tags (`checkout@v4.2.2`, `setup-python@v5.3.0`, `upload-artifact@v4.6.2`); the reusable workflows of the same organisation use the major tag `v2` as the organisation publishes it. |
|
||||||
|
| 9 | No secret is written in the file; a secret is read as `secrets.NAME`, passed to a step through `env:`, never placed on a command line or echoed | Pass | the only secret is read as `secrets.TOKEN_FOR_GITEA_SYNC` through `env:`. |
|
||||||
|
| 10 | Untrusted text (`inputs.*`, event fields such as a title, branch name or label, step outputs derived from them) is never interpolated into a `run:` script; it is passed through `env:` and used as a quoted variable | Pass | `inputs.dry_run` reaches the script only through `env:`. |
|
||||||
|
| 11 | A trigger that gives a run secrets or a write token on code from a fork or an untrusted branch (`pull_request_target`, `workflow_run`) does not check out or execute that code | Pass | no `pull_request_target` or `workflow_run`. |
|
||||||
|
| 12 | A workflow that changes state outside the repository (release, tag, publish, sync, deploy) is started by hand or by a protected branch, and a manual run defaults to a dry run with an explicit input to apply | Pass | `sync-milestones` runs on main or by hand and defaults to a dry run on a manual start. |
|
||||||
|
| 13 | The token the workflow uses has only the scope the job needs, and where the host's built-in token is enough it is used instead of a personal token | N-A | the token is created by S01; its scope cannot be checked here. |
|
||||||
|
| 14 | Every job sets `runs-on` to an explicit label and `timeout-minutes` | Pass | every job with `runs-on` sets `ubuntu-latest` and `timeout-minutes`; called workflows set their own. |
|
||||||
|
| 15 | `concurrency` is set where two runs would conflict: a check cancels the older run (`cancel-in-progress: true`); a publishing job queues (`cancel-in-progress: false`) | Pass | checks cancel older runs; the publishing job queues. |
|
||||||
|
| 16 | Triggers are filtered (`branches`, `paths`, `types`) so the workflow runs only for the changes it concerns | Pass | triggers use `branches` or `paths`. |
|
||||||
|
| 17 | Tools and dependencies are installed at pinned versions; a version is a workflow input with a default or a value in one place | Pass | Python 3.13 and the tool versions come from one place, `pyproject.toml`; actionlint and yamllint were run by hand at the versions stated here. |
|
||||||
|
| 18 | A dependency cache is used where the install is slow enough to matter | N-A | the install is small enough that a cache does not matter. |
|
||||||
|
| 19 | A failing step names the tool, the file and the reason in its log (annotation or `::error::` line) | N-A | the called tools write their own annotations. |
|
||||||
|
| 20 | A `run:` block is within the limit on embedded program text of criterion 7 of [QC-YML-001]; a longer script is a file (`tools/`, or a shared repository fetched at a pinned ref) that the step calls with one command, and the workflow passes its settings through `env:` | Pass | no `run:` block exceeds five lines. |
|
||||||
|
| 21 | Logic used by more than one workflow or repository is a reusable workflow (`workflow_call`) or a composite action; each input has a `description`, a `type` and a `default` where one is sensible | Pass | quality, comments and metrics reuse the shared workflows with `uses:`. |
|
||||||
|
| 22 | Every workflow feature used is verified on the instance that runs it, and the host and version are recorded; a feature the target hosts differ on is noted in the header | Pass | on pull request 35 on git.tirsystem.com the tests, quality and English-comments runs completed with success. |
|
||||||
|
| 23 | A reusable workflow states, in its header, how a caller pins it and the script it fetches, so the workflow and its script are the same release | Pass | the headers say the callers use the major tag `v2` of the same organisation. |
|
||||||
|
| 24 | A static test of the repository's workflows exists and runs on every pull request: valid YAML, required keys, explicit permissions and runner, no inline script above the limit | Fail | no static workflow test runs in this repository; actionlint was run by hand. |
|
||||||
|
|
||||||
|
## Overall Verdict
|
||||||
|
|
||||||
|
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||||
|
|
||||||
|
## Action Items
|
||||||
|
|
||||||
|
| Action | Owner | Due |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Add a static workflow test (optional criterion 24) | S01 | 2026-11-14 |
|
||||||
|
| Create the secret TOKEN_FOR_GITEA_SYNC (S01 only; the AI does not create secrets) | S01 | 2026-11-14 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
[MIL-001]: ../../milestones/mil-001-repository-setup.md
|
||||||
|
[QC-WF-001]: ../../../framework/qc/code/qc-workflow.md
|
||||||
|
[44a3b7b]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/44a3b7be335b657bdd91a533e7582743f9448b1b
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
# Review of MIL-001 YAML files
|
||||||
|
|
||||||
|
## Metadata
|
||||||
|
| Key | Value |
|
||||||
|
| --- | --- |
|
||||||
|
| ID | RC-011 |
|
||||||
|
| CrossReference | [MIL-001], [QC-YML-001] |
|
||||||
|
|
||||||
|
## Version History
|
||||||
|
| Date | Status | Author | Reviewer | Change | Commit |
|
||||||
|
| --- | --- | --- | --- | --- | --- |
|
||||||
|
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [44a3b7b] |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Artifact Under Review
|
||||||
|
|
||||||
|
- Instance reviewed: [MIL-001]
|
||||||
|
- Checklist used: [QC-YML-001]
|
||||||
|
- Scope: full review. Code reviewed: .gitea/workflows/*.yml, .yamllint.yml.
|
||||||
|
- Language and domain: n/a (technical type)
|
||||||
|
- Language reviewer: none
|
||||||
|
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||||
|
|
||||||
|
## Checklist Results
|
||||||
|
|
||||||
|
| # | Criterion | Status | Evidence/Notes |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| 1 | The file parses with a safe YAML loader and has no duplicate keys in any mapping | Pass | yamllint 1.38.0 (PyYAML based) parses all files and reports no duplicate key. |
|
||||||
|
| 2 | Indentation is 2 spaces everywhere, never tabs; lines end with LF; the file ends with one newline and has no trailing spaces | Pass | 2 spaces, LF, one final newline, no trailing spaces (yamllint relaxed profile). |
|
||||||
|
| 3 | Passes `yamllint` (or the project's YAML linter) with its configuration committed next to the code and no unexplained `disable` comments | Pass | `.yamllint.yml` is committed; no disable comments. |
|
||||||
|
| 4 | A scalar that YAML would coerce to another type is quoted: `yes`, `no`, `on`, `off`, `null`, `~`, numbers with a leading zero, versions such as `1.10`, and values that contain `: ` or ` #` | Pass | strings such as `"3.13"` are quoted; `on` is the Actions trigger key, not a value. |
|
||||||
|
| 5 | Only safe, plain types are used: no language-specific tags (`!!python/object`, `!ruby/object`) and no `!include`-style extension unless the consuming tool documents it | Pass | only plain types. |
|
||||||
|
| 6 | No secret is written in the file: no password, token, key or private connection string; a secret is referenced by name and supplied by the consuming tool | Pass | no secret in any file. |
|
||||||
|
| 7 | Embedded program text (shell, Python, SQL, `jq`, and so on) in a value is at most five lines; a longer one is moved to a file of its own, in its own language, and the YAML calls it | Pass | the longest `run:` block has two lines. |
|
||||||
|
| 8 | A block of keys or values repeated in the file or across files is defined once (an anchor, a template, an include or a reusable unit the consumer supports) | Pass | repeated blocks are not present; shared logic is reused through `uses:`. |
|
||||||
|
| 9 | Anchors, aliases and merge keys (`&`, `*`, `<<`) are used only where the consuming tool is known to support them | Pass | no anchors or aliases. |
|
||||||
|
| 10 | The file validates against the schema of the consuming tool where one exists, and the schema or tool version is stated | Pass | actionlint 1.7.12 validates the workflow schema. |
|
||||||
|
| 11 | A header comment states what the file is for, which tool reads it, and where its decision or design is recorded | Pass | each file has a header comment saying purpose, reader and where it is decided (MIL-001). |
|
||||||
|
| 12 | Key names follow the convention of the consuming tool; one case style is used within the file (kebab-case or snake_case, never mixed) | Pass | key names are those of the consuming tool (`cancel-in-progress`, `timeout-minutes`, `dry_run`), as the tool defines them. |
|
||||||
|
| 13 | Multi-line strings use a block scalar (`\|` or `>`) and state the chomping indicator (`\|-`, `\|+`) where the trailing newline matters | Pass | no multi-line strings other than `run: |` blocks where the trailing newline does not matter. |
|
||||||
|
| 14 | The file name is lowercase kebab-case with one extension style (`.yml` or `.yaml`) used throughout the repository | Pass | lowercase kebab-case `.yml` names. |
|
||||||
|
| 15 | A change is exercised by a check that runs without the real target: a parse and schema test, or a dry run of the consuming tool | Pass | pull request 35 ran the workflows on the host. |
|
||||||
|
|
||||||
|
## Overall Verdict
|
||||||
|
|
||||||
|
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||||
|
|
||||||
|
## Action Items
|
||||||
|
|
||||||
|
| Action | Owner | Due |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| None | S01 | 2026-11-14 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
[MIL-001]: ../../milestones/mil-001-repository-setup.md
|
||||||
|
[QC-YML-001]: ../../../framework/qc/code/qc-yaml.md
|
||||||
|
[44a3b7b]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/44a3b7be335b657bdd91a533e7582743f9448b1b
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=68"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "scrape-movies"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Day 45 of 100 Days of Code: web scraping with Requests and BeautifulSoup (Hacker News and the Empire 100 greatest movies)."
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.13"
|
||||||
|
license = { file = "LICENSE" }
|
||||||
|
dependencies = [
|
||||||
|
"requests>=2.32",
|
||||||
|
"beautifulsoup4>=4.12",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
dev = [
|
||||||
|
"pytest>=8.3",
|
||||||
|
"pytest-cov>=5.0",
|
||||||
|
"ruff>=0.6",
|
||||||
|
"mypy>=1.11",
|
||||||
|
"types-requests>=2.32",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.setuptools.packages.find]
|
||||||
|
where = ["src"]
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
pythonpath = ["src"]
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 88
|
||||||
|
src = ["src", "tests"]
|
||||||
|
extend-exclude = ["framework", ".agents", ".claude"]
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
select = ["E", "F", "I", "N", "UP", "B", "D"]
|
||||||
|
ignore = ["D203", "D213"]
|
||||||
|
|
||||||
|
[tool.ruff.lint.per-file-ignores]
|
||||||
|
"tests/**" = ["D"]
|
||||||
|
|
||||||
|
[tool.mypy]
|
||||||
|
strict = true
|
||||||
|
python_version = "3.13"
|
||||||
|
mypy_path = "src"
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Day 45 scraping exercises: soup basics, Hacker News and 100 movies."""
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Use cases and the ports they need; imports only the domain."""
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
"""@brief All constants of the project (URLs, limits, paths, parser name)."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
## Root folder of the repository (two levels above this file's package folder).
|
||||||
|
PROJECT_ROOT: Path = Path(__file__).resolve().parents[2]
|
||||||
|
## Folder with the saved pages the tests and the lecture steps read.
|
||||||
|
FIXTURES_DIR: Path = PROJECT_ROOT / "tests" / "fixtures"
|
||||||
|
## The sample page from the "Parsing HTML and Making Soup" lecture.
|
||||||
|
WEBSITE_HTML_PATH: Path = FIXTURES_DIR / "website.html"
|
||||||
|
|
||||||
|
## Parser handed to BeautifulSoup; it is part of the standard library.
|
||||||
|
HTML_PARSER: str = "html.parser"
|
||||||
|
|
||||||
|
## Seconds to wait for a web server before giving up.
|
||||||
|
REQUEST_TIMEOUT_SECONDS: int = 10
|
||||||
|
## Sent with every request so a site owner can see who asks.
|
||||||
|
USER_AGENT: str = "scrape-movies/0.1 (Udemy 100 Days of Code, Day 45)"
|
||||||
|
|
||||||
|
## The course's static copy of the Hacker News front page.
|
||||||
|
HACKER_NEWS_URL: str = "https://appbrewery.github.io/news.ycombinator.com/"
|
||||||
|
## The live Hacker News front page (scrape it at most once a minute).
|
||||||
|
HACKER_NEWS_LIVE_URL: str = "https://news.ycombinator.com/news"
|
||||||
|
|
||||||
|
## Internet Archive snapshot of the Empire list, as the assignment requires.
|
||||||
|
EMPIRE_WAYBACK_URL: str = (
|
||||||
|
"https://web.archive.org/web/20200518073855/"
|
||||||
|
"https://www.empireonline.com/movies/features/best-movies-2/"
|
||||||
|
)
|
||||||
|
## The live Empire page, an alternative that may have changed its markup.
|
||||||
|
EMPIRE_LIVE_URL: str = "https://www.empireonline.com/movies/features/best-movies-2/"
|
||||||
|
|
||||||
|
## File the movie list is written to.
|
||||||
|
MOVIES_OUTPUT_FILE: str = "movies.txt"
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Business rules and entities; imports nothing from other layers."""
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Adapters for the web, BeautifulSoup and files; implements the ports."""
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Command-line entry point; calls the use cases."""
|
||||||
Vendored
@@ -0,0 +1,6 @@
|
|||||||
|
from scrape_movies import constants
|
||||||
|
|
||||||
|
|
||||||
|
def test_constants_when_imported_point_into_project() -> None:
|
||||||
|
assert constants.PROJECT_ROOT.name
|
||||||
|
assert constants.FIXTURES_DIR.parent.name == "tests"
|
||||||
Reference in New Issue
Block a user