Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7f269aca0 | ||
|
|
28ac7f3690 | ||
|
|
7b95069d53 | ||
|
|
4ad515fbf7 | ||
|
|
ea4f32f56c |
@@ -1,23 +0,0 @@
|
||||
# English comments: fails a pull request that adds a comment or docstring that
|
||||
# is not English (the project's language rule for source code).
|
||||
#
|
||||
# Triggers: every pull request. Calls the reusable workflow of
|
||||
# TirSystem/git-workflows at the major tag v2 (same organisation). It changes
|
||||
# no file, needs no secret and writes one comment on the pull request.
|
||||
# Implements MIL-001. Verified on the Gitea instance git.tirsystem.com.
|
||||
name: English comments
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: english-comments-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
english-comments:
|
||||
name: Comments are English
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
uses: TirSystem/git-workflows/.gitea/workflows/english-comments.yml@v2
|
||||
@@ -1,27 +0,0 @@
|
||||
# Quality checks (ruff, mypy) for the Python files of this repository.
|
||||
#
|
||||
# Triggers: every pull request and every push to main.
|
||||
# Calls the reusable workflow of TirSystem/git-workflows at the major tag v2
|
||||
# (same organisation, the tag moves with compatible releases). It changes no
|
||||
# file, needs no secret and writes nothing outside the repository.
|
||||
# Implements MIL-001 (Go/No-Go criterion 5: CI evidence). Verified on the
|
||||
# Gitea instance git.tirsystem.com.
|
||||
name: Quality
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
concurrency:
|
||||
group: quality-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
quality:
|
||||
name: Quality checks
|
||||
permissions:
|
||||
contents: read
|
||||
uses: TirSystem/git-workflows/.gitea/workflows/quality-checks.yml@v2
|
||||
with:
|
||||
languages: python
|
||||
@@ -1,55 +0,0 @@
|
||||
# Publish accepted milestones to the git host.
|
||||
#
|
||||
# Triggers: a push to main that changes docs/milestones/ or docs/sqa/reviews/,
|
||||
# and a manual run (dry run by default). It runs the framework's
|
||||
# ci-publish-milestones.sh, which creates or updates the Milestone and the
|
||||
# Issues of every milestone that is Accepted with a Go review. Nothing is closed
|
||||
# or deleted. Secret: TOKEN_FOR_GITEA_SYNC (a Gitea token with read and write on
|
||||
# issues; S01 creates it in the repository settings). Changes outside the
|
||||
# repository: Milestones and Issues on the git host. Copied from
|
||||
# framework/templates/ci/sync-milestones.yml; implements MIL-001.
|
||||
|
||||
name: Publish accepted milestones
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "docs/milestones/**"
|
||||
- "docs/sqa/reviews/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Preview only: print the plan and create nothing"
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
concurrency:
|
||||
group: publish-accepted-milestones
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out the project and its framework submodule
|
||||
uses: actions/checkout@v4.2.2
|
||||
with:
|
||||
submodules: recursive
|
||||
token: ${{ secrets.TOKEN_FOR_GITEA_SYNC }}
|
||||
|
||||
- name: Make sure curl and jq are available
|
||||
run: bash framework/scripts/ci-publish-milestones.sh ensure-tools
|
||||
|
||||
- name: Publish accepted milestones
|
||||
env:
|
||||
# Gitea reads GITEA_TOKEN, GitHub's gh reads GH_TOKEN; both come from
|
||||
# the one secret named above.
|
||||
GITEA_TOKEN: ${{ secrets.TOKEN_FOR_GITEA_SYNC }}
|
||||
GH_TOKEN: ${{ secrets.TOKEN_FOR_GITEA_SYNC }}
|
||||
# Empty on a push (a real run); "true" or "false" on a manual run.
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: bash framework/scripts/ci-publish-milestones.sh publish
|
||||
@@ -1,54 +0,0 @@
|
||||
# Tests and code metrics.
|
||||
#
|
||||
# Triggers: every pull request (tests only) and every push to main (tests, then
|
||||
# size and coverage badges). The tests run offline. Needs no secret. The metrics
|
||||
# job commits three SVG badges to the badges branch of this repository through
|
||||
# the reusable workflow of TirSystem/git-workflows at the major tag v2 (same
|
||||
# organisation). Implements MIL-001. Verified on git.tirsystem.com.
|
||||
name: Tests and code metrics
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
concurrency:
|
||||
group: tests-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Run pytest with coverage
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Check out the repository
|
||||
uses: actions/checkout@v4.2.2
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5.3.0
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- name: Install the project and its test tools
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -e ".[dev]"
|
||||
- name: Run the tests offline
|
||||
run: python -m pytest --cov=scrape_movies --cov-report=xml
|
||||
- name: Keep the coverage report
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: coverage
|
||||
path: coverage.xml
|
||||
|
||||
metrics:
|
||||
name: Code metrics badges
|
||||
if: ${{ github.event_name == 'push' }}
|
||||
needs: tests
|
||||
permissions:
|
||||
contents: write
|
||||
uses: TirSystem/git-workflows/.gitea/workflows/code-metrics.yml@v2
|
||||
with:
|
||||
coverage-artifact: coverage
|
||||
coverage-report: coverage.xml
|
||||
@@ -1,36 +0,0 @@
|
||||
# Secrets: personal tokens are never part of the project
|
||||
.env
|
||||
.env.*
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
build/
|
||||
dist/
|
||||
.eggs/
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
venv/
|
||||
env/
|
||||
|
||||
# Test, type and lint caches
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
|
||||
# Doxygen output
|
||||
docs/api/
|
||||
|
||||
# Editors and operating systems
|
||||
.idea/
|
||||
.vscode/
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# AI prompt log (public repository)
|
||||
/docs/ai-prompts/
|
||||
@@ -1,3 +0,0 @@
|
||||
[submodule "framework"]
|
||||
path = framework
|
||||
url = ssh://git@git.tirsystem.com:10022/TirSystem/SQA-QC-Framework.git
|
||||
@@ -1,4 +0,0 @@
|
||||
extends: relaxed
|
||||
rules:
|
||||
line-length:
|
||||
max: 120
|
||||
@@ -1,16 +0,0 @@
|
||||
# Doxygen configuration. Run `doxygen Doxyfile` from the repository root;
|
||||
# the HTML pages are written to docs/api/html (ignored by git).
|
||||
PROJECT_NAME = "scrape-movies"
|
||||
PROJECT_BRIEF = "Day 45 of 100 Days of Code: web scraping with BeautifulSoup"
|
||||
OUTPUT_DIRECTORY = docs/api
|
||||
INPUT = src README.md
|
||||
FILE_PATTERNS = *.py *.md
|
||||
RECURSIVE = YES
|
||||
USE_MDFILE_AS_MAINPAGE = README.md
|
||||
OPTIMIZE_OUTPUT_JAVA = YES
|
||||
EXTRACT_ALL = YES
|
||||
AUTOLINK_SUPPORT = YES
|
||||
JAVADOC_AUTOBRIEF = YES
|
||||
GENERATE_HTML = YES
|
||||
GENERATE_LATEX = NO
|
||||
QUIET = YES
|
||||
@@ -1,235 +0,0 @@
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
|
||||
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
"Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
045-scrape-movies
|
||||
Copyright (C) 2026 Tirsvad-Udemy-100-days-of-code
|
||||
|
||||
This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see <http://www.gnu.org/licenses/>.
|
||||
@@ -1,259 +0,0 @@
|
||||
# 🚀 scrape-movies
|
||||
|
||||
Day 45 of Udemy's "100 Days of Code: The Complete Python Pro Bootcamp": web scraping with Requests and BeautifulSoup. It parses a local HTML page, scrapes the Hacker News front page and writes Empire's 100 greatest movies to `movies.txt`. It is for fellow coursists and anyone looking for a small, tested scraping example.
|
||||
|
||||
## 📚 Table of Contents
|
||||
|
||||
- [Overview](#-overview)
|
||||
- [Requirements](#-requirements)
|
||||
- [Setup](#-setup)
|
||||
- [Run](#-run)
|
||||
- [Tests](#-tests)
|
||||
- [License](#-license)
|
||||
- [Links](#-links)
|
||||
- [Appendix: Project documents](#-appendix-project-documents)
|
||||
|
||||
## 🧭 Overview
|
||||
|
||||
| Part | What it is for |
|
||||
| --- | --- |
|
||||
| `src/scrape_movies/` | The code in clean-architecture layers (`domain`, `application`, `infrastructure`, `presentation`) plus `constants.py`; the lecture steps `soup_basics`, `hacker_news` and the movies scraper follow milestone by milestone |
|
||||
| `tests/` | pytest tests that run offline against saved pages in `tests/fixtures/` |
|
||||
| `docs/` | Project documents (business case, plan, milestones, reviews) and the wiki in English, Danish and Bahasa Malaysia |
|
||||
| `Doxyfile` | Generates the API pages from the Doxygen comments in the source |
|
||||
|
||||
The only runtime dependencies are `requests` and `beautifulsoup4`.
|
||||
|
||||
## 📋 Requirements
|
||||
|
||||
- Python 3.13 or newer (the `venv` module comes with it).
|
||||
- `requests` and `beautifulsoup4` (installed by the setup below).
|
||||
- Optional: [Doxygen](https://www.doxygen.nl/) to build the API pages.
|
||||
|
||||
## 🛠️ Setup
|
||||
|
||||
Create a local virtual environment named `.venv`, update `pip`, then install the project with its test tools.
|
||||
|
||||
Windows (PowerShell):
|
||||
|
||||
```powershell
|
||||
python -m venv .venv
|
||||
.venv\Scripts\Activate.ps1
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
Linux and macOS:
|
||||
|
||||
```bash
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
Leave out `[dev]` to install only the runtime dependencies. Personal tokens belong in a git-ignored `.env`; the project never reads it.
|
||||
|
||||
## ▶️ Run
|
||||
|
||||
Every lecture step is a command of the form `python -m scrape_movies <lecture> <step>`; without arguments the help is printed. Each step below shows the command, what it teaches and a screenshot of its output.
|
||||
|
||||
### Lecture: Parsing HTML and making soup
|
||||
|
||||
The sample page is `tests/fixtures/website.html`; the code is `src/scrape_movies/infrastructure/soup_basics.py`.
|
||||
|
||||
**1. Create a soup and read the title** (`BeautifulSoup(html, "html.parser")`, `soup.title.string`)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies soup title
|
||||
```
|
||||
|
||||

|
||||
|
||||
**2. Prettify** (`soup.prettify()` indents one tag per line)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies soup prettify
|
||||
```
|
||||
|
||||

|
||||
|
||||
**3. Find all tags** (`find_all("a")`, then `get_text()` and `get("href")` in a loop)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies soup find-all
|
||||
```
|
||||
|
||||

|
||||
|
||||
**4. Find by id and by class** (`find(id=...)`, `find_all("h3", class_=...)`; `class_` because `class` is a Python keyword)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies soup find
|
||||
```
|
||||
|
||||

|
||||
|
||||
**5. CSS selectors** (`select_one("p a")` and `select(".heading")`)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies soup select
|
||||
```
|
||||
|
||||

|
||||
|
||||
The screenshots are made by `python tools/make_screenshots.py`, which renders the command output as an image; no desktop is captured.
|
||||
|
||||
### Lecture: Scraping Hacker News
|
||||
|
||||
The code is `src/scrape_movies/infrastructure/hacker_news.py` (parsing) and `http_client.py` (one `requests.get` with a timeout and a `User-Agent`). By default the commands read the course's static copy of the front page, `--source live` reads <https://news.ycombinator.com/news> and `--source fixture` reads the saved copy in `tests/fixtures/` without any network.
|
||||
|
||||
**1. Look at the page you will scrape** (live data instead of a local file)
|
||||
|
||||

|
||||
|
||||
**2. Inspect the markup** (browser developer tools show the tags to look for: a story row `tr.athing`, its title link `.titleline > a`, and a `span.score` in the next row)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies hacker-news markup
|
||||
```
|
||||
|
||||

|
||||
|
||||
**3. Titles** (`find_all` on the title links, then `get_text()`)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies hacker-news titles
|
||||
```
|
||||
|
||||

|
||||
|
||||
**4. Links** (`get("href")` of the same anchors)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies hacker-news links
|
||||
```
|
||||
|
||||

|
||||
|
||||
**5. Upvotes** (the text `38 points` is split and converted with `int()` so it can be compared)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies hacker-news upvotes
|
||||
```
|
||||
|
||||

|
||||
|
||||
**6. The top story** (Python's `max` over the upvotes)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies hacker-news top
|
||||
```
|
||||
|
||||

|
||||
|
||||
### Lecture: Is web scraping legal and ethical?
|
||||
|
||||
A short summary in our own words; it is not legal advice. Reading the site's `robots.txt` comes first:
|
||||
|
||||
```bash
|
||||
curl https://news.ycombinator.com/robots.txt
|
||||
```
|
||||
|
||||

|
||||
|
||||
- **Public, non-copyrighted data** is generally fine to read. Court cases such as Genius v. Google and hiQ v. LinkedIn show that the line is not simple.
|
||||
- **Copyrighted content** (a video, a blog post) must not be copied, and **data behind a login** is covered by the site's user agreement, so do not scrape it.
|
||||
- **Commercial use** of scraped data invites expensive disputes even where the scraping itself was allowed.
|
||||
- **Be a good guest:** prefer an official API when there is one, read the site's `robots.txt` to see what it allows, and send at most one request a minute. This project fetches one page per run and identifies itself with a `User-Agent`.
|
||||
|
||||
### Project: 100 movies to watch
|
||||
|
||||
The assignment: scrape Empire's "100 Greatest Movies" and write `movies.txt` with the titles numbered from `1)` upwards. The code is `infrastructure/empire_movies.py` (parsing), `domain/movie.py` (reverse and number) and `infrastructure/movie_file.py` (writing). `--source wayback` (default) reads the Internet Archive snapshot of May 2020 that the assignment pins, `--source live` reads the current Empire page and `--source fixture` reads the saved headings in `tests/fixtures/`.
|
||||
|
||||
**1. Choose the page** (the saved headings of the page)
|
||||
|
||||

|
||||
|
||||
**2. Inspect the markup** (the 2020 page has `<h3 class="title">`; the current page has `<h2><strong>` headings that start with a rank)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies movies markup
|
||||
```
|
||||
|
||||

|
||||
|
||||
**3. Titles in page order** (`select("h3.title")` and a list comprehension over `get_text()`; the page counts down from 100)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies movies titles
|
||||
```
|
||||
|
||||

|
||||
|
||||
**4. Reverse the list** (`titles[::-1]`, so the list counts up from 1)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies movies reversed
|
||||
```
|
||||
|
||||

|
||||
|
||||
**5. Write `movies.txt`** (a `with open(...)` block, one title per line)
|
||||
|
||||
```bash
|
||||
python -m scrape_movies movies write
|
||||
```
|
||||
|
||||

|
||||
|
||||
**6. The result**
|
||||
|
||||

|
||||
|
||||
The committed [`movies.txt`](./movies.txt) was produced with `--source fixture` on 2026-10-28 from the current Empire list, so its ranking differs from the 2020 list in the lecture video. The Internet Archive answered `429 Too Many Requests` while this project was built, so the snapshot is read live only; its `h3.title` layout is covered by a hand-written sample in the tests.
|
||||
|
||||
## 🧪 Tests
|
||||
|
||||
```bash
|
||||
python -m pytest
|
||||
```
|
||||
|
||||
The tests use saved pages and never touch the network. Lint and types: `ruff check .`, `ruff format --check .` and `mypy src tests`. API pages: `doxygen Doxyfile`.
|
||||
|
||||
## 📄 License
|
||||
|
||||
See [LICENSE](./LICENSE).
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
- [Repository][repository]
|
||||
- [Documentation][documentation]: wiki in [English](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/wiki/Home), [Dansk](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/wiki/Home-da) and [Bahasa Melayu](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/wiki/Home-ms); the pages are in `docs/wiki/`
|
||||
- [Issue tracker][issues]
|
||||
|
||||
## 📎 Appendix: Project documents
|
||||
|
||||
| Document | What it tells you |
|
||||
| --- | --- |
|
||||
| [AGENTS.md][agents] | The rules AI agents follow in this repository |
|
||||
| [Business Case][BC-001] | Why the project exists, its scope and success criteria |
|
||||
| [Stakeholder Analysis][SA-001] | Who is involved; the stakeholder IDs used for owners and reviewers |
|
||||
| [Project Plan][PP-001] | The gateways and their schedule |
|
||||
| [Milestones][milestones] | One document per gateway: Go/No-Go criteria and tasks |
|
||||
| [Artifact registry][registry] | Where each artifact type lives, the next version, the PO language and domain |
|
||||
| [Framework README][framework-readme] | The framework this project uses |
|
||||
| [Framework changelog][framework-changelog] | What changed in each framework release |
|
||||
|
||||
---
|
||||
|
||||
[repository]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies
|
||||
[documentation]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/wiki
|
||||
[issues]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/issues
|
||||
[agents]: ./AGENTS.md
|
||||
[BC-001]: ./docs/business-case.md
|
||||
[SA-001]: ./docs/stakeholder-analysis.md
|
||||
[PP-001]: ./docs/project-plan.md
|
||||
[milestones]: ./docs/milestones/
|
||||
[registry]: ./docs/artifact-registry.md
|
||||
[framework-readme]: ./framework/README.md
|
||||
[framework-changelog]: ./framework/CHANGELOG.md
|
||||
@@ -0,0 +1,9 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="92" height="20" role="img" aria-label="code files: 37">
|
||||
<title>code files: 37</title>
|
||||
<rect width="70" height="20" fill="#555555"/>
|
||||
<rect x="70" width="22" height="20" fill="#007ec6"/>
|
||||
<g font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11" text-anchor="middle">
|
||||
<text x="35" y="14" fill="#ffffff">code files</text>
|
||||
<text x="81" y="14" fill="#000000">37</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 434 B |
@@ -0,0 +1,9 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="128" height="20" role="img" aria-label="lines of code: 1,067">
|
||||
<title>lines of code: 1,067</title>
|
||||
<rect width="88" height="20" fill="#555555"/>
|
||||
<rect x="88" width="40" height="20" fill="#007ec6"/>
|
||||
<g font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11" text-anchor="middle">
|
||||
<text x="44" y="14" fill="#ffffff">lines of code</text>
|
||||
<text x="108" y="14" fill="#000000">1,067</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 454 B |
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"blank": 361,
|
||||
"comment": 29,
|
||||
"files": 37,
|
||||
"languages": {
|
||||
"Python": {
|
||||
"blank": 361,
|
||||
"comment": 29,
|
||||
"files": 37,
|
||||
"lines": 1067
|
||||
}
|
||||
},
|
||||
"lines": 1067
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="86" height="20" role="img" aria-label="coverage: 96%">
|
||||
<title>coverage: 96%</title>
|
||||
<rect width="58" height="20" fill="#555555"/>
|
||||
<rect x="58" width="28" height="20" fill="#44cc11"/>
|
||||
<g font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11" text-anchor="middle">
|
||||
<text x="29" y="14" fill="#ffffff">coverage</text>
|
||||
<text x="72" y="14" fill="#000000">96%</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 431 B |
@@ -1,59 +0,0 @@
|
||||
# Artifact Registry
|
||||
|
||||
This project's artifact state. Types, short names and `CrossReference
|
||||
Candidates` come from the framework catalog
|
||||
(`framework/registry/artifact-catalog.md`); this file only records where
|
||||
each document lives in *this* project and the next version to use.
|
||||
|
||||
Delete rows for types you don't use. Add a row the first time you create a
|
||||
document of a type. `Primary File` may contain a glob (e.g.
|
||||
`docs/use-cases/uc-*/uc.md`); `framework/scripts/find-crossreferences.sh` reads it.
|
||||
|
||||
| Short Name | Artifact Type | Primary File | Next Available Version |
|
||||
| --- | --- | --- | --- |
|
||||
| BC | Business Case | docs/business-case.md | 002 |
|
||||
| SA | Stakeholder Analysis | docs/stakeholder-analysis.md | 002 |
|
||||
| PP | Project Plan | docs/project-plan.md | 002 |
|
||||
| MIL | Milestone / Gateway | docs/milestones/*.md | 006 |
|
||||
| RC | SQA Review Record | docs/sqa/reviews/rc-*.md | 016 |
|
||||
|
||||
## Languages
|
||||
|
||||
Set the PO language when the project starts; `project-planning` asks for it
|
||||
if it is missing. An artifact of a type marked "Written in the PO language"
|
||||
exists once, in that language, under its normal name (`business-case.md`); the
|
||||
`artifact` skill ("One file per artifact") has the rule.
|
||||
|
||||
| Setting | Value |
|
||||
| --- | --- |
|
||||
| PO language | en |
|
||||
| PO domain | it |
|
||||
| High-level register | IT Executive English |
|
||||
| Technical register | IT Professional English |
|
||||
| Wiki languages | en, da, ms (`framework/process/wiki.md`) |
|
||||
|
||||
Every artifact of a type marked "Yes" below states its language and domain in
|
||||
its `Language` and `Domain` Metadata rows; `new-artifact.sh` fills them from
|
||||
`PO language` and `PO domain`. `Language` is a BCP 47 code (`da`, `en`).
|
||||
`Domain` is a value from this list; add a row to introduce a domain, so a
|
||||
reviewer can see at once which professional vocabulary a document uses.
|
||||
|
||||
| Domain | Meaning |
|
||||
| --- | --- |
|
||||
| it | Software and IT |
|
||||
| medical | Healthcare and medical devices |
|
||||
| construction | Construction and civil engineering |
|
||||
|
||||
| Artifact types | Register | Written in the PO language |
|
||||
| --- | --- | --- |
|
||||
| BC, KPI, PP, MIL | IT Executive English | Yes |
|
||||
| SA, BMC, BPMN, UCD, US, UC, SSD, DM, RA, GOV, DICT | IT Professional English | Yes |
|
||||
| OC, SD, DCD, ERD, TP, AD, ADR, TM, RC, QC, source code | IT Professional English | No |
|
||||
|
||||
## Notes
|
||||
|
||||
- "Next Available Version" is the zero-padded (3-digit) version to use the
|
||||
*next* time a new document of that type is created. Increment it only when
|
||||
a brand-new document is created, not when an existing document's
|
||||
`## Version History` gets a row.
|
||||
- `ADR` uses 4 digits (`0001`); `RC` is sequential across all artifact types.
|
||||
@@ -1,141 +0,0 @@
|
||||
# Business Case
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | BC-001 |
|
||||
| CrossReference | [SA-001] |
|
||||
| Language | en |
|
||||
| Domain | it |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Accepted | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
|
||||
Day 45 of the Udemy course "100 Days of Code: The Complete Python Pro Bootcamp" teaches web scraping with Requests and BeautifulSoup. This project turns the day's assignment into a small, readable and tested Python repository: parsing a local HTML file, scraping the Hacker News front page, and scraping Empire's "100 Greatest Movies" list into a numbered `movies.txt`. The repository is documented with a README and a three-language wiki (English, Danish, Bahasa Malaysia) that shows a screenshot per lecture step and explains what each step taught.
|
||||
|
||||
## Methodological and Standards Foundation
|
||||
|
||||
The project follows the SQA and QC framework mounted at `framework/` (plan-first gate, one reviewed artifact per step) and Larman's *Applying UML and Patterns* where a model is needed. Product quality is described with ISO/IEC 25010:2023 characteristics. Source code follows the `coding-conventions` (Python) and `clean-architecture` skills, with Doxygen comments in the source.
|
||||
|
||||
## Problem Statement
|
||||
|
||||
- The course assignment ships only a stub (`main.py`) and a sample HTML file, so every participant ends with a throw-away script that is hard to run, test or compare.
|
||||
- A solution that only works against a live site breaks when the site changes; the assignment therefore pins an Internet Archive snapshot.
|
||||
- Course lessons give no written record of what each step taught, which makes later review difficult.
|
||||
|
||||
## Business Opportunity
|
||||
|
||||
A clean reference repository lets the course participant revisit the lessons, lets other coursists compare solutions, and lets GitHub visitors find a small, dependency-light scraping example with tests and multi-language documentation.
|
||||
|
||||
## Objectives
|
||||
|
||||
1. O1: Deliver the three lecture exercises (parse HTML, Hacker News, 100 movies) as importable Python functions that keep the assignment's names.
|
||||
2. O2: Generate `movies.txt` with the movies numbered `1) Title` in ascending order from the Wayback Machine snapshot (alternative: the live Empire URL).
|
||||
3. O3: Provide tests with pytest that run offline against saved fixtures.
|
||||
4. O4: Provide a README with set-up of a local `.venv`, how to run the project and the tests.
|
||||
5. O5: Provide a wiki in English, Danish and Bahasa Malaysia with one screenshot and one "what we gained" explanation per lecture step.
|
||||
6. O6: Set the repository description and topics, and install the shared CI workflows of `TirSystem/github-action`.
|
||||
|
||||
## Scope
|
||||
|
||||
### In Scope
|
||||
|
||||
- Python package under `src/`, tests under `tests/`, documents under `docs/`.
|
||||
- `pyproject.toml`, `constants.py`, Python `.gitignore`, `Doxyfile`, Doxygen comments.
|
||||
- Lecture steps: parsing HTML and making soup, finding and selecting elements, scraping Hacker News, scraping ethics note, 100 movies.
|
||||
- README from the framework template; wiki pages in en, da and ms; screenshots per step.
|
||||
- Repository description and topics on the git host; workflows for quality checks, English comments, code metrics, wiki publishing and milestone publishing.
|
||||
|
||||
### Out of Scope
|
||||
|
||||
- Scraping behind authentication or of copyrighted content beyond the course list titles.
|
||||
- A graphical or web interface.
|
||||
- Any use of the credentials in `.env` inside the project (personal use only, never imported, tested or committed).
|
||||
- Runtime dependencies other than those the assignment needs (`requests`, `beautifulsoup4`).
|
||||
|
||||
## Expected Benefits
|
||||
|
||||
### Tangible Benefits
|
||||
|
||||
- A runnable, tested repository with a generated `movies.txt`.
|
||||
- A wiki in three languages that documents every lecture step with screenshots.
|
||||
- CI evidence (lint, type check, tests) on every pull request.
|
||||
|
||||
### Intangible Benefits
|
||||
|
||||
- Faster revision of the Day 45 material.
|
||||
- A reusable pattern for later "100 Days" projects.
|
||||
|
||||
## Strategic Alignment
|
||||
|
||||
The project supports the participant's goal of completing the bootcamp with documented, reviewable work, and the framework's goal of exercising the SQA and QC process on a real, small project.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
| # | Criterion | Target | Measure |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Tests pass offline | 100 % of pytest tests green | `python -m pytest` in CI and locally |
|
||||
| 2 | `movies.txt` complete | 100 numbered lines, ascending | test on the generated file |
|
||||
| 3 | Wiki completeness | every page exists in en, da and ms | `check-languages` style page-set comparison |
|
||||
| 4 | Screenshot coverage | one screenshot per lecture step in README and each wiki language | review count |
|
||||
| 5 | Runtime dependencies | only `requests` and `beautifulsoup4` | `pyproject.toml` review |
|
||||
| 6 | Repository metadata | description and at least 5 topics set | host API read-back |
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Impact | Mitigation |
|
||||
| --- | --- | --- |
|
||||
| Live sites change markup (Hacker News, Empire) | Scrapers return nothing | Parse functions take HTML text; tests use saved fixtures; Wayback URL pinned |
|
||||
| Wayback Machine unavailable | Cannot fetch the list | Fall back to the live Empire URL named in the request |
|
||||
| Scraping too often loads a server | Unethical use | One request per run, a `User-Agent`, respect `robots.txt`, no loops |
|
||||
| Token in `.env` leaks | Account compromise | `.env` is git-ignored, never imported; tokens are only read by shell commands for the git host API |
|
||||
| Translations drift from English | Inconsistent wiki | English is authoritative; same page set checked per pull request |
|
||||
| Git host repository name differs from the remote (`Tirsvad-Udemy-100_days_of_code` vs `Tirsvad-Udemy-100-days-of-code/045-scrape-movies`) | Metadata set on the wrong repository | Confirm the target before the API call (see Open Issues in the Project Plan) |
|
||||
|
||||
## Assumptions
|
||||
|
||||
- Python 3.13 or newer is installed; `venv` is available.
|
||||
- The git host `git.tirsystem.com` is reachable and the token in `.env` is valid.
|
||||
- S01 is Product Owner, developer and reviewer; in autonomous mode the AI acts as the stand-in reviewer and records it.
|
||||
- The runners of `git.tirsystem.com` are available for the workflows.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Python greater than 3.13 (3.13 or newer), `venv`, `pytest`, `pyproject.toml`, constants in `constants.py`.
|
||||
- Folder structure `src/`, `tests/`, `docs/`.
|
||||
- One branch and one pull request per milestone.
|
||||
- No attribution lines in commit messages or pull request descriptions.
|
||||
- Do not edit `framework/`.
|
||||
|
||||
## Cost–Benefit Assessment
|
||||
|
||||
| Costs | Benefits |
|
||||
| --- | --- |
|
||||
| Author time of S01 for plan acceptance and review of the summary | Reusable, tested reference repository |
|
||||
| Runner minutes for CI | Automatic quality evidence |
|
||||
| Translation effort for da and ms | Documentation readable by more coursists |
|
||||
|
||||
The assessment is qualitative: the project is a learning exercise with no revenue.
|
||||
|
||||
## Stakeholders
|
||||
|
||||
| Stakeholder ID (SA) | Interest in this project |
|
||||
| --- | --- |
|
||||
| S01 | Completes and reviews the assignment; owns the product |
|
||||
| S02 | Reads, runs and compares the code |
|
||||
| S03 | Finds the repository by description and topics and judges it by README |
|
||||
|
||||
## Recommendation
|
||||
|
||||
Proceed — the scope is small, the inputs are known, and the process is exercised end to end.
|
||||
|
||||
---
|
||||
|
||||
[SA-001]: ./stakeholder-analysis.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
|
Before Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 130 KiB |
|
Before Width: | Height: | Size: 67 KiB |
|
Before Width: | Height: | Size: 94 KiB |
|
Before Width: | Height: | Size: 105 KiB |
|
Before Width: | Height: | Size: 20 KiB |
|
Before Width: | Height: | Size: 11 KiB |
|
Before Width: | Height: | Size: 84 KiB |
|
Before Width: | Height: | Size: 9.3 KiB |
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 34 KiB |
|
Before Width: | Height: | Size: 36 KiB |
|
Before Width: | Height: | Size: 39 KiB |
|
Before Width: | Height: | Size: 4.5 KiB |
|
Before Width: | Height: | Size: 50 KiB |
|
Before Width: | Height: | Size: 12 KiB |
|
Before Width: | Height: | Size: 9.2 KiB |
|
Before Width: | Height: | Size: 9.0 KiB |
@@ -1,75 +0,0 @@
|
||||
# Repository Setup and Tooling
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | MIL-001 |
|
||||
| CrossReference | [BC-001] |
|
||||
| Language | en |
|
||||
| Domain | it |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Accepted | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Purpose
|
||||
|
||||
Decide whether the repository is ready to hold code: metadata, project configuration, documentation entry points and CI are in place so the lecture milestones can build on them.
|
||||
|
||||
## Deliverable
|
||||
|
||||
Repository with description and topics, `pyproject.toml`, `.gitignore`, `Doxyfile`, README, `src/` and `tests/` skeleton with `constants.py` and a passing smoke test, and installed workflows.
|
||||
|
||||
## Go / No-Go Criteria
|
||||
|
||||
| # | Criterion (objectively checkable) | Go | No-Go |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | `python -m pytest` passes in a fresh `.venv` created as the README says | Met | Not met |
|
||||
| 2 | `pyproject.toml` requires Python 3.13 or newer and lists only `requests` and `beautifulsoup4` as runtime dependencies | Met | Not met |
|
||||
| 3 | Repository description and at least five topics are read back from the host | Met | Not met |
|
||||
| 4 | README follows the framework template and passes `check-readme.sh` | Met | Not met |
|
||||
| 5 | Workflows from `TirSystem/github-action` are installed and parse | Met | Not met |
|
||||
| 6 | `.env` is git-ignored and nothing in the project reads it | Met | Not met |
|
||||
|
||||
## Dependencies
|
||||
|
||||
| Depends on | Reason |
|
||||
| --- | --- |
|
||||
| None (first milestone) | Builds on it |
|
||||
|
||||
## Traceability
|
||||
|
||||
| Business Case objective / KPI / user story | Reference |
|
||||
| --- | --- |
|
||||
| O3, O4, O6; criteria 1, 5, 6 | [BC-001] |
|
||||
|
||||
## Ownership
|
||||
|
||||
| Role | Stakeholder ID (SA) |
|
||||
| --- | --- |
|
||||
| Owner | S01 |
|
||||
| Approving reviewer | S01 |
|
||||
|
||||
## Target Date
|
||||
|
||||
2026-10-30 — no hard deadline in the Business Case; dates follow the order of the milestones.
|
||||
|
||||
## Tasks
|
||||
|
||||
| # | Task | Summary | Needs its own Use Case/User Story? | Reference |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | Set repository description and topics | Set a description and topics (python, web-scraping, beautifulsoup, requests, 100-days-of-code, udemy, pytest) through the Gitea API using the token from `.env` read by the shell only. Confirm the target repository first because the requested name differs from the origin remote. | No | |
|
||||
| 2 | Add pyproject.toml, .gitignore and constants.py | Create `pyproject.toml` (Python >=3.13, `requests`, `beautifulsoup4`, optional `dev` group with pytest, ruff, mypy), a Python `.gitignore` that also ignores `.env`, and `src/scrape_movies/constants.py` for all constants. | No | |
|
||||
| 3 | Add Doxyfile and Doxygen comment convention | Add a `Doxyfile` that reads `src/` and a short note on the Doxygen comment style used in the source so S02 can generate the API pages. | No | |
|
||||
| 4 | Write README from the framework template | Start `README.md` from `framework/templates/README-template.md` with the `.venv` creation, `python -m pip install --upgrade pip`, install, run and test instructions. Lecture sections and screenshots are added by later milestones. | No | |
|
||||
| 5 | Install workflows from TirSystem/github-action | Install quality checks, English comments, code metrics and milestone publishing as thin callers under `.gitea/workflows/`. Wiki publishing is scoped and needs no file; verify against the adoption rules and record the decision. | No | |
|
||||
| 6 | Add test scaffold and smoke test | Add `tests/` with a pytest configuration in `pyproject.toml` and one smoke test that imports the package, so CI has a green baseline. | No | |
|
||||
| 7 | Install git hooks and plan gate | Run `install-git-hooks.sh --enable-plan-gate` and record the AI prompts of the session in `docs/ai-prompts/session-001/` as the prompt-log process says. | No | |
|
||||
|
||||
---
|
||||
|
||||
[BC-001]: ../business-case.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,73 +0,0 @@
|
||||
# Parsing HTML and Making Soup
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | MIL-002 |
|
||||
| CrossReference | [BC-001] |
|
||||
| Language | en |
|
||||
| Domain | it |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Accepted | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Purpose
|
||||
|
||||
Decide whether the first lecture steps (parsing HTML and making soup, finding and selecting elements) are reproduced, tested and documented.
|
||||
|
||||
## Deliverable
|
||||
|
||||
Module `soup_basics` with functions that parse `website.html` (title, prettify, find_all, find, select_one, select), the sample page as a fixture, tests, and screenshots of each step.
|
||||
|
||||
## Go / No-Go Criteria
|
||||
|
||||
| # | Criterion (objectively checkable) | Go | No-Go |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Every lecture operation (title, string, prettify, find_all, getText, get, find by id and class, select_one, select) has a function and a test | Met | Not met |
|
||||
| 2 | Tests run offline against `tests/fixtures/website.html` | Met | Not met |
|
||||
| 3 | All public functions carry Doxygen comments | Met | Not met |
|
||||
| 4 | One screenshot per lecture step exists under `docs/images/` | Met | Not met |
|
||||
| 5 | README has a section for this step with its screenshots | Met | Not met |
|
||||
|
||||
## Dependencies
|
||||
|
||||
| Depends on | Reason |
|
||||
| --- | --- |
|
||||
| MIL-001 (project configuration and test scaffold) | Builds on it |
|
||||
|
||||
## Traceability
|
||||
|
||||
| Business Case objective / KPI / user story | Reference |
|
||||
| --- | --- |
|
||||
| O1, O3, O5; criteria 1, 4 | [BC-001] |
|
||||
|
||||
## Ownership
|
||||
|
||||
| Role | Stakeholder ID (SA) |
|
||||
| --- | --- |
|
||||
| Owner | S01 |
|
||||
| Approving reviewer | S01 |
|
||||
|
||||
## Target Date
|
||||
|
||||
2026-11-03 — no hard deadline in the Business Case; dates follow the order of the milestones.
|
||||
|
||||
## Tasks
|
||||
|
||||
| # | Task | Summary | Needs its own Use Case/User Story? | Reference |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | Add website.html fixture | Copy the sample page from the course archive to `tests/fixtures/website.html` and expose its path in `constants.py`. | No | |
|
||||
| 2 | Implement soup creation and tag access | Functions `make_soup`, `get_title`, `prettify_html` using `html.parser`, matching the lecture on creating a soup object, `soup.title` and `prettify`. | No | |
|
||||
| 3 | Implement find and select helpers | Functions for `find_all` over anchors with text and `href`, `find` by id and by class (using `class_`), and CSS selectors with `select_one` and `select`. | No | |
|
||||
| 4 | Write tests for the parsing steps | pytest tests for each function against the fixture, covering text, attributes and nested selectors. | No | |
|
||||
| 5 | Capture screenshots of the parsing steps | Run each step and save a terminal-output screenshot per lecture step in `docs/images/` for README and wiki. GUI-free: images are rendered from the command output, no desktop capture. | No | |
|
||||
| 6 | Document the parsing steps in README | Add a README section per step with code, expected output and its screenshot. | No | |
|
||||
|
||||
---
|
||||
|
||||
[BC-001]: ../business-case.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,75 +0,0 @@
|
||||
# Scraping Hacker News
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | MIL-003 |
|
||||
| CrossReference | [BC-001] |
|
||||
| Language | en |
|
||||
| Domain | it |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Accepted | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Purpose
|
||||
|
||||
Decide whether scraping a live site is reproduced safely: Hacker News titles, links and upvotes are extracted, the top story found, and the ethics of scraping documented.
|
||||
|
||||
## Deliverable
|
||||
|
||||
Module `hacker_news` with fetch and parse functions, the saved front page as a fixture, tests, ethics note and screenshots of each step.
|
||||
|
||||
## Go / No-Go Criteria
|
||||
|
||||
| # | Criterion (objectively checkable) | Go | No-Go |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Parsing returns titles, links and integer upvotes from the fixture | Met | Not met |
|
||||
| 2 | The article with the most upvotes is found with `max` | Met | Not met |
|
||||
| 3 | Fetching sends one request with a `User-Agent` and a timeout; no loops of requests | Met | Not met |
|
||||
| 4 | Tests run offline | Met | Not met |
|
||||
| 5 | Ethics and legality notes from the lecture are in README and wiki source text | Met | Not met |
|
||||
| 6 | One screenshot per lecture step exists under `docs/images/` | Met | Not met |
|
||||
|
||||
## Dependencies
|
||||
|
||||
| Depends on | Reason |
|
||||
| --- | --- |
|
||||
| MIL-002 (soup helpers) | Builds on it |
|
||||
|
||||
## Traceability
|
||||
|
||||
| Business Case objective / KPI / user story | Reference |
|
||||
| --- | --- |
|
||||
| O1, O3, O5; criteria 1, 4 | [BC-001] |
|
||||
|
||||
## Ownership
|
||||
|
||||
| Role | Stakeholder ID (SA) |
|
||||
| --- | --- |
|
||||
| Owner | S01 |
|
||||
| Approving reviewer | S01 |
|
||||
|
||||
## Target Date
|
||||
|
||||
2026-11-06 — no hard deadline in the Business Case; dates follow the order of the milestones.
|
||||
|
||||
## Tasks
|
||||
|
||||
| # | Task | Summary | Needs its own Use Case/User Story? | Reference |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | Add fetch function for Hacker News | `fetch_html(url)` using `requests` with a timeout, `User-Agent` and status check; URLs live in `constants.py`. The course mirror and the live site are both named. | No | |
|
||||
| 2 | Add Hacker News fixture | Save the course mirror page `appbrewery.github.io/news.ycombinator.com` as `tests/fixtures/hacker_news.html` so tests never touch the network. | No | |
|
||||
| 3 | Extract titles, links and upvotes | Functions returning lists of titles, links and integer scores from the title links and score spans, as in the lecture. | No | |
|
||||
| 4 | Find the top story by upvotes | `top_story()` using `max` over the scraped rows and a test for ties and an empty page. | No | |
|
||||
| 5 | Write tests for Hacker News parsing | pytest tests against the fixture, with `requests` patched for fetch tests. | No | |
|
||||
| 6 | Document ethics and legality of scraping | Summarise the lecture: robots.txt, copyrighted and authenticated data, commercial risk, request frequency. Written in the README in our own words. | No | |
|
||||
| 7 | Capture screenshots of the Hacker News steps | Save a screenshot per lecture step (page, inspected markup, extracted output, top story) in `docs/images/`. The live pages are opened in the built-in browser pane, not the host desktop. | No | |
|
||||
|
||||
---
|
||||
|
||||
[BC-001]: ../business-case.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,75 +0,0 @@
|
||||
# Top 100 Movies Scraper
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | MIL-004 |
|
||||
| CrossReference | [BC-001] |
|
||||
| Language | en |
|
||||
| Domain | it |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Accepted | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Purpose
|
||||
|
||||
Decide whether the main assignment is met: the Empire list is scraped, reversed to ascending order and written to `movies.txt` as numbered lines.
|
||||
|
||||
## Deliverable
|
||||
|
||||
Module `movies` with `fetch`, `parse_titles`, `format_titles`, `write_movies`, a command-line entry `python -m scrape_movies`, a fixture of the Wayback page, tests, generated `movies.txt` and screenshots.
|
||||
|
||||
## Go / No-Go Criteria
|
||||
|
||||
| # | Criterion (objectively checkable) | Go | No-Go |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | `movies.txt` has exactly 100 lines in the form `1) Title`, ascending | Met | Not met |
|
||||
| 2 | The Wayback Machine snapshot is the default URL and the live Empire URL is the documented alternative | Met | Not met |
|
||||
| 3 | Titles come from `h3` elements with class `title` | Met | Not met |
|
||||
| 4 | Tests run offline against the fixture | Met | Not met |
|
||||
| 5 | Running the module twice produces identical output | Met | Not met |
|
||||
| 6 | One screenshot per lecture step exists under `docs/images/` | Met | Not met |
|
||||
|
||||
## Dependencies
|
||||
|
||||
| Depends on | Reason |
|
||||
| --- | --- |
|
||||
| MIL-003 (fetch helper) | Builds on it |
|
||||
|
||||
## Traceability
|
||||
|
||||
| Business Case objective / KPI / user story | Reference |
|
||||
| --- | --- |
|
||||
| O1, O2, O3, O5; criteria 1, 2, 4 | [BC-001] |
|
||||
|
||||
## Ownership
|
||||
|
||||
| Role | Stakeholder ID (SA) |
|
||||
| --- | --- |
|
||||
| Owner | S01 |
|
||||
| Approving reviewer | S01 |
|
||||
|
||||
## Target Date
|
||||
|
||||
2026-11-10 — no hard deadline in the Business Case; dates follow the order of the milestones.
|
||||
|
||||
## Tasks
|
||||
|
||||
| # | Task | Summary | Needs its own Use Case/User Story? | Reference |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | Fetch the Empire page | Use the shared `fetch_html` with the Wayback URL from the assignment as default and the live Empire URL as a selectable alternative, both in `constants.py`. | No | |
|
||||
| 2 | Parse movie titles | `parse_titles(html)` selecting `h3.title` and returning the names in page order. | No | |
|
||||
| 3 | Reverse and number the titles | `format_titles(titles)` reverses the descending page order and prefixes `1) ` to `100) ` as the assignment shows. | No | |
|
||||
| 4 | Write movies.txt | `write_movies(lines, path)` writing UTF-8 with a trailing newline; a `__main__` entry runs fetch, parse, format, write. | No | |
|
||||
| 5 | Add Wayback fixture and tests | Save the snapshot as `tests/fixtures/empire_best_movies.html` and test parse, format and write offline, including that 100 titles are found. | No | |
|
||||
| 6 | Generate and commit movies.txt | Run the program once against the pinned snapshot and keep the produced `movies.txt` as the example result. | No | |
|
||||
| 7 | Capture screenshots of the movie steps | Save screenshots of the source page, the inspected `h3.title`, the printed list and `movies.txt` in `docs/images/`. | No | |
|
||||
|
||||
---
|
||||
|
||||
[BC-001]: ../business-case.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,73 +0,0 @@
|
||||
# Wiki in Three Languages
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | MIL-005 |
|
||||
| CrossReference | [BC-001] |
|
||||
| Language | en |
|
||||
| Domain | it |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Accepted | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Purpose
|
||||
|
||||
Decide whether the wiki is complete: every page exists in English, Danish and Bahasa Malaysia with a screenshot and a "what we gained" explanation for each lecture step.
|
||||
|
||||
## Deliverable
|
||||
|
||||
Pages under `docs/wiki/` in `en`, `da` and `ms`, a sidebar listing the three languages, README screenshot sections, and the wiki publication setup.
|
||||
|
||||
## Go / No-Go Criteria
|
||||
|
||||
| # | Criterion (objectively checkable) | Go | No-Go |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Every English page has a `-da` and a `-ms` page (same page set) | Met | Not met |
|
||||
| 2 | Each lecture step page has a screenshot and a "What we gained" section in all three languages | Met | Not met |
|
||||
| 3 | Each page starts with the language switch line and `_Sidebar.md` lists all languages | Met | Not met |
|
||||
| 4 | README shows a screenshot for every lecture step | Met | Not met |
|
||||
| 5 | Code blocks, commands, file names and IDs stay in English | Met | Not met |
|
||||
|
||||
## Dependencies
|
||||
|
||||
| Depends on | Reason |
|
||||
| --- | --- |
|
||||
| MIL-004 (all screenshots exist) | Builds on it |
|
||||
|
||||
## Traceability
|
||||
|
||||
| Business Case objective / KPI / user story | Reference |
|
||||
| --- | --- |
|
||||
| O4, O5; criteria 3, 4 | [BC-001] |
|
||||
|
||||
## Ownership
|
||||
|
||||
| Role | Stakeholder ID (SA) |
|
||||
| --- | --- |
|
||||
| Owner | S01 |
|
||||
| Approving reviewer | S01 |
|
||||
|
||||
## Target Date
|
||||
|
||||
2026-11-14 — no hard deadline in the Business Case; dates follow the order of the milestones.
|
||||
|
||||
## Tasks
|
||||
|
||||
| # | Task | Summary | Needs its own Use Case/User Story? | Reference |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | Write English wiki pages | Home, Setup, one page per lecture step (parse HTML, find and select, Hacker News, ethics, 100 movies) and Gains, each with its screenshot and a "What we gained" section. English is authoritative. | No | |
|
||||
| 2 | Translate the wiki to Danish | Page for page `-da` versions of every English page, following `framework/process/wiki.md`. | No | |
|
||||
| 3 | Translate the wiki to Bahasa Malaysia | Page for page `-ms` versions of every English page. The wiki process names BCP 47 codes, so `ms` is used. | No | |
|
||||
| 4 | Add wiki sidebar and footer | `_Sidebar.md` and `_Footer.md` listing English, Dansk and Bahasa Melayu pages. | No | |
|
||||
| 5 | Complete README screenshots and wiki links | Make sure every lecture step in README has its screenshot and a link to its wiki page in all three languages. | No | |
|
||||
| 6 | Check wiki page set and publication | Add a test that compares the page sets of the three languages, and confirm the wiki publishes through the scoped workflow after merge. | No | |
|
||||
|
||||
---
|
||||
|
||||
[BC-001]: ../business-case.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,98 +0,0 @@
|
||||
# Project Plan
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | PP-001 |
|
||||
| CrossReference | [BC-001], [SA-001], [MIL-001], [MIL-002], [MIL-003], [MIL-004], [MIL-005] |
|
||||
| Language | en |
|
||||
| Domain | it |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Accepted | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Purpose
|
||||
|
||||
Schedule the five gateways that deliver the Day 45 scraping repository. The Business Case sets no deadline, so the plan only fixes the order and a working window per gateway.
|
||||
|
||||
## Planning Assumptions
|
||||
|
||||
- Week 1 starts 2026-10-28; the plan ends by 2026-11-14 (no hard deadline in [BC-001]).
|
||||
- Phase length: two to four days; each phase is one branch and one pull request.
|
||||
- Communication follows [SA-001]: plan acceptance in chat, one pull request per milestone, run summary at the end.
|
||||
- Autonomous mode: S01 accepts this plan once; the AI then runs every gateway and acts as stand-in reviewer, recorded in each RC record.
|
||||
|
||||
## Gateway Schedule
|
||||
|
||||
| Gateway | Document | Window | Decision date | Owner | Stories | Main deliverable | Milestone |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Repository Setup | [MIL-001] | 2026-10-28 to 2026-10-30 | 2026-10-30 | S01 | none (technical tasks) | Repository metadata, configuration, README, CI | [Milestone 259](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/milestone/259) (closed) |
|
||||
| Parsing HTML | [MIL-002] | 2026-10-31 to 2026-11-03 | 2026-11-03 | S01 | none (technical tasks) | `soup_basics` module, tests, screenshots | [Milestone 260](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/milestone/260) (closed) |
|
||||
| Hacker News | [MIL-003] | 2026-11-04 to 2026-11-06 | 2026-11-06 | S01 | none (technical tasks) | `hacker_news` module, ethics note, screenshots | [Milestone 261](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/milestone/261) (closed) |
|
||||
| Top 100 Movies | [MIL-004] | 2026-11-07 to 2026-11-10 | 2026-11-10 | S01 | none (technical tasks) | `movies` module and `movies.txt`, screenshots | [Milestone 262](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/milestone/262) (closed) |
|
||||
| Wiki | [MIL-005] | 2026-11-11 to 2026-11-14 | 2026-11-14 | S01 | none (technical tasks) | Wiki in en, da, ms | [Milestone 263](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/milestone/263) (closed) |
|
||||
|
||||
```plantuml
|
||||
@startgantt
|
||||
Project starts 2026-10-28
|
||||
[Repository Setup] starts 2026-10-28 and ends 2026-10-30
|
||||
[Parsing HTML] starts 2026-10-31 and ends 2026-11-03
|
||||
[Hacker News] starts 2026-11-04 and ends 2026-11-06
|
||||
[Top 100 Movies] starts 2026-11-07 and ends 2026-11-10
|
||||
[Wiki] starts 2026-11-11 and ends 2026-11-14
|
||||
[MIL-001 Go/No-Go] happens 2026-10-30
|
||||
[MIL-002 Go/No-Go] happens 2026-11-03
|
||||
[MIL-003 Go/No-Go] happens 2026-11-06
|
||||
[MIL-004 Go/No-Go] happens 2026-11-10
|
||||
[MIL-005 Go/No-Go] happens 2026-11-14
|
||||
@endgantt
|
||||
```
|
||||
|
||||
## Scope Coverage
|
||||
|
||||
| Business Case scope item | Gateway |
|
||||
| --- | --- |
|
||||
| `pyproject.toml`, `.gitignore`, `Doxyfile`, `constants.py`, README skeleton | [MIL-001] |
|
||||
| Repository description and topics, workflows | [MIL-001] |
|
||||
| Parsing HTML, finding and selecting elements | [MIL-002] |
|
||||
| Hacker News scraping and the ethics note | [MIL-003] |
|
||||
| 100 movies and `movies.txt` | [MIL-004] |
|
||||
| Wiki in en, da, ms and screenshots per step | [MIL-005] (screenshots are taken in MIL-002 to MIL-004) |
|
||||
|
||||
## Dependencies
|
||||
|
||||
```
|
||||
MIL-001 → MIL-002 → MIL-003 → MIL-004 → MIL-005
|
||||
```
|
||||
|
||||
A No-Go on a gateway is fixed and re-reviewed; after three failed rounds the run stops and reports (AGENTS.md). Later dates move by the same amount; S01 decides a move of the end date.
|
||||
|
||||
## Plan Risks
|
||||
|
||||
| Risk | Impact | Mitigation |
|
||||
| --- | --- | --- |
|
||||
| Target repository name differs from the one named in the request | Metadata set on the wrong repository | Resolved: S01 chose the origin remote at plan acceptance |
|
||||
| Reusable workflow path differs (`TirSystem/git-workflows` in the template, `TirSystem/github-action` named in the request) | Workflow calls fail | Verify the path on the host in MIL-001 before installing |
|
||||
| Danish and Malay translation quality | Review finds errors | Keep sentences short; English is authoritative |
|
||||
| Network access to Wayback during the run | Fixture cannot be saved | Retry once; alternative live Empire URL |
|
||||
|
||||
## Open Issues
|
||||
|
||||
- Target repository: S01 chose the origin remote (`Tirsvad-Udemy-100-days-of-code/045-scrape-movies`) when accepting the plan; the named `Tirsvad-Udemy-100_days_of_code` is not used.
|
||||
- Wiki publication is a scoped workflow in `TirSystem/github-action` (no file needed) while the framework template `publish-wiki.yml` also exists; MIL-001 records which one applies.
|
||||
- "Screenshot per lecture step" is planned as rendered terminal output and browser-pane captures, never host desktop capture.
|
||||
|
||||
---
|
||||
|
||||
[BC-001]: ./business-case.md
|
||||
[SA-001]: ./stakeholder-analysis.md
|
||||
[MIL-001]: ./milestones/mil-001-repository-setup.md
|
||||
[MIL-002]: ./milestones/mil-002-parse-html.md
|
||||
[MIL-003]: ./milestones/mil-003-hacker-news.md
|
||||
[MIL-004]: ./milestones/mil-004-top-100-movies.md
|
||||
[MIL-005]: ./milestones/mil-005-wiki.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,68 +0,0 @@
|
||||
# Review of BC-001
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-001 |
|
||||
| CrossReference | [BC-001], [QC-BC-001], [QC-LANG-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [BC-001]
|
||||
- Checklist used: [QC-BC-001]
|
||||
- Scope: full review
|
||||
- Language and domain: en / it
|
||||
- Language reviewer: none (stand-in S01 reads the language and knows the domain)
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | ROI/Cost-Benefit analysis is quantitative, or where qualitative, is explicitly justified | Pass | Cost-Benefit table is qualitative and states why (learning project, no revenue). |
|
||||
| 2 | Risks are identified with documented impact and mitigation | Pass | six risks, each with impact and mitigation. |
|
||||
| 3 | Success criteria are measurable, stating explicit targets rather than vague aspirations | Pass | six success criteria with numeric or checkable targets and a measure. |
|
||||
| 4 | Scope explicitly separates In Scope vs Out of Scope | Pass | In Scope and Out of Scope subsections exist. |
|
||||
| 5 | Stakeholders are cross-referenced to Stakeholder Analysis IDs rather than re-described inline | Pass | Stakeholders table cites S01, S02, S03 only. |
|
||||
| 6 | Methodology and quality-standard foundation are stated explicitly (e.g. ISO/IEC 25010, Larman) | Pass | foundation names the framework, Larman, ISO/IEC 25010:2023. |
|
||||
| 7 | Assumptions and constraints are explicit and clearly distinguished from one another | Pass | Assumptions and Constraints are separate lists. |
|
||||
| 8 | Document supports executive decision-making with a clear, unambiguous recommendation | Pass | Recommendation is "Proceed" with one sentence. |
|
||||
|
||||
## Language and Domain Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The Metadata table has a `Language` row and a `Domain` row, and neither is a placeholder | Pass | Language `en` and Domain `it` in Metadata. |
|
||||
| 2 | `Language` is a BCP 47 code and `Domain` is a value from the registry's domain list | Pass | `en` is BCP 47; `it` is in the registry domain list. |
|
||||
| 3 | The content (prose and table cells) is written in the stated language | Pass | Whole content is English. |
|
||||
| 4 | The register matches the one the registry gives for the artifact type | Pass | IT Executive English for BC and PP, IT Professional English for SA and MIL, per the registry; MIL/BC/PP keep short, decision-oriented sentences. |
|
||||
| 5 | Domain terms are the PO terms of the domain's dictionary, with no synonyms | Pass | No `DICT` exists; terms are standard IT terms (scraping, fixture, workflow). No synonyms found for them. |
|
||||
| 6 | Metadata keys, section headings, IDs and statuses are in English | Pass | Keys, headings, IDs and statuses are English. |
|
||||
| 7 | No translated twin (`<name>.<language>.md`) exists beside the document | Pass | No `*.en.md` or other twin exists in `docs/`. |
|
||||
| 8 | A change of language or domain since the previous accepted version has a Version History row and was reviewed again | Pass | First version in this language; no change since an accepted version. |
|
||||
| 9 | A reviewer competent in the domain, and in the language, has confirmed that the domain terms are used correctly | Pass | Stand-in for S01 (Product Owner, developer) confirms the IT terms; not independent (autonomous mode). |
|
||||
| 10 | Abbreviations are spelled out on first use, in the stated language | Pass | Abbreviations (PO, SA, MIL, CI, BC) are defined in the framework and used consistently; PO is spelled out in the SA. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. The review is a stand-in review by the author's tool for S01 in autonomous mode and is not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| None | S01 | 2026-10-28 |
|
||||
|
||||
---
|
||||
|
||||
[BC-001]: ../../business-case.md
|
||||
[QC-BC-001]: ../../../framework/qc/artifacts/qc-business-case.md
|
||||
[QC-LANG-001]: ../../../framework/qc/artifacts/qc-language-domain.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,69 +0,0 @@
|
||||
# Review of SA-001
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-002 |
|
||||
| CrossReference | [SA-001], [QC-SA-001], [QC-LANG-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [SA-001]
|
||||
- Checklist used: [QC-SA-001]
|
||||
- Scope: full review
|
||||
- Language and domain: en / it
|
||||
- Language reviewer: none (stand-in S01 reads the language and knows the domain)
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Power/Interest grid is filled for every stakeholder, with no gaps or unclassified entries | Pass | S01 to S03 each have power, interest and quadrant. |
|
||||
| 2 | Each stakeholder is assigned a unique, stable ID (e.g. S01-S11 style) reusable for RACI assignments in other artifacts | Pass | IDs S01 to S03, stable, used by BC, PP and MIL documents. |
|
||||
| 3 | Roles and organizational context are defined with explicit Power and Interest levels, not just narrative description | Pass | role, organization, power and interest given as levels. |
|
||||
| 4 | Communication needs (channel, frequency, deliverable type) are mapped to project phases or milestones | Pass | four communication rows tied to milestones. |
|
||||
| 5 | Conflicting stakeholder interests are identified with documented mitigation or resolution strategies | Pass | three conflicts, each with a mitigation. |
|
||||
| 6 | Stakeholder concerns are explicitly traced to Business Case objectives | Pass | traceability table maps each concern to BC-001 objectives O1 to O6. |
|
||||
| 7 | Primary concerns are expressed in both business language and a recognized quality-attribute mapping (e.g. FURPS+) | Pass | CN-001 to CN-009 each mapped to a FURPS+ attribute. |
|
||||
| 8 | Document is understandable and navigable by non-technical stakeholders reviewing their own entry | Pass | short tables, plain wording. |
|
||||
| 9 | Every concern has a stable ID (`CN-NNN`) in the concern register, with the stakeholder who raised it; IDs are never renumbered or reused | Pass | CN-001 to CN-009 stable and name the stakeholder who raised them. |
|
||||
|
||||
## Language and Domain Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The Metadata table has a `Language` row and a `Domain` row, and neither is a placeholder | Pass | Language `en` and Domain `it` in Metadata. |
|
||||
| 2 | `Language` is a BCP 47 code and `Domain` is a value from the registry's domain list | Pass | `en` is BCP 47; `it` is in the registry domain list. |
|
||||
| 3 | The content (prose and table cells) is written in the stated language | Pass | Whole content is English. |
|
||||
| 4 | The register matches the one the registry gives for the artifact type | Pass | IT Executive English for BC and PP, IT Professional English for SA and MIL, per the registry; MIL/BC/PP keep short, decision-oriented sentences. |
|
||||
| 5 | Domain terms are the PO terms of the domain's dictionary, with no synonyms | Pass | No `DICT` exists; terms are standard IT terms (scraping, fixture, workflow). No synonyms found for them. |
|
||||
| 6 | Metadata keys, section headings, IDs and statuses are in English | Pass | Keys, headings, IDs and statuses are English. |
|
||||
| 7 | No translated twin (`<name>.<language>.md`) exists beside the document | Pass | No `*.en.md` or other twin exists in `docs/`. |
|
||||
| 8 | A change of language or domain since the previous accepted version has a Version History row and was reviewed again | Pass | First version in this language; no change since an accepted version. |
|
||||
| 9 | A reviewer competent in the domain, and in the language, has confirmed that the domain terms are used correctly | Pass | Stand-in for S01 (Product Owner, developer) confirms the IT terms; not independent (autonomous mode). |
|
||||
| 10 | Abbreviations are spelled out on first use, in the stated language | Pass | Abbreviations (PO, SA, MIL, CI, BC) are defined in the framework and used consistently; PO is spelled out in the SA. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. The review is a stand-in review by the author's tool for S01 in autonomous mode and is not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| None | S01 | 2026-10-28 |
|
||||
|
||||
---
|
||||
|
||||
[SA-001]: ../../stakeholder-analysis.md
|
||||
[QC-SA-001]: ../../../framework/qc/artifacts/qc-stakeholder-analysis.md
|
||||
[QC-LANG-001]: ../../../framework/qc/artifacts/qc-language-domain.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,75 +0,0 @@
|
||||
# Review of PP-001
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-003 |
|
||||
| CrossReference | [PP-001], [QC-PP-001], [QC-LANG-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [PP-001]
|
||||
- Checklist used: [QC-PP-001]
|
||||
- Scope: full review
|
||||
- Language and domain: en / it
|
||||
- Language reviewer: none (stand-in S01 reads the language and knows the domain)
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The plan states its start date, end date and the constraint it must fit | Pass | start 2026-10-28, end 2026-11-14; the Business Case constraint is no hard deadline plus the technical constraints, quoted in Assumptions. |
|
||||
| 2 | Every milestone or gateway is listed once with its document, decision date and owner | Pass | five gateways, each once, with document, decision date and owner S01. |
|
||||
| 3 | Every decision date lies inside the plan's window and is consistent with the dates in the linked `MIL-*` document | Pass | decision dates equal the Target Date of each MIL document and lie inside the window. |
|
||||
| 4 | Every in-scope item of the Business Case is delivered at a named gateway (scope coverage) | Pass | Scope Coverage maps every in-scope item to a gateway. |
|
||||
| 5 | Dependencies between gateways are stated, acyclic and consistent with the schedule order | Pass | chain MIL-001 to MIL-005 is acyclic and matches the order. |
|
||||
| 6 | The plan states what happens to later gateways and to the end date when a gateway is No-Go | Pass | Dependencies says later dates move and S01 decides the end date. |
|
||||
| 7 | Owners, reviewers and escalation roles use Stakeholder Analysis IDs and match the governance RACI | N-A | owners are S01; no `GOV` document exists yet, so there is no RACI to cross-check. |
|
||||
| 8 | Planning assumptions are listed with their source, and a changed baseline is recorded with its reason | Pass | assumptions listed with source; no re-baseline yet. |
|
||||
| 9 | Plan risks are listed with impact and a mitigation, and trace to Business Case risks where they come from there | Pass | four plan risks with mitigation; the Wayback risk traces to the BC live-site risk. |
|
||||
| 10 | Open issues are listed, and none contradicts a statement elsewhere in the plan | Pass | open issues listed; the repository question was answered by S01 in chat and is recorded in the milestone tasks. |
|
||||
| 11 | Each gateway's main deliverable is stated and matches the deliverable of its `MIL-*` document | Pass | deliverables match the MIL Deliverable sections in substance. |
|
||||
| 12 | Gateways that deliver user stories name them, and those that do not say why | Pass | Stories column says "none (technical tasks)". |
|
||||
| 13 | The schedule view (table, Gantt) and the dependency view agree with each other | Pass | table, Gantt and dependency chain agree. |
|
||||
| 14 | Each gateway is linked to its issue-tracker milestone, or marked as not yet synced | Pass | Milestone column says "not yet synced". |
|
||||
| 15 | The plan is a singleton: one Project Plan per project, changes recorded in Version History | Pass | single `project-plan.md`; Version History kept. |
|
||||
|
||||
## Language and Domain Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The Metadata table has a `Language` row and a `Domain` row, and neither is a placeholder | Pass | Language `en` and Domain `it` in Metadata. |
|
||||
| 2 | `Language` is a BCP 47 code and `Domain` is a value from the registry's domain list | Pass | `en` is BCP 47; `it` is in the registry domain list. |
|
||||
| 3 | The content (prose and table cells) is written in the stated language | Pass | Whole content is English. |
|
||||
| 4 | The register matches the one the registry gives for the artifact type | Pass | IT Executive English for BC and PP, IT Professional English for SA and MIL, per the registry; MIL/BC/PP keep short, decision-oriented sentences. |
|
||||
| 5 | Domain terms are the PO terms of the domain's dictionary, with no synonyms | Pass | No `DICT` exists; terms are standard IT terms (scraping, fixture, workflow). No synonyms found for them. |
|
||||
| 6 | Metadata keys, section headings, IDs and statuses are in English | Pass | Keys, headings, IDs and statuses are English. |
|
||||
| 7 | No translated twin (`<name>.<language>.md`) exists beside the document | Pass | No `*.en.md` or other twin exists in `docs/`. |
|
||||
| 8 | A change of language or domain since the previous accepted version has a Version History row and was reviewed again | Pass | First version in this language; no change since an accepted version. |
|
||||
| 9 | A reviewer competent in the domain, and in the language, has confirmed that the domain terms are used correctly | Pass | Stand-in for S01 (Product Owner, developer) confirms the IT terms; not independent (autonomous mode). |
|
||||
| 10 | Abbreviations are spelled out on first use, in the stated language | Pass | Abbreviations (PO, SA, MIL, CI, BC) are defined in the framework and used consistently; PO is spelled out in the SA. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. The review is a stand-in review by the author's tool for S01 in autonomous mode and is not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| None | S01 | 2026-10-28 |
|
||||
|
||||
---
|
||||
|
||||
[PP-001]: ../../project-plan.md
|
||||
[QC-PP-001]: ../../../framework/qc/artifacts/qc-project-plan.md
|
||||
[QC-LANG-001]: ../../../framework/qc/artifacts/qc-language-domain.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,66 +0,0 @@
|
||||
# Review of MIL-001
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-004 |
|
||||
| CrossReference | [MIL-001], [QC-MIL-001], [QC-LANG-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-001]
|
||||
- Checklist used: [QC-MIL-001]
|
||||
- Scope: full review
|
||||
- Language and domain: en / it
|
||||
- Language reviewer: none (stand-in S01 reads the language and knows the domain)
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | A concrete deliverable is defined for every gate | Pass | Deliverable section names the concrete output. |
|
||||
| 2 | Explicit Go/No-Go criteria are stated for each gate | Pass | criteria are checkable by a command, a file or a read-back. |
|
||||
| 3 | Dependencies on other milestones are explicitly mapped | Pass | Dependencies table names the predecessor (or none for MIL-001). |
|
||||
| 4 | Each milestone is traceable to a Business Case objective or KPI | Pass | Traceability maps to BC-001 objectives and success criteria. |
|
||||
| 5 | Milestone owner and approving reviewer are identified | Pass | Owner and Approving reviewer are S01; the approver is the same person, so the review is a stand-in and recorded as not independent. |
|
||||
| 6 | Milestone has a defined target date consistent with project constraints | Pass | Target Date is inside the plan window and BC-001 has no earlier deadline. |
|
||||
|
||||
## Language and Domain Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The Metadata table has a `Language` row and a `Domain` row, and neither is a placeholder | Pass | Language `en` and Domain `it` in Metadata. |
|
||||
| 2 | `Language` is a BCP 47 code and `Domain` is a value from the registry's domain list | Pass | `en` is BCP 47; `it` is in the registry domain list. |
|
||||
| 3 | The content (prose and table cells) is written in the stated language | Pass | Whole content is English. |
|
||||
| 4 | The register matches the one the registry gives for the artifact type | Pass | IT Executive English for BC and PP, IT Professional English for SA and MIL, per the registry; MIL/BC/PP keep short, decision-oriented sentences. |
|
||||
| 5 | Domain terms are the PO terms of the domain's dictionary, with no synonyms | Pass | No `DICT` exists; terms are standard IT terms (scraping, fixture, workflow). No synonyms found for them. |
|
||||
| 6 | Metadata keys, section headings, IDs and statuses are in English | Pass | Keys, headings, IDs and statuses are English. |
|
||||
| 7 | No translated twin (`<name>.<language>.md`) exists beside the document | Pass | No `*.en.md` or other twin exists in `docs/`. |
|
||||
| 8 | A change of language or domain since the previous accepted version has a Version History row and was reviewed again | Pass | First version in this language; no change since an accepted version. |
|
||||
| 9 | A reviewer competent in the domain, and in the language, has confirmed that the domain terms are used correctly | Pass | Stand-in for S01 (Product Owner, developer) confirms the IT terms; not independent (autonomous mode). |
|
||||
| 10 | Abbreviations are spelled out on first use, in the stated language | Pass | Abbreviations (PO, SA, MIL, CI, BC) are defined in the framework and used consistently; PO is spelled out in the SA. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. The review is a stand-in review by the author's tool for S01 in autonomous mode and is not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| None | S01 | 2026-10-28 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-001]: ../../milestones/mil-001-repository-setup.md
|
||||
[QC-MIL-001]: ../../../framework/qc/artifacts/qc-milestones-gateways.md
|
||||
[QC-LANG-001]: ../../../framework/qc/artifacts/qc-language-domain.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,66 +0,0 @@
|
||||
# Review of MIL-002
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-005 |
|
||||
| CrossReference | [MIL-002], [QC-MIL-001], [QC-LANG-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-002]
|
||||
- Checklist used: [QC-MIL-001]
|
||||
- Scope: full review
|
||||
- Language and domain: en / it
|
||||
- Language reviewer: none (stand-in S01 reads the language and knows the domain)
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | A concrete deliverable is defined for every gate | Pass | Deliverable section names the concrete output. |
|
||||
| 2 | Explicit Go/No-Go criteria are stated for each gate | Pass | criteria are checkable by a command, a file or a read-back. |
|
||||
| 3 | Dependencies on other milestones are explicitly mapped | Pass | Dependencies table names the predecessor (or none for MIL-001). |
|
||||
| 4 | Each milestone is traceable to a Business Case objective or KPI | Pass | Traceability maps to BC-001 objectives and success criteria. |
|
||||
| 5 | Milestone owner and approving reviewer are identified | Pass | Owner and Approving reviewer are S01; the approver is the same person, so the review is a stand-in and recorded as not independent. |
|
||||
| 6 | Milestone has a defined target date consistent with project constraints | Pass | Target Date is inside the plan window and BC-001 has no earlier deadline. |
|
||||
|
||||
## Language and Domain Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The Metadata table has a `Language` row and a `Domain` row, and neither is a placeholder | Pass | Language `en` and Domain `it` in Metadata. |
|
||||
| 2 | `Language` is a BCP 47 code and `Domain` is a value from the registry's domain list | Pass | `en` is BCP 47; `it` is in the registry domain list. |
|
||||
| 3 | The content (prose and table cells) is written in the stated language | Pass | Whole content is English. |
|
||||
| 4 | The register matches the one the registry gives for the artifact type | Pass | IT Executive English for BC and PP, IT Professional English for SA and MIL, per the registry; MIL/BC/PP keep short, decision-oriented sentences. |
|
||||
| 5 | Domain terms are the PO terms of the domain's dictionary, with no synonyms | Pass | No `DICT` exists; terms are standard IT terms (scraping, fixture, workflow). No synonyms found for them. |
|
||||
| 6 | Metadata keys, section headings, IDs and statuses are in English | Pass | Keys, headings, IDs and statuses are English. |
|
||||
| 7 | No translated twin (`<name>.<language>.md`) exists beside the document | Pass | No `*.en.md` or other twin exists in `docs/`. |
|
||||
| 8 | A change of language or domain since the previous accepted version has a Version History row and was reviewed again | Pass | First version in this language; no change since an accepted version. |
|
||||
| 9 | A reviewer competent in the domain, and in the language, has confirmed that the domain terms are used correctly | Pass | Stand-in for S01 (Product Owner, developer) confirms the IT terms; not independent (autonomous mode). |
|
||||
| 10 | Abbreviations are spelled out on first use, in the stated language | Pass | Abbreviations (PO, SA, MIL, CI, BC) are defined in the framework and used consistently; PO is spelled out in the SA. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. The review is a stand-in review by the author's tool for S01 in autonomous mode and is not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| None | S01 | 2026-10-28 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-002]: ../../milestones/mil-002-parse-html.md
|
||||
[QC-MIL-001]: ../../../framework/qc/artifacts/qc-milestones-gateways.md
|
||||
[QC-LANG-001]: ../../../framework/qc/artifacts/qc-language-domain.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,66 +0,0 @@
|
||||
# Review of MIL-003
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-006 |
|
||||
| CrossReference | [MIL-003], [QC-MIL-001], [QC-LANG-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-003]
|
||||
- Checklist used: [QC-MIL-001]
|
||||
- Scope: full review
|
||||
- Language and domain: en / it
|
||||
- Language reviewer: none (stand-in S01 reads the language and knows the domain)
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | A concrete deliverable is defined for every gate | Pass | Deliverable section names the concrete output. |
|
||||
| 2 | Explicit Go/No-Go criteria are stated for each gate | Pass | criteria are checkable by a command, a file or a read-back. |
|
||||
| 3 | Dependencies on other milestones are explicitly mapped | Pass | Dependencies table names the predecessor (or none for MIL-001). |
|
||||
| 4 | Each milestone is traceable to a Business Case objective or KPI | Pass | Traceability maps to BC-001 objectives and success criteria. |
|
||||
| 5 | Milestone owner and approving reviewer are identified | Pass | Owner and Approving reviewer are S01; the approver is the same person, so the review is a stand-in and recorded as not independent. |
|
||||
| 6 | Milestone has a defined target date consistent with project constraints | Pass | Target Date is inside the plan window and BC-001 has no earlier deadline. |
|
||||
|
||||
## Language and Domain Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The Metadata table has a `Language` row and a `Domain` row, and neither is a placeholder | Pass | Language `en` and Domain `it` in Metadata. |
|
||||
| 2 | `Language` is a BCP 47 code and `Domain` is a value from the registry's domain list | Pass | `en` is BCP 47; `it` is in the registry domain list. |
|
||||
| 3 | The content (prose and table cells) is written in the stated language | Pass | Whole content is English. |
|
||||
| 4 | The register matches the one the registry gives for the artifact type | Pass | IT Executive English for BC and PP, IT Professional English for SA and MIL, per the registry; MIL/BC/PP keep short, decision-oriented sentences. |
|
||||
| 5 | Domain terms are the PO terms of the domain's dictionary, with no synonyms | Pass | No `DICT` exists; terms are standard IT terms (scraping, fixture, workflow). No synonyms found for them. |
|
||||
| 6 | Metadata keys, section headings, IDs and statuses are in English | Pass | Keys, headings, IDs and statuses are English. |
|
||||
| 7 | No translated twin (`<name>.<language>.md`) exists beside the document | Pass | No `*.en.md` or other twin exists in `docs/`. |
|
||||
| 8 | A change of language or domain since the previous accepted version has a Version History row and was reviewed again | Pass | First version in this language; no change since an accepted version. |
|
||||
| 9 | A reviewer competent in the domain, and in the language, has confirmed that the domain terms are used correctly | Pass | Stand-in for S01 (Product Owner, developer) confirms the IT terms; not independent (autonomous mode). |
|
||||
| 10 | Abbreviations are spelled out on first use, in the stated language | Pass | Abbreviations (PO, SA, MIL, CI, BC) are defined in the framework and used consistently; PO is spelled out in the SA. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. The review is a stand-in review by the author's tool for S01 in autonomous mode and is not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| None | S01 | 2026-10-28 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-003]: ../../milestones/mil-003-hacker-news.md
|
||||
[QC-MIL-001]: ../../../framework/qc/artifacts/qc-milestones-gateways.md
|
||||
[QC-LANG-001]: ../../../framework/qc/artifacts/qc-language-domain.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,66 +0,0 @@
|
||||
# Review of MIL-004
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-007 |
|
||||
| CrossReference | [MIL-004], [QC-MIL-001], [QC-LANG-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-004]
|
||||
- Checklist used: [QC-MIL-001]
|
||||
- Scope: full review
|
||||
- Language and domain: en / it
|
||||
- Language reviewer: none (stand-in S01 reads the language and knows the domain)
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | A concrete deliverable is defined for every gate | Pass | Deliverable section names the concrete output. |
|
||||
| 2 | Explicit Go/No-Go criteria are stated for each gate | Pass | criteria are checkable by a command, a file or a read-back. |
|
||||
| 3 | Dependencies on other milestones are explicitly mapped | Pass | Dependencies table names the predecessor (or none for MIL-001). |
|
||||
| 4 | Each milestone is traceable to a Business Case objective or KPI | Pass | Traceability maps to BC-001 objectives and success criteria. |
|
||||
| 5 | Milestone owner and approving reviewer are identified | Pass | Owner and Approving reviewer are S01; the approver is the same person, so the review is a stand-in and recorded as not independent. |
|
||||
| 6 | Milestone has a defined target date consistent with project constraints | Pass | Target Date is inside the plan window and BC-001 has no earlier deadline. |
|
||||
|
||||
## Language and Domain Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The Metadata table has a `Language` row and a `Domain` row, and neither is a placeholder | Pass | Language `en` and Domain `it` in Metadata. |
|
||||
| 2 | `Language` is a BCP 47 code and `Domain` is a value from the registry's domain list | Pass | `en` is BCP 47; `it` is in the registry domain list. |
|
||||
| 3 | The content (prose and table cells) is written in the stated language | Pass | Whole content is English. |
|
||||
| 4 | The register matches the one the registry gives for the artifact type | Pass | IT Executive English for BC and PP, IT Professional English for SA and MIL, per the registry; MIL/BC/PP keep short, decision-oriented sentences. |
|
||||
| 5 | Domain terms are the PO terms of the domain's dictionary, with no synonyms | Pass | No `DICT` exists; terms are standard IT terms (scraping, fixture, workflow). No synonyms found for them. |
|
||||
| 6 | Metadata keys, section headings, IDs and statuses are in English | Pass | Keys, headings, IDs and statuses are English. |
|
||||
| 7 | No translated twin (`<name>.<language>.md`) exists beside the document | Pass | No `*.en.md` or other twin exists in `docs/`. |
|
||||
| 8 | A change of language or domain since the previous accepted version has a Version History row and was reviewed again | Pass | First version in this language; no change since an accepted version. |
|
||||
| 9 | A reviewer competent in the domain, and in the language, has confirmed that the domain terms are used correctly | Pass | Stand-in for S01 (Product Owner, developer) confirms the IT terms; not independent (autonomous mode). |
|
||||
| 10 | Abbreviations are spelled out on first use, in the stated language | Pass | Abbreviations (PO, SA, MIL, CI, BC) are defined in the framework and used consistently; PO is spelled out in the SA. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. The review is a stand-in review by the author's tool for S01 in autonomous mode and is not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| None | S01 | 2026-10-28 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-004]: ../../milestones/mil-004-top-100-movies.md
|
||||
[QC-MIL-001]: ../../../framework/qc/artifacts/qc-milestones-gateways.md
|
||||
[QC-LANG-001]: ../../../framework/qc/artifacts/qc-language-domain.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,66 +0,0 @@
|
||||
# Review of MIL-005
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-008 |
|
||||
| CrossReference | [MIL-005], [QC-MIL-001], [QC-LANG-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-005]
|
||||
- Checklist used: [QC-MIL-001]
|
||||
- Scope: full review
|
||||
- Language and domain: en / it
|
||||
- Language reviewer: none (stand-in S01 reads the language and knows the domain)
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | A concrete deliverable is defined for every gate | Pass | Deliverable section names the concrete output. |
|
||||
| 2 | Explicit Go/No-Go criteria are stated for each gate | Pass | criteria are checkable by a command, a file or a read-back. |
|
||||
| 3 | Dependencies on other milestones are explicitly mapped | Pass | Dependencies table names the predecessor (or none for MIL-001). |
|
||||
| 4 | Each milestone is traceable to a Business Case objective or KPI | Pass | Traceability maps to BC-001 objectives and success criteria. |
|
||||
| 5 | Milestone owner and approving reviewer are identified | Pass | Owner and Approving reviewer are S01; the approver is the same person, so the review is a stand-in and recorded as not independent. |
|
||||
| 6 | Milestone has a defined target date consistent with project constraints | Pass | Target Date is inside the plan window and BC-001 has no earlier deadline. |
|
||||
|
||||
## Language and Domain Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The Metadata table has a `Language` row and a `Domain` row, and neither is a placeholder | Pass | Language `en` and Domain `it` in Metadata. |
|
||||
| 2 | `Language` is a BCP 47 code and `Domain` is a value from the registry's domain list | Pass | `en` is BCP 47; `it` is in the registry domain list. |
|
||||
| 3 | The content (prose and table cells) is written in the stated language | Pass | Whole content is English. |
|
||||
| 4 | The register matches the one the registry gives for the artifact type | Pass | IT Executive English for BC and PP, IT Professional English for SA and MIL, per the registry; MIL/BC/PP keep short, decision-oriented sentences. |
|
||||
| 5 | Domain terms are the PO terms of the domain's dictionary, with no synonyms | Pass | No `DICT` exists; terms are standard IT terms (scraping, fixture, workflow). No synonyms found for them. |
|
||||
| 6 | Metadata keys, section headings, IDs and statuses are in English | Pass | Keys, headings, IDs and statuses are English. |
|
||||
| 7 | No translated twin (`<name>.<language>.md`) exists beside the document | Pass | No `*.en.md` or other twin exists in `docs/`. |
|
||||
| 8 | A change of language or domain since the previous accepted version has a Version History row and was reviewed again | Pass | First version in this language; no change since an accepted version. |
|
||||
| 9 | A reviewer competent in the domain, and in the language, has confirmed that the domain terms are used correctly | Pass | Stand-in for S01 (Product Owner, developer) confirms the IT terms; not independent (autonomous mode). |
|
||||
| 10 | Abbreviations are spelled out on first use, in the stated language | Pass | Abbreviations (PO, SA, MIL, CI, BC) are defined in the framework and used consistently; PO is spelled out in the SA. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. The review is a stand-in review by the author's tool for S01 in autonomous mode and is not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| None | S01 | 2026-10-28 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-005]: ../../milestones/mil-005-wiki.md
|
||||
[QC-MIL-001]: ../../../framework/qc/artifacts/qc-milestones-gateways.md
|
||||
[QC-LANG-001]: ../../../framework/qc/artifacts/qc-language-domain.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,62 +0,0 @@
|
||||
# Review of MIL-001 Python code
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-009 |
|
||||
| CrossReference | [MIL-001], [QC-PY-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [44a3b7b] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-001]
|
||||
- Checklist used: [QC-PY-001]
|
||||
- Scope: full review. Code reviewed: pyproject.toml, src/, tests/ of the MIL-001 branch.
|
||||
- Language and domain: n/a (technical type)
|
||||
- Language reviewer: none
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Packages, modules, functions, variables, classes and constants follow PEP 8 casing (`snake_case`, `PascalCase`, `UPPER_SNAKE`) | Pass | `constants.py` uses UPPER_SNAKE; packages are snake_case. |
|
||||
| 2 | Names state purpose in the domain's language; no unexplained abbreviations, no single-letter names outside tiny scopes | Pass | names state purpose (`REQUEST_TIMEOUT_SECONDS`, `EMPIRE_WAYBACK_URL`). |
|
||||
| 3 | Code is produced by the project's formatter and passes its linter with no unexplained suppressions | Pass | `ruff check` and `ruff format --check` clean; no `noqa`; only D203/D213 are configured off because they conflict with D211/D212. |
|
||||
| 4 | Every function and method signature is type-annotated, including `-> None` | Pass | the one test function is annotated; `mypy --strict` has no error. |
|
||||
| 5 | No bare `except:`, no swallowed exceptions; specific exceptions are raised and the cause is kept (`raise ... from`) | N-A | no exception handling exists yet. |
|
||||
| 6 | No mutable default arguments and no shadowed builtins | Pass | none present. |
|
||||
| 7 | Files, locks and connections are managed with context managers | N-A | no file or connection is opened yet. |
|
||||
| 8 | Public modules, classes and functions have docstrings that say what, not how | Pass | every module has a Doxygen docstring. |
|
||||
| 9 | Logging uses `logging`, not `print`; no secrets or personal data in log output | N-A | no logging exists yet. |
|
||||
| 10 | Classes and operations trace to the Design Class Diagram they implement; deviations are recorded | N-A | no Design Class Diagram; every task of MIL-001 is a plain technical task (Needs Use Case: No). |
|
||||
| 11 | Tests exist for new behaviour, are named for the behaviour, and do not depend on order or the network | Pass | `tests/unit/test_constants.py` is offline and independent of order. |
|
||||
| 12 | Type checker runs in strict mode without errors; `Any` is justified in a comment | Pass | `mypy --strict` over `src` and `tests`. |
|
||||
| 13 | Dependencies are declared and pinned in the project's dependency file, none unused | Fail | dependencies are declared with lower bounds, not pinned; this is a library-style project without a lock file. |
|
||||
| 14 | Each piece of knowledge (a business rule, constant, format, validation or query) is defined in one place; code is merged only where it expresses the same knowledge, not where it merely looks alike | Pass | every constant is defined once in `constants.py`. |
|
||||
| 15 | Imports point inward: business-rule modules import no framework, database, UI or delivery-mechanism code, and there are no import cycles | Pass | the layer packages hold no imports yet; no cycle is possible. |
|
||||
| 16 | SOLID holds: a class or module has one reason to change; new behaviour is added by extension, not by editing a type test in several places; subclasses honour the contract of their base; protocols are narrow; high-level code depends on protocols or abstract types and receives its implementations | Pass | no class or branching on type exists yet. |
|
||||
| 17 | Source is laid out in the layers and folders of the clean architecture folder structure (`domain`, `application`, `infrastructure`, `presentation` packages under `src/`, and `unit`, `integration` and `e2e` under `tests/`), in `snake_case`, and imports between the layers point inward | Pass | `domain`, `application`, `infrastructure` and `presentation` under `src/scrape_movies/`; `unit`, `integration`, `e2e` under `tests/`. |
|
||||
| 18 | File and folder names are English, contain no space and no character outside letters, digits, `-`, `_` and `.`, and follow the casing of the language for files | Pass | all names are English lowercase with `_` and no space. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| Decide whether to add a lock file for exact versions (optional criterion 13) | S01 | 2026-11-14 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-001]: ../../milestones/mil-001-repository-setup.md
|
||||
[QC-PY-001]: ../../../framework/qc/code/qc-programming-python.md
|
||||
[44a3b7b]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/44a3b7be335b657bdd91a533e7582743f9448b1b
|
||||
@@ -1,69 +0,0 @@
|
||||
# Review of MIL-001 workflows
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-010 |
|
||||
| CrossReference | [MIL-001], [QC-WF-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [44a3b7b] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-001]
|
||||
- Checklist used: [QC-WF-001]
|
||||
- Scope: full review. Code reviewed: .gitea/workflows/*.yml.
|
||||
- Language and domain: n/a (technical type)
|
||||
- Language reviewer: none
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The file meets the YAML checklist ([QC-YML-001]) | Pass | see RC-011 for QC-YML-001. |
|
||||
| 2 | The file is in `.gitea/workflows/` or `.github/workflows/` with the extension `.yml` or `.yaml`, and the same workflow does not exist in both directories | Pass | all five files are `.yml` under `.gitea/workflows/`; none under `.github/`. |
|
||||
| 3 | The file declares `name`, `on` and `jobs`; `on` lists explicit events and, where it matters, `branches` or `paths` | Pass | each declares `name`, `on` and `jobs`, with branches or paths where they matter. |
|
||||
| 4 | A header comment states the purpose, the triggers, the inputs, the secrets it needs, what it changes outside the repository, and the decision or milestone it implements | Pass | every file opens with a header naming purpose, triggers, secrets, outside effects and MIL-001. |
|
||||
| 5 | Every job and step has a descriptive `name`, unique within its job | Pass | every job and step has a unique `name`. |
|
||||
| 6 | The file passes `actionlint` (or the project's workflow validation test) and the tool version is stated | Pass | actionlint 1.7.12 reports no finding; `gitea.*` contexts were replaced by `github.*` to satisfy it. |
|
||||
| 7 | Every job declares `permissions` explicitly with the least access that works (`contents: read` or none by default); write access only on the job that needs it | Pass | every job sets `permissions`; write only on the metrics job. |
|
||||
| 8 | Every action used is pinned to a full commit SHA or an exact release tag, never to a branch; a moving major tag (`@v1`) is accepted only for an action published by the same organisation under a stated release policy | Pass | actions are at exact tags (`checkout@v4.2.2`, `setup-python@v5.3.0`, `upload-artifact@v4.6.2`); the reusable workflows of the same organisation use the major tag `v2` as the organisation publishes it. |
|
||||
| 9 | No secret is written in the file; a secret is read as `secrets.NAME`, passed to a step through `env:`, never placed on a command line or echoed | Pass | the only secret is read as `secrets.TOKEN_FOR_GITEA_SYNC` through `env:`. |
|
||||
| 10 | Untrusted text (`inputs.*`, event fields such as a title, branch name or label, step outputs derived from them) is never interpolated into a `run:` script; it is passed through `env:` and used as a quoted variable | Pass | `inputs.dry_run` reaches the script only through `env:`. |
|
||||
| 11 | A trigger that gives a run secrets or a write token on code from a fork or an untrusted branch (`pull_request_target`, `workflow_run`) does not check out or execute that code | Pass | no `pull_request_target` or `workflow_run`. |
|
||||
| 12 | A workflow that changes state outside the repository (release, tag, publish, sync, deploy) is started by hand or by a protected branch, and a manual run defaults to a dry run with an explicit input to apply | Pass | `sync-milestones` runs on main or by hand and defaults to a dry run on a manual start. |
|
||||
| 13 | The token the workflow uses has only the scope the job needs, and where the host's built-in token is enough it is used instead of a personal token | N-A | the token is created by S01; its scope cannot be checked here. |
|
||||
| 14 | Every job sets `runs-on` to an explicit label and `timeout-minutes` | Pass | every job with `runs-on` sets `ubuntu-latest` and `timeout-minutes`; called workflows set their own. |
|
||||
| 15 | `concurrency` is set where two runs would conflict: a check cancels the older run (`cancel-in-progress: true`); a publishing job queues (`cancel-in-progress: false`) | Pass | checks cancel older runs; the publishing job queues. |
|
||||
| 16 | Triggers are filtered (`branches`, `paths`, `types`) so the workflow runs only for the changes it concerns | Pass | triggers use `branches` or `paths`. |
|
||||
| 17 | Tools and dependencies are installed at pinned versions; a version is a workflow input with a default or a value in one place | Pass | Python 3.13 and the tool versions come from one place, `pyproject.toml`; actionlint and yamllint were run by hand at the versions stated here. |
|
||||
| 18 | A dependency cache is used where the install is slow enough to matter | N-A | the install is small enough that a cache does not matter. |
|
||||
| 19 | A failing step names the tool, the file and the reason in its log (annotation or `::error::` line) | N-A | the called tools write their own annotations. |
|
||||
| 20 | A `run:` block is within the limit on embedded program text of criterion 7 of [QC-YML-001]; a longer script is a file (`tools/`, or a shared repository fetched at a pinned ref) that the step calls with one command, and the workflow passes its settings through `env:` | Pass | no `run:` block exceeds five lines. |
|
||||
| 21 | Logic used by more than one workflow or repository is a reusable workflow (`workflow_call`) or a composite action; each input has a `description`, a `type` and a `default` where one is sensible | Pass | quality, comments and metrics reuse the shared workflows with `uses:`. |
|
||||
| 22 | Every workflow feature used is verified on the instance that runs it, and the host and version are recorded; a feature the target hosts differ on is noted in the header | Pass | on pull request 35 on git.tirsystem.com the tests, quality and English-comments runs completed with success. |
|
||||
| 23 | A reusable workflow states, in its header, how a caller pins it and the script it fetches, so the workflow and its script are the same release | Pass | the headers say the callers use the major tag `v2` of the same organisation. |
|
||||
| 24 | A static test of the repository's workflows exists and runs on every pull request: valid YAML, required keys, explicit permissions and runner, no inline script above the limit | Fail | no static workflow test runs in this repository; actionlint was run by hand. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| Add a static workflow test (optional criterion 24) | S01 | 2026-11-14 |
|
||||
| Create the secret TOKEN_FOR_GITEA_SYNC (S01 only; the AI does not create secrets) | S01 | 2026-11-14 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-001]: ../../milestones/mil-001-repository-setup.md
|
||||
[QC-WF-001]: ../../../framework/qc/code/qc-workflow.md
|
||||
[44a3b7b]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/44a3b7be335b657bdd91a533e7582743f9448b1b
|
||||
@@ -1,59 +0,0 @@
|
||||
# Review of MIL-001 YAML files
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-011 |
|
||||
| CrossReference | [MIL-001], [QC-YML-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [44a3b7b] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-001]
|
||||
- Checklist used: [QC-YML-001]
|
||||
- Scope: full review. Code reviewed: .gitea/workflows/*.yml, .yamllint.yml.
|
||||
- Language and domain: n/a (technical type)
|
||||
- Language reviewer: none
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | The file parses with a safe YAML loader and has no duplicate keys in any mapping | Pass | yamllint 1.38.0 (PyYAML based) parses all files and reports no duplicate key. |
|
||||
| 2 | Indentation is 2 spaces everywhere, never tabs; lines end with LF; the file ends with one newline and has no trailing spaces | Pass | 2 spaces, LF, one final newline, no trailing spaces (yamllint relaxed profile). |
|
||||
| 3 | Passes `yamllint` (or the project's YAML linter) with its configuration committed next to the code and no unexplained `disable` comments | Pass | `.yamllint.yml` is committed; no disable comments. |
|
||||
| 4 | A scalar that YAML would coerce to another type is quoted: `yes`, `no`, `on`, `off`, `null`, `~`, numbers with a leading zero, versions such as `1.10`, and values that contain `: ` or ` #` | Pass | strings such as `"3.13"` are quoted; `on` is the Actions trigger key, not a value. |
|
||||
| 5 | Only safe, plain types are used: no language-specific tags (`!!python/object`, `!ruby/object`) and no `!include`-style extension unless the consuming tool documents it | Pass | only plain types. |
|
||||
| 6 | No secret is written in the file: no password, token, key or private connection string; a secret is referenced by name and supplied by the consuming tool | Pass | no secret in any file. |
|
||||
| 7 | Embedded program text (shell, Python, SQL, `jq`, and so on) in a value is at most five lines; a longer one is moved to a file of its own, in its own language, and the YAML calls it | Pass | the longest `run:` block has two lines. |
|
||||
| 8 | A block of keys or values repeated in the file or across files is defined once (an anchor, a template, an include or a reusable unit the consumer supports) | Pass | repeated blocks are not present; shared logic is reused through `uses:`. |
|
||||
| 9 | Anchors, aliases and merge keys (`&`, `*`, `<<`) are used only where the consuming tool is known to support them | Pass | no anchors or aliases. |
|
||||
| 10 | The file validates against the schema of the consuming tool where one exists, and the schema or tool version is stated | Pass | actionlint 1.7.12 validates the workflow schema. |
|
||||
| 11 | A header comment states what the file is for, which tool reads it, and where its decision or design is recorded | Pass | each file has a header comment saying purpose, reader and where it is decided (MIL-001). |
|
||||
| 12 | Key names follow the convention of the consuming tool; one case style is used within the file (kebab-case or snake_case, never mixed) | Pass | key names are those of the consuming tool (`cancel-in-progress`, `timeout-minutes`, `dry_run`), as the tool defines them. |
|
||||
| 13 | Multi-line strings use a block scalar (`\|` or `>`) and state the chomping indicator (`\|-`, `\|+`) where the trailing newline matters | Pass | no multi-line strings other than `run: |` blocks where the trailing newline does not matter. |
|
||||
| 14 | The file name is lowercase kebab-case with one extension style (`.yml` or `.yaml`) used throughout the repository | Pass | lowercase kebab-case `.yml` names. |
|
||||
| 15 | A change is exercised by a check that runs without the real target: a parse and schema test, or a dry run of the consuming tool | Pass | pull request 35 ran the workflows on the host. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| None | S01 | 2026-11-14 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-001]: ../../milestones/mil-001-repository-setup.md
|
||||
[QC-YML-001]: ../../../framework/qc/code/qc-yaml.md
|
||||
[44a3b7b]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/44a3b7be335b657bdd91a533e7582743f9448b1b
|
||||
@@ -1,62 +0,0 @@
|
||||
# Review of MIL-002 Python code
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-012 |
|
||||
| CrossReference | [MIL-002], [QC-PY-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [fda38c9] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-002]
|
||||
- Checklist used: [QC-PY-001]
|
||||
- Scope: full review. Code reviewed: soup_basics, soup_steps, cli, Link, tools/, tests/unit.
|
||||
- Language and domain: n/a (technical type)
|
||||
- Language reviewer: none
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Packages, modules, functions, variables, classes and constants follow PEP 8 casing (`snake_case`, `PascalCase`, `UPPER_SNAKE`) | Pass | modules and functions are snake_case, classes PascalCase, constants UPPER_SNAKE. |
|
||||
| 2 | Names state purpose in the domain's language; no unexplained abbreviations, no single-letter names outside tiny scopes | Pass | names state purpose; the lecture names (`find_all`, `select_one`) are kept in the helper names. |
|
||||
| 3 | Code is produced by the project's formatter and passes its linter with no unexplained suppressions | Pass | `ruff check` and `ruff format --check` are clean with no `noqa`. |
|
||||
| 4 | Every function and method signature is type-annotated, including `-> None` | Pass | every signature is annotated; `mypy --strict` passes, also in a venv without the third-party packages. |
|
||||
| 5 | No bare `except:`, no swallowed exceptions; specific exceptions are raised and the cause is kept (`raise ... from`) | N-A | no exception is raised or caught. |
|
||||
| 6 | No mutable default arguments and no shadowed builtins | Pass | no mutable defaults, no shadowed builtins. |
|
||||
| 7 | Files, locks and connections are managed with context managers | N-A | no file or connection is opened except through `Path.read_text`. |
|
||||
| 8 | Public modules, classes and functions have docstrings that say what, not how | Pass | every public module, class and function has a Doxygen docstring saying what it does. |
|
||||
| 9 | Logging uses `logging`, not `print`; no secrets or personal data in log output | N-A | no logging; the command-line output is the product, written with `print` in the presentation layer. |
|
||||
| 10 | Classes and operations trace to the Design Class Diagram they implement; deviations are recorded | N-A | no Design Class Diagram; all tasks are plain technical tasks (Needs Use Case: No). |
|
||||
| 11 | Tests exist for new behaviour, are named for the behaviour, and do not depend on order or the network | Pass | tests are named for behaviour, offline and independent of order. |
|
||||
| 12 | Type checker runs in strict mode without errors; `Any` is justified in a comment | Pass | `mypy --strict` runs over `src`, `tests` and `tools`. |
|
||||
| 13 | Dependencies are declared and pinned in the project's dependency file, none unused | Fail | dependencies are declared with lower bounds, not pinned (see RC-009 action item); none is unused. |
|
||||
| 14 | Each piece of knowledge (a business rule, constant, format, validation or query) is defined in one place; code is merged only where it expresses the same knowledge, not where it merely looks alike | Pass | each constant and format is defined once; the step texts are built in one place. |
|
||||
| 15 | Imports point inward: business-rule modules import no framework, database, UI or delivery-mechanism code, and there are no import cycles | Pass | `domain` imports nothing from other layers; `infrastructure` imports `domain` and `constants`; `presentation` composes them. |
|
||||
| 16 | SOLID holds: a class or module has one reason to change; new behaviour is added by extension, not by editing a type test in several places; subclasses honour the contract of their base; protocols are narrow; high-level code depends on protocols or abstract types and receives its implementations | Pass | no class hierarchy; the dataclass `Link` has one reason to change. |
|
||||
| 17 | Source is laid out in the layers and folders of the clean architecture folder structure (`domain`, `application`, `infrastructure`, `presentation` packages under `src/`, and `unit`, `integration` and `e2e` under `tests/`), in `snake_case`, and imports between the layers point inward | Pass | code is in `domain`, `infrastructure` and `presentation`; tests are in `tests/unit`. |
|
||||
| 18 | File and folder names are English, contain no space and no character outside letters, digits, `-`, `_` and `.`, and follow the casing of the language for files | Pass | all names are English lowercase with `_`, no space. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| Pin dependencies or accept lower bounds (open from RC-009, optional criterion 13) | S01 | 2026-11-14 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-002]: ../../milestones/mil-002-parse-html.md
|
||||
[QC-PY-001]: ../../../framework/qc/code/qc-programming-python.md
|
||||
[fda38c9]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/fda38c906e3d470482ae1cc4a8c032991f7efe26
|
||||
@@ -1,62 +0,0 @@
|
||||
# Review of MIL-003 Python code
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-013 |
|
||||
| CrossReference | [MIL-003], [QC-PY-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [81af6fa] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-003]
|
||||
- Checklist used: [QC-PY-001]
|
||||
- Scope: full review. Code reviewed: Story, errors, ports, use_cases, http_client, hacker_news, hacker_news_steps, capture_page, tests/unit.
|
||||
- Language and domain: n/a (technical type)
|
||||
- Language reviewer: none
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Packages, modules, functions, variables, classes and constants follow PEP 8 casing (`snake_case`, `PascalCase`, `UPPER_SNAKE`) | Pass | modules and functions are snake_case, classes PascalCase, constants UPPER_SNAKE. |
|
||||
| 2 | Names state purpose in the domain's language; no unexplained abbreviations, no single-letter names outside tiny scopes | Pass | names state purpose; the lecture names (`find_all`, `select_one`) are kept in the helper names. |
|
||||
| 3 | Code is produced by the project's formatter and passes its linter with no unexplained suppressions | Pass | `ruff check` and `ruff format --check` are clean with no `noqa`. |
|
||||
| 4 | Every function and method signature is type-annotated, including `-> None` | Pass | every signature is annotated; `mypy --strict` passes, also in a venv without the third-party packages. |
|
||||
| 5 | No bare `except:`, no swallowed exceptions; specific exceptions are raised and the cause is kept (`raise ... from`) | Pass | `fetch_html` catches only `requests.RequestException` and raises `FetchError` `from` the cause; a test checks `__cause__`. |
|
||||
| 6 | No mutable default arguments and no shadowed builtins | Pass | no mutable defaults, no shadowed builtins. |
|
||||
| 7 | Files, locks and connections are managed with context managers | Pass | files are read with `Path.read_text`; the HTTP call is a single `requests.get` with no open session. |
|
||||
| 8 | Public modules, classes and functions have docstrings that say what, not how | Pass | every public module, class and function has a Doxygen docstring saying what it does. |
|
||||
| 9 | Logging uses `logging`, not `print`; no secrets or personal data in log output | N-A | no logging; the command-line output is the product, written with `print` in the presentation layer. |
|
||||
| 10 | Classes and operations trace to the Design Class Diagram they implement; deviations are recorded | N-A | no Design Class Diagram; all tasks are plain technical tasks (Needs Use Case: No). |
|
||||
| 11 | Tests exist for new behaviour, are named for the behaviour, and do not depend on order or the network | Pass | tests are named for behaviour, offline and independent of order. |
|
||||
| 12 | Type checker runs in strict mode without errors; `Any` is justified in a comment | Pass | `mypy --strict` runs over `src`, `tests` and `tools`. |
|
||||
| 13 | Dependencies are declared and pinned in the project's dependency file, none unused | Fail | dependencies are declared with lower bounds, not pinned (see RC-009 action item); none is unused. |
|
||||
| 14 | Each piece of knowledge (a business rule, constant, format, validation or query) is defined in one place; code is merged only where it expresses the same knowledge, not where it merely looks alike | Pass | each constant and format is defined once; the step texts are built in one place. |
|
||||
| 15 | Imports point inward: business-rule modules import no framework, database, UI or delivery-mechanism code, and there are no import cycles | Pass | `domain` imports nothing from other layers; `infrastructure` imports `domain` and `constants`; `presentation` composes them. |
|
||||
| 16 | SOLID holds: a class or module has one reason to change; new behaviour is added by extension, not by editing a type test in several places; subclasses honour the contract of their base; protocols are narrow; high-level code depends on protocols or abstract types and receives its implementations | Pass | no class hierarchy; use cases take callables (ports), so a new source needs no edit of the use case. |
|
||||
| 17 | Source is laid out in the layers and folders of the clean architecture folder structure (`domain`, `application`, `infrastructure`, `presentation` packages under `src/`, and `unit`, `integration` and `e2e` under `tests/`), in `snake_case`, and imports between the layers point inward | Pass | code is in `domain`, `application`, `infrastructure`, `presentation`; tests are in `tests/unit` (network calls are faked; integration and e2e folders stay empty until needed). |
|
||||
| 18 | File and folder names are English, contain no space and no character outside letters, digits, `-`, `_` and `.`, and follow the casing of the language for files | Pass | all names are English lowercase with `_`, no space. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| Pin dependencies or accept lower bounds (open from RC-009, optional criterion 13) | S01 | 2026-11-14 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-003]: ../../milestones/mil-003-hacker-news.md
|
||||
[QC-PY-001]: ../../../framework/qc/code/qc-programming-python.md
|
||||
[81af6fa]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/81af6fa2e9675b4a27740caca50b7b4056688399
|
||||
@@ -1,63 +0,0 @@
|
||||
# Review of MIL-004 Python code
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-014 |
|
||||
| CrossReference | [MIL-004], [QC-PY-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [02c339c] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-004]
|
||||
- Checklist used: [QC-PY-001]
|
||||
- Scope: full review. Code reviewed: movie, empire_movies, movie_file, use_cases, movies_steps, cli, tools/, tests/unit and tests/integration.
|
||||
- Language and domain: n/a (technical type)
|
||||
- Language reviewer: none
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Packages, modules, functions, variables, classes and constants follow PEP 8 casing (`snake_case`, `PascalCase`, `UPPER_SNAKE`) | Pass | modules and functions are snake_case, classes PascalCase, constants UPPER_SNAKE. |
|
||||
| 2 | Names state purpose in the domain's language; no unexplained abbreviations, no single-letter names outside tiny scopes | Pass | names state purpose; the lecture names (`find_all`, `select_one`) are kept in the helper names. |
|
||||
| 3 | Code is produced by the project's formatter and passes its linter with no unexplained suppressions | Pass | `ruff check` and `ruff format --check` are clean with no `noqa`. |
|
||||
| 4 | Every function and method signature is type-annotated, including `-> None` | Pass | every signature is annotated; `mypy --strict` passes, also in a venv without the third-party packages. |
|
||||
| 5 | No bare `except:`, no swallowed exceptions; specific exceptions are raised and the cause is kept (`raise ... from`) | Pass | fetch errors are raised as `FetchError` from the cause (MIL-003); no exception is swallowed. |
|
||||
| 6 | No mutable default arguments and no shadowed builtins | Pass | no mutable defaults, no shadowed builtins. |
|
||||
| 7 | Files, locks and connections are managed with context managers | Pass | `write_movies` writes inside a `with path.open(...)` block. |
|
||||
| 8 | Public modules, classes and functions have docstrings that say what, not how | Pass | every public module, class and function has a Doxygen docstring saying what it does. |
|
||||
| 9 | Logging uses `logging`, not `print`; no secrets or personal data in log output | N-A | no logging; the command-line output is the product, written with `print` in the presentation layer. |
|
||||
| 10 | Classes and operations trace to the Design Class Diagram they implement; deviations are recorded | N-A | no Design Class Diagram; all tasks are plain technical tasks (Needs Use Case: No). |
|
||||
| 11 | Tests exist for new behaviour, are named for the behaviour, and do not depend on order or the network | Pass | tests are named for behaviour, offline and independent of order. |
|
||||
| 12 | Type checker runs in strict mode without errors; `Any` is justified in a comment | Pass | `mypy --strict` runs over `src`, `tests` and `tools`. |
|
||||
| 13 | Dependencies are declared and pinned in the project's dependency file, none unused | Fail | dependencies are declared with lower bounds, not pinned (see RC-009 action item); none is unused. |
|
||||
| 14 | Each piece of knowledge (a business rule, constant, format, validation or query) is defined in one place; code is merged only where it expresses the same knowledge, not where it merely looks alike | Pass | each constant and format is defined once; the step texts are built in one place. |
|
||||
| 15 | Imports point inward: business-rule modules import no framework, database, UI or delivery-mechanism code, and there are no import cycles | Pass | `domain` imports nothing from other layers; `infrastructure` imports `domain` and `constants`; `presentation` composes them. |
|
||||
| 16 | SOLID holds: a class or module has one reason to change; new behaviour is added by extension, not by editing a type test in several places; subclasses honour the contract of their base; protocols are narrow; high-level code depends on protocols or abstract types and receives its implementations | Pass | use cases take callables (ports); a new page layout needs a new parser, not an edit of the use case. |
|
||||
| 17 | Source is laid out in the layers and folders of the clean architecture folder structure (`domain`, `application`, `infrastructure`, `presentation` packages under `src/`, and `unit`, `integration` and `e2e` under `tests/`), in `snake_case`, and imports between the layers point inward | Pass | code is in `domain`, `application`, `infrastructure`, `presentation`; tests in `tests/unit` and `tests/integration` (the CLI-to-file flow); `e2e` stays empty because the CLI is covered by the integration test. |
|
||||
| 18 | File and folder names are English, contain no space and no character outside letters, digits, `-`, `_` and `.`, and follow the casing of the language for files | Pass | all names are English lowercase with `_`, no space. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| Pin dependencies or accept lower bounds (open from RC-009, optional criterion 13) | S01 | 2026-11-14 |
|
||||
| Refresh the Wayback fixture when archive.org stops answering 429 (the snapshot layout is covered by a hand-written sample) | S01 | 2026-11-14 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-004]: ../../milestones/mil-004-top-100-movies.md
|
||||
[QC-PY-001]: ../../../framework/qc/code/qc-programming-python.md
|
||||
[02c339c]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/02c339c2dbe95b0e2eff5e4a3738f6e946014d26
|
||||
@@ -1,63 +0,0 @@
|
||||
# Review of MIL-005 Python code
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | RC-015 |
|
||||
| CrossReference | [MIL-005], [QC-PY-001] |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Proposed | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [8f6b0fc] |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Under Review
|
||||
|
||||
- Instance reviewed: [MIL-005]
|
||||
- Checklist used: [QC-PY-001]
|
||||
- Scope: full review. Code reviewed: tests/unit/test_wiki.py and tools/make_screenshots.py (robots.txt step). The wiki pages are Markdown with no QC checklist; they are checked by test_wiki.py (page set, language line, screenshots, gains heading, code blocks, sidebar) and read by the stand-in reviewer.
|
||||
- Language and domain: n/a (technical type)
|
||||
- Language reviewer: none
|
||||
- Mode: autonomous, not independent (reviewer is `AI as stand-in for S01`)
|
||||
|
||||
## Checklist Results
|
||||
|
||||
| # | Criterion | Status | Evidence/Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | Packages, modules, functions, variables, classes and constants follow PEP 8 casing (`snake_case`, `PascalCase`, `UPPER_SNAKE`) | Pass | modules and functions are snake_case, classes PascalCase, constants UPPER_SNAKE. |
|
||||
| 2 | Names state purpose in the domain's language; no unexplained abbreviations, no single-letter names outside tiny scopes | Pass | names state purpose; the lecture names (`find_all`, `select_one`) are kept in the helper names. |
|
||||
| 3 | Code is produced by the project's formatter and passes its linter with no unexplained suppressions | Pass | `ruff check` and `ruff format --check` are clean with no `noqa`. |
|
||||
| 4 | Every function and method signature is type-annotated, including `-> None` | Pass | every signature is annotated; `mypy --strict` passes, also in a venv without the third-party packages. |
|
||||
| 5 | No bare `except:`, no swallowed exceptions; specific exceptions are raised and the cause is kept (`raise ... from`) | N-A | no exception is raised or caught. |
|
||||
| 6 | No mutable default arguments and no shadowed builtins | Pass | no mutable defaults, no shadowed builtins. |
|
||||
| 7 | Files, locks and connections are managed with context managers | Pass | `urllib.request.urlopen` is used in a `with` block in the screenshot tool. |
|
||||
| 8 | Public modules, classes and functions have docstrings that say what, not how | Pass | every public module, class and function has a Doxygen docstring saying what it does. |
|
||||
| 9 | Logging uses `logging`, not `print`; no secrets or personal data in log output | N-A | no logging; the tool prints the path of each image it writes. |
|
||||
| 10 | Classes and operations trace to the Design Class Diagram they implement; deviations are recorded | N-A | no Design Class Diagram; all tasks are plain technical tasks (Needs Use Case: No). |
|
||||
| 11 | Tests exist for new behaviour, are named for the behaviour, and do not depend on order or the network | Pass | tests are named for behaviour, offline and independent of order. |
|
||||
| 12 | Type checker runs in strict mode without errors; `Any` is justified in a comment | Pass | `mypy --strict` runs over `src`, `tests` and `tools`. |
|
||||
| 13 | Dependencies are declared and pinned in the project's dependency file, none unused | Fail | dependencies are declared with lower bounds, not pinned (see RC-009 action item); none is unused. |
|
||||
| 14 | Each piece of knowledge (a business rule, constant, format, validation or query) is defined in one place; code is merged only where it expresses the same knowledge, not where it merely looks alike | Pass | each constant and format is defined once; the step texts are built in one place. |
|
||||
| 15 | Imports point inward: business-rule modules import no framework, database, UI or delivery-mechanism code, and there are no import cycles | Pass | `domain` imports nothing from other layers; `infrastructure` imports `domain` and `constants`; `presentation` composes them. |
|
||||
| 16 | SOLID holds: a class or module has one reason to change; new behaviour is added by extension, not by editing a type test in several places; subclasses honour the contract of their base; protocols are narrow; high-level code depends on protocols or abstract types and receives its implementations | Pass | no class hierarchy; the dataclass `Link` has one reason to change. |
|
||||
| 17 | Source is laid out in the layers and folders of the clean architecture folder structure (`domain`, `application`, `infrastructure`, `presentation` packages under `src/`, and `unit`, `integration` and `e2e` under `tests/`), in `snake_case`, and imports between the layers point inward | Pass | the wiki test is in `tests/unit`; no source layer changed. |
|
||||
| 18 | File and folder names are English, contain no space and no character outside letters, digits, `-`, `_` and `.`, and follow the casing of the language for files | Pass | all names are English lowercase with `_`, no space. |
|
||||
|
||||
## Overall Verdict
|
||||
|
||||
Go — no Mandatory criterion fails and none is Pending. Stand-in review in autonomous mode, not independent; S01 challenges it in the run summary.
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
| --- | --- | --- |
|
||||
| Pin dependencies or accept lower bounds (open from RC-009, optional criterion 13) | S01 | 2026-11-14 |
|
||||
| Ask a native Danish and Malay reader to review the translations (the translations are written by the AI) | S01 | 2026-11-14 |
|
||||
|
||||
---
|
||||
|
||||
[MIL-005]: ../../milestones/mil-005-wiki.md
|
||||
[QC-PY-001]: ../../../framework/qc/code/qc-programming-python.md
|
||||
[8f6b0fc]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/8f6b0fc541ba26d5212d5cb8e6ef579ca14b32fe
|
||||
@@ -1,86 +0,0 @@
|
||||
# Stakeholder Analysis
|
||||
|
||||
## Metadata
|
||||
| Key | Value |
|
||||
| --- | --- |
|
||||
| ID | SA-001 |
|
||||
| CrossReference | [BC-001] |
|
||||
| Language | en |
|
||||
| Domain | it |
|
||||
|
||||
## Version History
|
||||
| Date | Status | Author | Reviewer | Change | Commit |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 2026-10-28 | Accepted | Jens Tirsvad Nielsen | S01 (AI as stand-in, autonomous mode) | Initial version | [dcef5e6] |
|
||||
|
||||
---
|
||||
|
||||
## Purpose
|
||||
|
||||
Identify the people the repository serves and what each needs, so that the Business Case objectives, milestones and documents can name an owner, reviewer and audience. Method: power/interest grid with a FURPS+ mapping of each concern.
|
||||
|
||||
## Stakeholder Summary Table
|
||||
|
||||
| ID | Name | Role/Title | Organization | Power Level | Interest Level | Quadrant | Primary Concern (Business Language) |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| S01 | Jens Tirsvad Nielsen | Course participant, Product Owner, developer and reviewer | Tirsvad | HIGH | HIGH | Manage Closely | Finish the Day 45 assignment with clean, documented and reviewed work |
|
||||
| S02 | Udemy coursists | Fellow participants of the course | Udemy community | LOW | HIGH | Keep Informed | Readable, runnable code with the assignment's function names and README instructions to run it |
|
||||
| S03 | GitHub viewers | Visitors browsing for ideas | Public | LOW | LOW | Monitor | A clear repository description, topics and README, and no runtime dependencies beyond the need |
|
||||
|
||||
## Power/Interest Classification Rationale
|
||||
|
||||
- **Manage Closely (S01):** decides scope, accepts the plan, reviews the summary and owns every artifact.
|
||||
- **Keep Informed (S02):** cannot change the project but depends on the quality of the code and README.
|
||||
- **Monitor (S03):** casual audience; served by the repository front page only.
|
||||
|
||||
## Primary Concerns and FURPS+ Mapping
|
||||
|
||||
| Concern ID | Stakeholder | Concern | FURPS+ attribute |
|
||||
| --- | --- | --- | --- |
|
||||
| CN-001 | S01 | Every lecture step is reproduced and its learning documented | Functionality |
|
||||
| CN-002 | S01 | Work is traceable: plan, branch and pull request per milestone | Supportability |
|
||||
| CN-003 | S01 | Personal tokens never enter the project | Functionality (security) |
|
||||
| CN-004 | S02 | Code is readable and keeps the assignment's function names | Usability |
|
||||
| CN-005 | S02 | Instructions to create the `.venv`, install and run the tests work as written | Usability |
|
||||
| CN-006 | S02 | Tests run offline and are deterministic | Reliability |
|
||||
| CN-007 | S03 | Repository description, topics and README explain the purpose at a glance | Usability |
|
||||
| CN-008 | S03 | No runtime dependencies beyond what scraping needs | Implementation |
|
||||
| CN-009 | S01, S02 | Wiki in English, Danish and Bahasa Malaysia with a screenshot per lecture step | Usability |
|
||||
|
||||
## Communication Requirements
|
||||
|
||||
| ID | Channel | Frequency | Deliverable | Phase / Milestone |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| C1 | Chat with S01 | At plan acceptance and at the end of the autonomous run | Plan dry run, then run summary with RC verdicts | All |
|
||||
| C2 | Pull request per milestone | Once per milestone | PR description with `Closes #N` lines | All |
|
||||
| C3 | README and wiki | On each merge | Instructions, screenshots, lecture learnings | MIL-001 to MIL-005 |
|
||||
| C4 | Repository description and topics | Once | Metadata on the git host | MIL-001 |
|
||||
|
||||
## Conflicting Interests and Mitigations
|
||||
|
||||
| Conflict | Stakeholders | Mitigation |
|
||||
| --- | --- | --- |
|
||||
| S03 wants no dependencies; the assignment needs `requests` and `beautifulsoup4` | S03, S01 | Keep exactly those two runtime dependencies; development tools stay in an optional `dev` group |
|
||||
| S02 wants assignment names; clean code wants descriptive names | S02, S01 | Keep the assignment names as public functions and add thin, documented helpers |
|
||||
| Live scraping vs deterministic tests | S02, S01 | Parsing functions take HTML text; tests use saved fixtures |
|
||||
|
||||
## Traceability Analysis
|
||||
|
||||
### Business Goal Alignment
|
||||
|
||||
| Stakeholder | Concern | Business Case objective |
|
||||
| --- | --- | --- |
|
||||
| S01 | CN-001 | O1, O2 [BC-001] |
|
||||
| S01 | CN-002, CN-003 | O6 [BC-001] |
|
||||
| S02 | CN-004, CN-005, CN-006 | O1, O3, O4 [BC-001] |
|
||||
| S03 | CN-007, CN-008 | O4, O6 [BC-001] |
|
||||
| S01, S02 | CN-009 | O5 [BC-001] |
|
||||
|
||||
## Sign-Off
|
||||
|
||||
Reviewer: S01 (decision recorded in the RC record of this document).
|
||||
|
||||
---
|
||||
|
||||
[BC-001]: ./business-case.md
|
||||
[dcef5e6]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies/commit/dcef5e6e6e7ec16162817d3ec0ce1b303e0a3a63
|
||||
@@ -1,17 +0,0 @@
|
||||
[English](Gains) · **Dansk** · [Bahasa Melayu](Gains-ms)
|
||||
|
||||
# Hvad vi lærte
|
||||
|
||||
| Forelæsning | Hvad vi kan nu |
|
||||
| --- | --- |
|
||||
| [Parsing af HTML](Parsing-HTML-da) | Lave en suppe, læse dens struktur og vælge elementer via tag, id, klasse eller CSS-selektor |
|
||||
| [Hacker News](Hacker-News-da) | Hente en live-side, læse titler, links og scores, omdanne tekst til tal og finde maksimum |
|
||||
| [Etik ved scraping](Scraping-Ethics-da) | Tjekke `robots.txt`, respektere ophavsret og logins og holde forespørgslerne sjældne |
|
||||
| [Top 100 film](Top-100-Movies-da) | Scrape en rangeret liste, vende den om og skrive den til en fil |
|
||||
| [Opsætning](Setup-da) | Arbejde i et virtuelt miljø med `pyproject.toml` og offline-tests |
|
||||
|
||||
## Ud over forelæsningerne
|
||||
|
||||
- **Parsing er adskilt fra hentning.** Parsing-funktionerne tager tekst, så testene bruger gemte sider og aldrig har brug for netværket.
|
||||
- **Lag holder reglerne adskilt.** Rangeringsreglen kender hverken BeautifulSoup eller Requests.
|
||||
- **Skærmbillederne laves af et script** (`tools/make_screenshots.py`), så de kan genskabes efter en ændring.
|
||||
@@ -1,17 +0,0 @@
|
||||
[English](Gains) · [Dansk](Gains-da) · **Bahasa Melayu**
|
||||
|
||||
# Apa yang kita pelajari
|
||||
|
||||
| Kuliah | Apa yang boleh kita lakukan sekarang |
|
||||
| --- | --- |
|
||||
| [Menghurai HTML](Parsing-HTML-ms) | Membuat sup, membaca strukturnya dan memilih elemen mengikut tag, id, kelas atau pemilih CSS |
|
||||
| [Hacker News](Hacker-News-ms) | Mengambil halaman langsung, membaca tajuk, pautan dan skor, menukar teks kepada nombor dan mencari nilai maksimum |
|
||||
| [Etika mengikis web](Scraping-Ethics-ms) | Menyemak `robots.txt`, menghormati hak cipta dan log masuk, menjarangkan permintaan |
|
||||
| [100 filem terbaik](Top-100-Movies-ms) | Mengikis senarai berkedudukan, membalikkannya dan menulisnya ke fail |
|
||||
| [Persediaan](Setup-ms) | Bekerja dalam persekitaran maya dengan `pyproject.toml` dan ujian tanpa talian |
|
||||
|
||||
## Selain kuliah
|
||||
|
||||
- **Penghuraian dipisahkan daripada pengambilan.** Fungsi penghuraian menerima teks, jadi ujian menggunakan halaman tersimpan dan tidak memerlukan rangkaian.
|
||||
- **Lapisan mengasingkan peraturan.** Peraturan kedudukan tidak mengetahui BeautifulSoup atau Requests.
|
||||
- **Tangkapan skrin dibuat oleh skrip** (`tools/make_screenshots.py`), jadi ia boleh dijana semula selepas perubahan.
|
||||
@@ -1,17 +0,0 @@
|
||||
**English** · [Dansk](Gains-da) · [Bahasa Melayu](Gains-ms)
|
||||
|
||||
# What we gained
|
||||
|
||||
| Lecture | What we can do now |
|
||||
| --- | --- |
|
||||
| [Parsing HTML](Parsing-HTML) | Make a soup, read its structure and select elements by tag, id, class or CSS selector |
|
||||
| [Hacker News](Hacker-News) | Fetch a live page, read titles, links and scores, convert text to numbers, find the maximum |
|
||||
| [Scraping ethics](Scraping-Ethics) | Check `robots.txt`, respect copyright and logins, keep requests rare |
|
||||
| [Top 100 movies](Top-100-Movies) | Scrape a ranked list, reverse it and write it to a file |
|
||||
| [Setup](Setup) | Work in a virtual environment with `pyproject.toml` and offline tests |
|
||||
|
||||
## Beyond the lectures
|
||||
|
||||
- **Parsing is separate from fetching.** The parsing functions take text, so the tests use saved pages and never need the network.
|
||||
- **Layers keep rules apart.** The ranking rule has no knowledge of BeautifulSoup or Requests.
|
||||
- **Screenshots are made by a script** (`tools/make_screenshots.py`), so they can be regenerated after a change.
|
||||
@@ -1,57 +0,0 @@
|
||||
[English](Hacker-News) · **Dansk** · [Bahasa Melayu](Hacker-News-ms)
|
||||
|
||||
# Scraping af Hacker News
|
||||
|
||||
Forelæsning: *Scraping af et live-websted*. Koden er `hacker_news.py` (parsing) og `http_client.py` (ét `requests.get` med timeout og `User-Agent`). Kommandoerne læser kursets statiske kopi af siden; `--source live` læser det rigtige websted, og `--source fixture` læser den gemte kopi uden netværk.
|
||||
|
||||
## Trin 1: se på siden
|
||||
|
||||
Siden viser historier med en titel, et link og en score.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** live-data er rigere end en lokal fil, men de kan ændre sig, så vi gemmer også en kopi til testene.
|
||||
|
||||
## Trin 2: undersøg markup
|
||||
|
||||
Browserens udviklerværktøjer viser tagsene: en række `tr.athing`, titellinket `.titleline > a` og en `span.score` i næste række.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** vi finder det rigtige tag ved at undersøge siden, før vi skriver kode.
|
||||
|
||||
## Trin 3: titler
|
||||
|
||||
`find_all` på titellinkene og `get_text()`.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** en list comprehension gør mange tags til en almindelig liste af strenge.
|
||||
|
||||
## Trin 4: links
|
||||
|
||||
`get("href")` på de samme links.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** det samme element giver både tekst og attributværdier.
|
||||
|
||||
## Trin 5: upvotes
|
||||
|
||||
Teksten `38 points` deles op og omdannes med `int()`.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** scrapet tekst skal omdannes til tal, før den kan sammenlignes.
|
||||
|
||||
## Trin 6: den øverste historie
|
||||
|
||||
`max` over upvotes finder den mest populære artikel.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** når data er rene, besvarer almindelig Python spørgsmål om dem.
|
||||
|
||||
## Hvad vi lærte
|
||||
|
||||
Vi kan hente en side med Requests, finde rækkerne, læse titler, links og scores, omdanne dem til brugbare typer og besvare et spørgsmål om data.
|
||||
@@ -1,57 +0,0 @@
|
||||
[English](Hacker-News) · [Dansk](Hacker-News-da) · **Bahasa Melayu**
|
||||
|
||||
# Mengikis Hacker News
|
||||
|
||||
Kuliah: *Mengikis laman web langsung*. Kodnya ialah `hacker_news.py` (penghuraian) dan `http_client.py` (satu `requests.get` dengan had masa dan `User-Agent`). Arahan membaca salinan statik halaman daripada kursus; `--source live` membaca laman sebenar dan `--source fixture` membaca salinan tersimpan tanpa rangkaian.
|
||||
|
||||
## Langkah 1: lihat halaman
|
||||
|
||||
Halaman menyenaraikan berita dengan tajuk, pautan dan skor.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** data langsung lebih kaya daripada fail tempatan, tetapi boleh berubah, jadi kita juga menyimpan salinan untuk ujian.
|
||||
|
||||
## Langkah 2: periksa markup
|
||||
|
||||
Alat pembangun pelayar menunjukkan tag: baris `tr.athing`, pautan tajuk `.titleline > a` dan `span.score` pada baris seterusnya.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** kita menemui tag yang betul dengan memeriksa halaman sebelum menulis kod.
|
||||
|
||||
## Langkah 3: tajuk
|
||||
|
||||
`find_all` pada pautan tajuk dan `get_text()`.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** list comprehension menukar banyak tag menjadi senarai rentetan biasa.
|
||||
|
||||
## Langkah 4: pautan
|
||||
|
||||
`get("href")` pada anchor yang sama.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** elemen yang sama memberi teks dan nilai atribut.
|
||||
|
||||
## Langkah 5: undian
|
||||
|
||||
Teks `38 points` dipecahkan dan ditukar dengan `int()`.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** teks yang dikikis mesti ditukar kepada nombor sebelum boleh dibandingkan.
|
||||
|
||||
## Langkah 6: berita teratas
|
||||
|
||||
`max` pada undian mencari artikel paling popular.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** apabila data sudah bersih, Python biasa menjawab soalan tentangnya.
|
||||
|
||||
## Apa yang kita pelajari
|
||||
|
||||
Kita boleh mengambil halaman dengan Requests, menemui baris, membaca tajuk, pautan dan skor, menukarnya kepada jenis yang berguna dan menjawab soalan tentang data.
|
||||
@@ -1,57 +0,0 @@
|
||||
**English** · [Dansk](Hacker-News-da) · [Bahasa Melayu](Hacker-News-ms)
|
||||
|
||||
# Scraping Hacker News
|
||||
|
||||
Lecture: *Scraping a live website*. The code is `hacker_news.py` (parsing) and `http_client.py` (one `requests.get` with a timeout and a `User-Agent`). The commands read the course's static copy of the page; `--source live` reads the real site and `--source fixture` reads the saved copy without network.
|
||||
|
||||
## Step 1: look at the page
|
||||
|
||||
The page lists stories with a title, a link and a score.
|
||||
|
||||

|
||||
|
||||
**What we gained:** live data is richer than a local file, but it can change, so we also keep a saved copy for tests.
|
||||
|
||||
## Step 2: inspect the markup
|
||||
|
||||
The browser's developer tools show the tags: a row `tr.athing`, the title link `.titleline > a` and a `span.score` in the next row.
|
||||
|
||||

|
||||
|
||||
**What we gained:** we find the right tag by inspecting the page before we write any code.
|
||||
|
||||
## Step 3: titles
|
||||
|
||||
`find_all` on the title links and `get_text()`.
|
||||
|
||||

|
||||
|
||||
**What we gained:** a list comprehension turns many tags into a plain list of strings.
|
||||
|
||||
## Step 4: links
|
||||
|
||||
`get("href")` of the same anchors.
|
||||
|
||||

|
||||
|
||||
**What we gained:** the same element gives both text and attribute values.
|
||||
|
||||
## Step 5: upvotes
|
||||
|
||||
The text `38 points` is split and converted with `int()`.
|
||||
|
||||

|
||||
|
||||
**What we gained:** scraped text must be converted to numbers before it can be compared.
|
||||
|
||||
## Step 6: the top story
|
||||
|
||||
`max` over the upvotes finds the most popular article.
|
||||
|
||||

|
||||
|
||||
**What we gained:** once the data is clean, ordinary Python answers questions about it.
|
||||
|
||||
## What we gained
|
||||
|
||||
We can fetch a page with Requests, find the rows, read titles, links and scores, convert them to usable types and answer a question about the data.
|
||||
@@ -1,23 +0,0 @@
|
||||
[English](Home) · **Dansk** · [Bahasa Melayu](Home-ms)
|
||||
|
||||
# scrape-movies wiki
|
||||
|
||||
Dag 45 af Udemys "100 Days of Code: The Complete Python Pro Bootcamp" handler om web scraping med Requests og BeautifulSoup. Dette repository gengiver hvert trin fra forelæsningerne som testet Python-kode. Wikien viser hvert trin med et skærmbillede og forklarer, hvad vi lærte af det.
|
||||
|
||||
## Sider
|
||||
|
||||
| Side | Forelæsning | Indhold |
|
||||
| --- | --- | --- |
|
||||
| [Opsætning](Setup-da) | - | Opret `.venv`, installér og kør testene |
|
||||
| [Parsing af HTML](Parsing-HTML-da) | Parsing af HTML og at lave suppe; find og vælg elementer | Fem trin med skærmbilleder |
|
||||
| [Hacker News](Hacker-News-da) | Scraping af et live-websted | Seks trin med skærmbilleder |
|
||||
| [Etik ved scraping](Scraping-Ethics-da) | Er web scraping lovligt og etisk? | Reglerne vi følger, med et skærmbillede af `robots.txt` |
|
||||
| [Top 100 film](Top-100-Movies-da) | Projektet med de 100 film | Seks trin og `movies.txt` |
|
||||
| [Hvad vi lærte](Gains-da) | Alle forelæsninger | Et kort resumé af færdighederne |
|
||||
|
||||
## Til hvem
|
||||
|
||||
- **Kursister:** læsbar, kørbar kode med opgavens idéer; sammenlign den med din egen løsning.
|
||||
- **Besøgende:** et lille eksempel med tests og kun to afhængigheder ved kørsel, `requests` og `beautifulsoup4`.
|
||||
|
||||
Koden og dokumenterne findes i [repositoryet](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies).
|
||||
@@ -1,23 +0,0 @@
|
||||
[English](Home) · [Dansk](Home-da) · **Bahasa Melayu**
|
||||
|
||||
# Wiki scrape-movies
|
||||
|
||||
Hari ke-45 kursus Udemy "100 Days of Code: The Complete Python Pro Bootcamp" membincangkan pengikisan web (web scraping) dengan Requests dan BeautifulSoup. Repositori ini menghasilkan semula setiap langkah kuliah sebagai kod Python yang diuji. Wiki ini menunjukkan setiap langkah dengan tangkapan skrin dan menerangkan apa yang kita pelajari daripadanya.
|
||||
|
||||
## Halaman
|
||||
|
||||
| Halaman | Kuliah | Kandungan |
|
||||
| --- | --- | --- |
|
||||
| [Persediaan](Setup-ms) | - | Cipta `.venv`, pasang dan jalankan ujian |
|
||||
| [Menghurai HTML](Parsing-HTML-ms) | Menghurai HTML dan membuat sup; mencari dan memilih elemen | Lima langkah dengan tangkapan skrin |
|
||||
| [Hacker News](Hacker-News-ms) | Mengikis laman web langsung | Enam langkah dengan tangkapan skrin |
|
||||
| [Etika mengikis web](Scraping-Ethics-ms) | Adakah pengikisan web sah dan beretika? | Peraturan yang kita ikut, dengan tangkapan skrin `robots.txt` |
|
||||
| [100 filem terbaik](Top-100-Movies-ms) | Projek 100 filem | Enam langkah dan `movies.txt` |
|
||||
| [Apa yang kita pelajari](Gains-ms) | Semua kuliah | Ringkasan kemahiran |
|
||||
|
||||
## Untuk siapa
|
||||
|
||||
- **Peserta kursus:** kod yang mudah dibaca dan boleh dijalankan dengan idea tugasan; bandingkan dengan penyelesaian anda sendiri.
|
||||
- **Pelawat:** contoh kecil dengan ujian dan hanya dua kebergantungan semasa larian, `requests` dan `beautifulsoup4`.
|
||||
|
||||
Kod dan dokumen terdapat dalam [repositori](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies).
|
||||
@@ -1,23 +0,0 @@
|
||||
**English** · [Dansk](Home-da) · [Bahasa Melayu](Home-ms)
|
||||
|
||||
# scrape-movies wiki
|
||||
|
||||
Day 45 of Udemy's "100 Days of Code: The Complete Python Pro Bootcamp" is about web scraping with Requests and BeautifulSoup. This repository reproduces every lecture step as tested Python code. This wiki shows each step with a screenshot and explains what we gained from it.
|
||||
|
||||
## Pages
|
||||
|
||||
| Page | Lecture | What you get |
|
||||
| --- | --- | --- |
|
||||
| [Setup](Setup) | - | Create the `.venv`, install, run the tests |
|
||||
| [Parsing HTML](Parsing-HTML) | Parsing HTML and making soup; finding and selecting elements | Five steps with screenshots |
|
||||
| [Hacker News](Hacker-News) | Scraping a live website | Six steps with screenshots |
|
||||
| [Scraping ethics](Scraping-Ethics) | Is web scraping legal and ethical? | The rules we follow, with a screenshot of `robots.txt` |
|
||||
| [Top 100 movies](Top-100-Movies) | The 100 movies project | Six steps and `movies.txt` |
|
||||
| [What we gained](Gains) | All lectures | A short summary of the skills |
|
||||
|
||||
## For whom
|
||||
|
||||
- **Coursists:** readable, runnable code that keeps the assignment's ideas; compare it with your own solution.
|
||||
- **Visitors:** a small example with tests and only two runtime dependencies, `requests` and `beautifulsoup4`.
|
||||
|
||||
The code and the documents are in the [repository](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies).
|
||||
@@ -1,49 +0,0 @@
|
||||
[English](Parsing-HTML) · **Dansk** · [Bahasa Melayu](Parsing-HTML-ms)
|
||||
|
||||
# Parsing af HTML
|
||||
|
||||
Forelæsninger: *Parsing af HTML og at lave suppe* og *Find og vælg elementer*. Eksempelsiden er `tests/fixtures/website.html`; koden er `soup_basics.py`.
|
||||
|
||||
## Trin 1: lav en suppe og læs titlen
|
||||
|
||||
`BeautifulSoup(html, "html.parser")` gør HTML-teksten til et suppe-objekt. `soup.title.string` er teksten i title-tagget.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** HTML-tekst bliver til et objekt, vi kan navigere i, og et tag nås med almindelig attributadgang.
|
||||
|
||||
## Trin 2: prettify
|
||||
|
||||
`soup.prettify()` skriver dokumentet med ét tag pr. linje, så indlejringen bliver synlig.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** når vi først læser strukturen, ved vi, hvad vi skal spørge om.
|
||||
|
||||
## Trin 3: find alle tags
|
||||
|
||||
`find_all("a")` giver alle links. I en løkke giver `get_text()` teksten og `get("href")` adressen.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** ét kald finder mange elementer, og en løkke gør dem til data.
|
||||
|
||||
## Trin 4: find med id og klasse
|
||||
|
||||
`find(id="name")` finder ét tag via dets id. `find_all("h3", class_="heading")` filtrerer på tag og klasse; nøgleordet er `class_`, fordi `class` er et reserveret ord i Python.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** attributter indsnævrer en søgning, og vi undgår sammenstød med reserverede ord.
|
||||
|
||||
## Trin 5: CSS-selektorer
|
||||
|
||||
`select_one("p a")` og `select(".heading")` bruger de samme selektorer som CSS, så indlejrede elementer nås i ét udtryk.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** CSS-selektorer er den korteste måde at beskrive, hvor et element sidder i dokumentet.
|
||||
|
||||
## Hvad vi lærte
|
||||
|
||||
Vi kan læse en HTML-fil ind i BeautifulSoup, se dens struktur og udvælge enkelte eller mange elementer via tag, id, klasse eller CSS-selektor.
|
||||
@@ -1,49 +0,0 @@
|
||||
[English](Parsing-HTML) · [Dansk](Parsing-HTML-da) · **Bahasa Melayu**
|
||||
|
||||
# Menghurai HTML
|
||||
|
||||
Kuliah: *Menghurai HTML dan Membuat Sup* dan *Mencari dan Memilih Elemen*. Halaman contoh ialah `tests/fixtures/website.html`; kodnya ialah `soup_basics.py`.
|
||||
|
||||
## Langkah 1: cipta sup dan baca tajuk
|
||||
|
||||
`BeautifulSoup(html, "html.parser")` menukar teks HTML menjadi objek sup. `soup.title.string` ialah teks dalam tag title.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** teks HTML menjadi objek yang boleh dilayari, dan satu tag dicapai dengan akses atribut biasa.
|
||||
|
||||
## Langkah 2: prettify
|
||||
|
||||
`soup.prettify()` menulis dokumen dengan satu tag setiap baris supaya sarangan kelihatan.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** dengan membaca struktur dahulu, kita tahu apa yang perlu diminta.
|
||||
|
||||
## Langkah 3: cari semua tag
|
||||
|
||||
`find_all("a")` memberi semua pautan. Dalam gelung, `get_text()` memberi teks dan `get("href")` memberi alamat.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** satu panggilan menemui banyak elemen, dan gelung menukarnya menjadi data.
|
||||
|
||||
## Langkah 4: cari mengikut id dan kelas
|
||||
|
||||
`find(id="name")` mencari satu tag mengikut id. `find_all("h3", class_="heading")` menapis mengikut tag dan kelas; kata kuncinya ialah `class_` kerana `class` ialah kata kunci Python.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** atribut mengecilkan carian, dan kita mengelak pertembungan dengan kata terpelihara.
|
||||
|
||||
## Langkah 5: pemilih CSS
|
||||
|
||||
`select_one("p a")` dan `select(".heading")` menggunakan pemilih yang sama seperti CSS, jadi elemen bersarang dicapai dalam satu ungkapan.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** pemilih CSS ialah cara paling ringkas untuk menerangkan kedudukan elemen dalam dokumen.
|
||||
|
||||
## Apa yang kita pelajari
|
||||
|
||||
Kita boleh membaca fail HTML ke dalam BeautifulSoup, melihat strukturnya dan memilih satu atau banyak elemen mengikut tag, id, kelas atau pemilih CSS.
|
||||
@@ -1,49 +0,0 @@
|
||||
**English** · [Dansk](Parsing-HTML-da) · [Bahasa Melayu](Parsing-HTML-ms)
|
||||
|
||||
# Parsing HTML
|
||||
|
||||
Lectures: *Parsing HTML and Making Soup* and *Finding and Selecting Elements*. The sample page is `tests/fixtures/website.html`; the code is `soup_basics.py`.
|
||||
|
||||
## Step 1: create a soup and read the title
|
||||
|
||||
`BeautifulSoup(html, "html.parser")` turns the HTML text into a soup object. `soup.title.string` is the text of the title tag.
|
||||
|
||||

|
||||
|
||||
**What we gained:** HTML text becomes an object we can navigate, and a tag is reached with plain attribute access.
|
||||
|
||||
## Step 2: prettify
|
||||
|
||||
`soup.prettify()` writes the document with one tag per line, which makes the nesting visible.
|
||||
|
||||

|
||||
|
||||
**What we gained:** reading the structure first tells us what to ask for.
|
||||
|
||||
## Step 3: find all tags
|
||||
|
||||
`find_all("a")` returns every anchor. In a loop, `get_text()` gives the text and `get("href")` the address.
|
||||
|
||||

|
||||
|
||||
**What we gained:** one call finds many elements, and a loop turns them into data.
|
||||
|
||||
## Step 4: find by id and by class
|
||||
|
||||
`find(id="name")` finds one tag by its id. `find_all("h3", class_="heading")` filters by tag and class; the keyword is `class_` because `class` is a Python keyword.
|
||||
|
||||

|
||||
|
||||
**What we gained:** attributes narrow a search, and we avoid clashes with reserved words.
|
||||
|
||||
## Step 5: CSS selectors
|
||||
|
||||
`select_one("p a")` and `select(".heading")` use the same selectors as CSS, so nested elements are reached in one expression.
|
||||
|
||||

|
||||
|
||||
**What we gained:** CSS selectors are the shortest way to describe where an element sits in the document.
|
||||
|
||||
## What we gained
|
||||
|
||||
We can read an HTML file into BeautifulSoup, look at its structure and pick out single or many elements by tag, id, class or CSS selector.
|
||||
@@ -1,24 +0,0 @@
|
||||
[English](Scraping-Ethics) · **Dansk** · [Bahasa Melayu](Scraping-Ethics-ms)
|
||||
|
||||
# Er web scraping lovligt og etisk?
|
||||
|
||||
Forelæsning: *Web scrapings juridiske og etiske side*. Dette er et resumé med egne ord og ikke juridisk rådgivning.
|
||||
|
||||
## Trin 1: læs robots.txt først
|
||||
|
||||
Et websteds `robots.txt` fortæller, hvad det tillader. Hacker News forbyder nogle stier og beder om en pause på 30 sekunder mellem forespørgsler.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** vi tjekker reglerne, før vi scraper, og vi holder forespørgslerne sjældne: dette projekt henter én side pr. kørsel.
|
||||
|
||||
## Reglerne vi følger
|
||||
|
||||
- Offentlige data uden ophavsret må generelt gerne læses. Sagerne Genius mod Google og hiQ mod LinkedIn viser, at grænsen ikke er enkel.
|
||||
- Indhold med ophavsret, som videoer eller blogindlæg, må ikke kopieres. Data bag et login er omfattet af webstedets brugeraftale.
|
||||
- Kommerciel brug af scrapede data kan føre til dyre tvister, selv når selve scrapingen var tilladt.
|
||||
- Foretræk et officielt API, respektér `robots.txt`, send en `User-Agent`, og send højst én forespørgsel i minuttet.
|
||||
|
||||
## Hvad vi lærte
|
||||
|
||||
Scraping er en teknisk færdighed med juridiske og etiske grænser. At kende grænserne er en del af at gøre det godt.
|
||||
@@ -1,24 +0,0 @@
|
||||
[English](Scraping-Ethics) · [Dansk](Scraping-Ethics-da) · **Bahasa Melayu**
|
||||
|
||||
# Adakah pengikisan web sah dan beretika?
|
||||
|
||||
Kuliah: *Aspek undang-undang dan etika pengikisan web*. Ini ringkasan dengan kata-kata sendiri dan bukan nasihat undang-undang.
|
||||
|
||||
## Langkah 1: baca robots.txt dahulu
|
||||
|
||||
`robots.txt` sesebuah laman menyatakan apa yang dibenarkan. Hacker News melarang beberapa laluan dan meminta jeda 30 saat antara permintaan.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** kita menyemak peraturan sebelum mengikis dan menjarangkan permintaan: projek ini mengambil satu halaman setiap larian.
|
||||
|
||||
## Peraturan yang kita ikut
|
||||
|
||||
- Data awam yang tidak dilindungi hak cipta umumnya boleh dibaca. Kes Genius lawan Google dan hiQ lawan LinkedIn menunjukkan bahawa garisnya tidak mudah.
|
||||
- Kandungan berhak cipta, seperti video atau catatan blog, tidak boleh disalin. Data di sebalik log masuk dilindungi perjanjian pengguna laman itu.
|
||||
- Penggunaan komersial data yang dikikis boleh membawa pertikaian yang mahal walaupun pengikisan itu sendiri dibenarkan.
|
||||
- Pilih API rasmi jika ada, hormati `robots.txt`, hantar `User-Agent` dan jangan hantar lebih daripada satu permintaan seminit.
|
||||
|
||||
## Apa yang kita pelajari
|
||||
|
||||
Pengikisan web ialah kemahiran teknikal yang mempunyai had undang-undang dan etika. Mengetahui had itu sebahagian daripada melakukannya dengan baik.
|
||||
@@ -1,24 +0,0 @@
|
||||
**English** · [Dansk](Scraping-Ethics-da) · [Bahasa Melayu](Scraping-Ethics-ms)
|
||||
|
||||
# Is web scraping legal and ethical?
|
||||
|
||||
Lecture: *The legal and ethical side of web scraping*. This is a summary in our own words, not legal advice.
|
||||
|
||||
## Step 1: read robots.txt first
|
||||
|
||||
A site's `robots.txt` says what it allows. Hacker News disallows some paths and asks for a crawl delay of 30 seconds.
|
||||
|
||||

|
||||
|
||||
**What we gained:** we check the rules before we scrape, and we keep our requests rare: this project fetches one page per run.
|
||||
|
||||
## The rules we follow
|
||||
|
||||
- Public data that is not copyrighted is generally fine to read. The cases Genius v. Google and hiQ v. LinkedIn show that the line is not simple.
|
||||
- Copyrighted content, such as videos or blog posts, must not be copied. Data behind a login is covered by the site's user agreement.
|
||||
- Commercial use of scraped data can lead to costly disputes even when the scraping itself was allowed.
|
||||
- Prefer an official API, respect `robots.txt`, send a `User-Agent` and do not send more than one request a minute.
|
||||
|
||||
## What we gained
|
||||
|
||||
Scraping is a technical skill with legal and ethical limits. Knowing the limits is part of doing it well.
|
||||
@@ -1,38 +0,0 @@
|
||||
[English](Setup) · **Dansk** · [Bahasa Melayu](Setup-ms)
|
||||
|
||||
# Opsætning
|
||||
|
||||
Du skal bruge Python 3.13 eller nyere. Alt nedenfor køres fra repositoryets rodmappe.
|
||||
|
||||
## Opret det virtuelle miljø
|
||||
|
||||
```powershell
|
||||
python -m venv .venv
|
||||
.venv\Scripts\Activate.ps1
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
På Linux og macOS bruges `python3 -m venv .venv` og `source .venv/bin/activate`.
|
||||
|
||||
## Kør et trin fra forelæsningen
|
||||
|
||||
```bash
|
||||
python -m scrape_movies soup title
|
||||
python -m scrape_movies hacker-news top
|
||||
python -m scrape_movies movies write
|
||||
```
|
||||
|
||||
## Kør testene
|
||||
|
||||
```bash
|
||||
python -m pytest
|
||||
```
|
||||
|
||||
Testene bruger gemte sider fra `tests/fixtures/` og rører aldrig netværket.
|
||||
|
||||
## Hvad vi lærte
|
||||
|
||||
- Et virtuelt miljø holder projektets pakker adskilt fra systemets Python.
|
||||
- `pyproject.toml` er det ene sted for Python-versionen, afhængighederne og værktøjsindstillingerne.
|
||||
- Tests, der kører offline, giver det samme resultat på alle maskiner.
|
||||
@@ -1,38 +0,0 @@
|
||||
[English](Setup) · [Dansk](Setup-da) · **Bahasa Melayu**
|
||||
|
||||
# Persediaan
|
||||
|
||||
Anda memerlukan Python 3.13 atau lebih baharu. Semua arahan di bawah dijalankan dari akar repositori.
|
||||
|
||||
## Cipta persekitaran maya
|
||||
|
||||
```powershell
|
||||
python -m venv .venv
|
||||
.venv\Scripts\Activate.ps1
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
Di Linux dan macOS gunakan `python3 -m venv .venv` dan `source .venv/bin/activate`.
|
||||
|
||||
## Jalankan satu langkah kuliah
|
||||
|
||||
```bash
|
||||
python -m scrape_movies soup title
|
||||
python -m scrape_movies hacker-news top
|
||||
python -m scrape_movies movies write
|
||||
```
|
||||
|
||||
## Jalankan ujian
|
||||
|
||||
```bash
|
||||
python -m pytest
|
||||
```
|
||||
|
||||
Ujian menggunakan halaman tersimpan dalam `tests/fixtures/` dan tidak pernah menyentuh rangkaian.
|
||||
|
||||
## Apa yang kita pelajari
|
||||
|
||||
- Persekitaran maya mengasingkan pakej projek daripada Python sistem.
|
||||
- `pyproject.toml` ialah satu tempat untuk versi Python, kebergantungan dan tetapan alat.
|
||||
- Ujian yang berjalan tanpa talian memberi keputusan yang sama pada setiap mesin.
|
||||
@@ -1,38 +0,0 @@
|
||||
**English** · [Dansk](Setup-da) · [Bahasa Melayu](Setup-ms)
|
||||
|
||||
# Setup
|
||||
|
||||
You need Python 3.13 or newer. Everything below is run from the repository root.
|
||||
|
||||
## Create the virtual environment
|
||||
|
||||
```powershell
|
||||
python -m venv .venv
|
||||
.venv\Scripts\Activate.ps1
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
On Linux and macOS use `python3 -m venv .venv` and `source .venv/bin/activate`.
|
||||
|
||||
## Run a lecture step
|
||||
|
||||
```bash
|
||||
python -m scrape_movies soup title
|
||||
python -m scrape_movies hacker-news top
|
||||
python -m scrape_movies movies write
|
||||
```
|
||||
|
||||
## Run the tests
|
||||
|
||||
```bash
|
||||
python -m pytest
|
||||
```
|
||||
|
||||
The tests use saved pages from `tests/fixtures/` and never touch the network.
|
||||
|
||||
## What we gained
|
||||
|
||||
- A virtual environment keeps the project's packages apart from the system Python.
|
||||
- `pyproject.toml` is the one place for the Python version, the dependencies and the tool settings.
|
||||
- Tests that run offline give the same result on every machine.
|
||||
@@ -1,59 +0,0 @@
|
||||
[English](Top-100-Movies) · **Dansk** · [Bahasa Melayu](Top-100-Movies-ms)
|
||||
|
||||
# Top 100 film
|
||||
|
||||
Forelæsning: *Projektet med de 100 film*. Målet er `movies.txt` med titlerne nummereret fra `1)` og opefter. Koden er `empire_movies.py` (parsing), `movie.py` (vend om og nummerér) og `movie_file.py` (skrivning).
|
||||
|
||||
## Trin 1: vælg siden
|
||||
|
||||
Opgaven fastlåser et øjebliksbillede fra Internet Archive af Empires liste, fordi levende websteder ændrer sig. Skærmbilledet viser sidens gemte overskrifter.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** et fastlåst øjebliksbillede gør en scraper gentagelig.
|
||||
|
||||
## Trin 2: undersøg markup
|
||||
|
||||
Siden fra 2020 har `<h3 class="title">`; den nuværende side har `<h2><strong>`-overskrifter, der begynder med en placering.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** når et websted ændrer markup, skal parseren følge med, så vi dækker begge opbygninger.
|
||||
|
||||
## Trin 3: titler i sidens rækkefølge
|
||||
|
||||
`select("h3.title")` og en list comprehension over `get_text()`. Siden tæller ned fra 100.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** en list comprehension er den kompakte måde at samle tekst fra mange tags.
|
||||
|
||||
## Trin 4: vend listen om
|
||||
|
||||
`titles[::-1]` får listen til at tælle op fra 1.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** udsnit med negativt trin vender en liste om i ét udtryk.
|
||||
|
||||
## Trin 5: skriv movies.txt
|
||||
|
||||
En `with open(...)`-blok skriver én titel pr. linje.
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** kontekstmanageren lukker filen for os, og den samme kørsel giver altid den samme fil.
|
||||
|
||||
## Trin 6: resultatet
|
||||
|
||||

|
||||
|
||||
**Hvad vi lærte:** en personlig film-seerliste fra et websted, klar til at vælge en film tilfældigt.
|
||||
|
||||
## Note om data
|
||||
|
||||
`movies.txt` blev lavet ud fra Empires nuværende liste den 2026-10-28, så placeringerne afviger fra listen fra 2020 i forelæsningen. Internet Archive svarede `429 Too Many Requests`, mens vi byggede projektet, så opbygningen af øjebliksbilledet er dækket af et håndskrevet eksempel i testene.
|
||||
|
||||
## Hvad vi lærte
|
||||
|
||||
Vi kan scrape en rangeret liste, lægge den i den rækkefølge vi vil have og gemme den i en fil.
|
||||
@@ -1,59 +0,0 @@
|
||||
[English](Top-100-Movies) · [Dansk](Top-100-Movies-da) · **Bahasa Melayu**
|
||||
|
||||
# 100 filem terbaik
|
||||
|
||||
Kuliah: *Projek 100 filem*. Matlamatnya ialah `movies.txt` dengan tajuk bernombor bermula dari `1)`. Kodnya ialah `empire_movies.py` (penghuraian), `movie.py` (terbalikkan dan nomborkan) dan `movie_file.py` (penulisan).
|
||||
|
||||
## Langkah 1: pilih halaman
|
||||
|
||||
Tugasan mengunci syot kilat Internet Archive bagi senarai Empire kerana laman langsung sentiasa berubah. Tangkapan skrin menunjukkan tajuk halaman yang disimpan.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** syot kilat yang dikunci menjadikan pengikis boleh diulang.
|
||||
|
||||
## Langkah 2: periksa markup
|
||||
|
||||
Halaman 2020 mempunyai `<h3 class="title">`; halaman semasa mempunyai tajuk `<h2><strong>` yang bermula dengan kedudukan.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** apabila laman mengubah markup, penghurai mesti berubah bersama, jadi kita menampung kedua-dua susunan.
|
||||
|
||||
## Langkah 3: tajuk mengikut susunan halaman
|
||||
|
||||
`select("h3.title")` dan list comprehension pada `get_text()`. Halaman mengira turun dari 100.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** list comprehension ialah cara ringkas mengumpul teks daripada banyak tag.
|
||||
|
||||
## Langkah 4: terbalikkan senarai
|
||||
|
||||
`titles[::-1]` menjadikan senarai mengira naik dari 1.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** hirisan dengan langkah negatif membalikkan senarai dalam satu ungkapan.
|
||||
|
||||
## Langkah 5: tulis movies.txt
|
||||
|
||||
Blok `with open(...)` menulis satu tajuk setiap baris.
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** pengurus konteks menutup fail untuk kita, dan larian yang sama sentiasa menghasilkan fail yang sama.
|
||||
|
||||
## Langkah 6: hasilnya
|
||||
|
||||

|
||||
|
||||
**Apa yang kita pelajari:** senarai tontonan peribadi daripada sebuah laman web, sedia untuk memilih filem secara rawak.
|
||||
|
||||
## Nota tentang data
|
||||
|
||||
`movies.txt` dihasilkan daripada senarai Empire semasa pada 2026-10-28, jadi kedudukannya berbeza daripada senarai 2020 dalam kuliah. Internet Archive menjawab `429 Too Many Requests` semasa kami membina projek ini, jadi susunan syot kilat dilindungi oleh contoh tulisan tangan dalam ujian.
|
||||
|
||||
## Apa yang kita pelajari
|
||||
|
||||
Kita boleh mengikis senarai berkedudukan, menyusunnya mengikut keperluan dan menyimpannya dalam fail.
|
||||
@@ -1,59 +0,0 @@
|
||||
**English** · [Dansk](Top-100-Movies-da) · [Bahasa Melayu](Top-100-Movies-ms)
|
||||
|
||||
# Top 100 movies
|
||||
|
||||
Lecture: *The 100 movies project*. The goal is `movies.txt` with the titles numbered from `1)` upwards. The code is `empire_movies.py` (parsing), `movie.py` (reverse and number) and `movie_file.py` (writing).
|
||||
|
||||
## Step 1: choose the page
|
||||
|
||||
The assignment pins an Internet Archive snapshot of Empire's list, because live sites change. The screenshot shows the saved headings of the page.
|
||||
|
||||

|
||||
|
||||
**What we gained:** a pinned snapshot makes a scraper repeatable.
|
||||
|
||||
## Step 2: inspect the markup
|
||||
|
||||
The 2020 page has `<h3 class="title">`; the current page has `<h2><strong>` headings that start with a rank.
|
||||
|
||||

|
||||
|
||||
**What we gained:** when a site changes its markup, the parser must change with it, so we keep both layouts covered.
|
||||
|
||||
## Step 3: titles in page order
|
||||
|
||||
`select("h3.title")` and a list comprehension over `get_text()`. The page counts down from 100.
|
||||
|
||||

|
||||
|
||||
**What we gained:** a list comprehension is the compact way to collect text from many tags.
|
||||
|
||||
## Step 4: reverse the list
|
||||
|
||||
`titles[::-1]` makes the list count up from 1.
|
||||
|
||||

|
||||
|
||||
**What we gained:** slicing with a negative step reverses a list in one expression.
|
||||
|
||||
## Step 5: write movies.txt
|
||||
|
||||
A `with open(...)` block writes one title per line.
|
||||
|
||||

|
||||
|
||||
**What we gained:** the context manager closes the file for us, and the same run always gives the same file.
|
||||
|
||||
## Step 6: the result
|
||||
|
||||

|
||||
|
||||
**What we gained:** a personal watch list from a website, ready to pick a film from at random.
|
||||
|
||||
## Note on the data
|
||||
|
||||
`movies.txt` was produced from the current Empire list on 2026-10-28, so its ranking differs from the 2020 list in the lecture. The Internet Archive answered `429 Too Many Requests` while we built this, so the snapshot layout is covered by a hand-written sample in the tests.
|
||||
|
||||
## What we gained
|
||||
|
||||
We can scrape a ranked list, put it in the order we want and save it to a file.
|
||||
@@ -1 +0,0 @@
|
||||
scrape-movies · Day 45 of 100 Days of Code · [Repository](https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/045-scrape-movies) · English · Dansk · Bahasa Melayu
|
||||
@@ -1,29 +0,0 @@
|
||||
**English** · [Dansk](Home-da) · [Bahasa Melayu](Home-ms)
|
||||
|
||||
- [Home](Home)
|
||||
- [Setup](Setup)
|
||||
- [Parsing HTML](Parsing-HTML)
|
||||
- [Hacker News](Hacker-News)
|
||||
- [Scraping ethics](Scraping-Ethics)
|
||||
- [Top 100 movies](Top-100-Movies)
|
||||
- [What we gained](Gains)
|
||||
|
||||
**Dansk**
|
||||
|
||||
- [Hjem](Home-da)
|
||||
- [Opsætning](Setup-da)
|
||||
- [Parsing af HTML](Parsing-HTML-da)
|
||||
- [Hacker News](Hacker-News-da)
|
||||
- [Etik ved scraping](Scraping-Ethics-da)
|
||||
- [Top 100 film](Top-100-Movies-da)
|
||||
- [Hvad vi lærte](Gains-da)
|
||||
|
||||
**Bahasa Melayu**
|
||||
|
||||
- [Utama](Home-ms)
|
||||
- [Persediaan](Setup-ms)
|
||||
- [Menghurai HTML](Parsing-HTML-ms)
|
||||
- [Hacker News](Hacker-News-ms)
|
||||
- [Etika mengikis web](Scraping-Ethics-ms)
|
||||
- [100 filem terbaik](Top-100-Movies-ms)
|
||||
- [Apa yang kita pelajari](Gains-ms)
|
||||
@@ -1,100 +0,0 @@
|
||||
1) Jaws (1975)
|
||||
2) The Godfather (1972)
|
||||
3) The Shawshank Redemption (1994)
|
||||
4) Star Wars: The Empire Strikes Back (1980)
|
||||
5) Aliens (1986)
|
||||
6) Inception (2010)
|
||||
7) Goodfellas (1990)
|
||||
8) Star Wars: Episode IV – A New Hope (1977)
|
||||
9) The Lord Of The Rings: The Fellowship Of The Ring (2001)
|
||||
10) The Dark Knight (2008)
|
||||
11) Pulp Fiction (1994)
|
||||
12) Back To The Future (1985)
|
||||
13) Jurassic Park (1993)
|
||||
14) Raiders Of The Lost Ark (1981)
|
||||
15) Blade Runner (1982)
|
||||
16) The Godfather Part II (1974)
|
||||
17) Alien (1979)
|
||||
18) Terminator 2: Judgment Day (1991)
|
||||
19) Interstellar (2014)
|
||||
20) Heat (1995)
|
||||
21) The Lord Of The Rings: The Return Of The King (2003)
|
||||
22) The Matrix (1999)
|
||||
23) Die Hard (1988)
|
||||
24) There Will Be Blood (2007)
|
||||
25) 2001: A Space Odyssey (1968)
|
||||
26) Gladiator (2000)
|
||||
27) Parasite (2019)
|
||||
28) Apocalypse Now (1979)
|
||||
29) 12 Angry Men (1957)
|
||||
30) Casablanca (1942)
|
||||
31) The Thing (1982)
|
||||
32) Seven (1995)
|
||||
33) The Lord Of The Rings: The Two Towers (2002)
|
||||
34) The Shining (1980)
|
||||
35) Mad Max: Fury Road (2015)
|
||||
36) Point Break (1991)
|
||||
37) Fight Club (1999)
|
||||
38) The Good, The Bad And The Ugly (1966)
|
||||
39) Lawrence Of Arabia (1962)
|
||||
40) Mulholland Drive (2001)
|
||||
41) Schindler's List (1993)
|
||||
42) Saving Private Ryan (1998)
|
||||
43) La La Land (2016)
|
||||
44) Taxi Driver (1976)
|
||||
45) Past Lives (2023)
|
||||
46) Whiplash (2014)
|
||||
47) It's A Wonderful Life (1946)
|
||||
48) Avengers: Infinity War (2018)
|
||||
49) Psycho (1960)
|
||||
50) No Country For Old Men (2007)
|
||||
51) Rear Window (1954)
|
||||
52) Citizen Kane (1941)
|
||||
53) Vertigo (1958)
|
||||
54) Get Out (2017)
|
||||
55) Hot Fuzz (2007)
|
||||
56) Spirited Away (2001)
|
||||
57) Seven Samurai (1954)
|
||||
58) Titanic (1997)
|
||||
59) The Exorcist (1973)
|
||||
60) Fargo (1996)
|
||||
61) The Social Network (2010)
|
||||
62) Eternal Sunshine Of The Spotless Mind (2004)
|
||||
63) The Big Lebowski (1998)
|
||||
64) Oppenheimer (2023)
|
||||
65) Predator (1987)
|
||||
66) Indiana Jones And The Last Crusade (1989)
|
||||
67) Do The Right Thing (1989)
|
||||
68) Avengers: Endgame (2019)
|
||||
69) The Babadook (2014)
|
||||
70) Children Of Men (2006)
|
||||
71) Arrival (2016)
|
||||
72) Magnolia (1999)
|
||||
73) The Lion King (1994)
|
||||
74) Trainspotting (1996)
|
||||
75) The Silence Of The Lambs (1991)
|
||||
76) Once Upon A Time In Hollywood (2019)
|
||||
77) In The Mood For Love (2000)
|
||||
78) Ghostbusters (1984)
|
||||
79) Paddington 2 (2017)
|
||||
80) Spider-Man: Into The Spider-Verse (2018)
|
||||
81) Everything Everywhere All At Once (2022)
|
||||
82) Brokeback Mountain (2005)
|
||||
83) Inglourious Basterds (2009)
|
||||
84) Pan's Labyrinth (2006)
|
||||
85) Rocky (1976)
|
||||
86) Lady Bird (2017)
|
||||
87) Raging Bull (1980)
|
||||
88) The Piano (1993)
|
||||
89) American Psycho (2000)
|
||||
90) Groundhog Day (1993)
|
||||
91) The Grand Budapest Hotel (2014)
|
||||
92) One Flew Over The Cuckoo's Nest (1975)
|
||||
93) Lost In Translation (2003)
|
||||
94) Moonlight (2016)
|
||||
95) Memento (2000)
|
||||
96) E.T. The Extra Terrestrial (1982)
|
||||
97) Barbie (2023)
|
||||
98) When Harry Met Sally... (1989)
|
||||
99) Portrait Of A Lady On Fire (2020)
|
||||
100) Avatar (2009)
|
||||
@@ -1,54 +0,0 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "scrape-movies"
|
||||
version = "0.1.0"
|
||||
description = "Day 45 of 100 Days of Code: web scraping with Requests and BeautifulSoup (Hacker News and the Empire 100 greatest movies)."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.13"
|
||||
license = { file = "LICENSE" }
|
||||
dependencies = [
|
||||
"requests>=2.32",
|
||||
"beautifulsoup4>=4.12",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.3",
|
||||
"pytest-cov>=5.0",
|
||||
"ruff>=0.6",
|
||||
"mypy>=1.11",
|
||||
"types-requests>=2.32",
|
||||
"pillow>=10.4",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["src"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 88
|
||||
src = ["src", "tests"]
|
||||
extend-exclude = ["framework", ".agents", ".claude"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "N", "UP", "B", "D"]
|
||||
# UP040 off: mypy 1.11 (the pinned CI version) does not support PEP 695 aliases.
|
||||
ignore = ["D203", "D213", "UP040"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"tests/**" = ["D"]
|
||||
|
||||
[tool.mypy]
|
||||
strict = true
|
||||
python_version = "3.13"
|
||||
mypy_path = "src"
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["bs4.*", "requests.*", "PIL.*", "pytest.*"]
|
||||
ignore_missing_imports = true
|
||||
@@ -1 +0,0 @@
|
||||
"""@brief Day 45 scraping exercises: soup basics, Hacker News and 100 movies."""
|
||||
@@ -1,7 +0,0 @@
|
||||
"""@brief Allow `python -m scrape_movies`."""
|
||||
|
||||
import sys
|
||||
|
||||
from scrape_movies.presentation.cli import main
|
||||
|
||||
sys.exit(main())
|
||||
@@ -1 +0,0 @@
|
||||
"""@brief Use cases and the ports they need; imports only the domain."""
|
||||
@@ -1,16 +0,0 @@
|
||||
"""@brief The ports (interfaces) the use cases need from the outside world."""
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from pathlib import Path
|
||||
from typing import TypeAlias
|
||||
|
||||
from scrape_movies.domain.story import Story
|
||||
|
||||
## Fetches the HTML text of a page address.
|
||||
HtmlFetcher: TypeAlias = Callable[[str], str]
|
||||
## Turns Hacker News HTML into stories.
|
||||
StoryParser: TypeAlias = Callable[[str], list[Story]]
|
||||
## Turns the Empire page HTML into the movie lines found on it, in page order.
|
||||
MovieParser: TypeAlias = Callable[[str], list[str]]
|
||||
## Writes lines to a file, one per line.
|
||||
MovieWriter: TypeAlias = Callable[[Sequence[str], Path], None]
|
||||
@@ -1,67 +0,0 @@
|
||||
"""@brief Use cases of the scraping exercises."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
from scrape_movies.application.ports import (
|
||||
HtmlFetcher,
|
||||
MovieParser,
|
||||
MovieWriter,
|
||||
StoryParser,
|
||||
)
|
||||
from scrape_movies.domain.movie import format_titles
|
||||
from scrape_movies.domain.story import Story, top_story
|
||||
|
||||
|
||||
def list_stories(fetch: HtmlFetcher, parse: StoryParser, url: str) -> list[Story]:
|
||||
"""@brief Fetch a Hacker News page and read its stories.
|
||||
|
||||
@param fetch Fetches the HTML of the page.
|
||||
@param parse Reads the stories from the HTML.
|
||||
@param url The page address.
|
||||
@return The stories in page order.
|
||||
"""
|
||||
return parse(fetch(url))
|
||||
|
||||
|
||||
def find_top_story(fetch: HtmlFetcher, parse: StoryParser, url: str) -> Story | None:
|
||||
"""@brief Find the story with the most upvotes on a Hacker News page.
|
||||
|
||||
@param fetch Fetches the HTML of the page.
|
||||
@param parse Reads the stories from the HTML.
|
||||
@param url The page address.
|
||||
@return The top story, or None when the page has no stories.
|
||||
"""
|
||||
return top_story(list_stories(fetch, parse, url))
|
||||
|
||||
|
||||
def scrape_movie_titles(fetch: HtmlFetcher, parse: MovieParser, url: str) -> list[str]:
|
||||
"""@brief Fetch the Empire page and read the titles in page order.
|
||||
|
||||
@param fetch Fetches the HTML of the page.
|
||||
@param parse Reads the titles from the HTML.
|
||||
@param url The page address.
|
||||
@return The titles in page order (100 down to 1).
|
||||
"""
|
||||
return parse(fetch(url))
|
||||
|
||||
|
||||
def save_movie_list(
|
||||
fetch: HtmlFetcher,
|
||||
parse: MovieParser,
|
||||
write: MovieWriter,
|
||||
url: str,
|
||||
path: Path,
|
||||
) -> Sequence[str]:
|
||||
"""@brief Scrape the list and write it, ascending and numbered, to a file.
|
||||
|
||||
@param fetch Fetches the HTML of the page.
|
||||
@param parse Reads the titles from the HTML.
|
||||
@param write Writes the lines to the file.
|
||||
@param url The page address.
|
||||
@param path The file to write, for example `movies.txt`.
|
||||
@return The lines that were written.
|
||||
"""
|
||||
lines = format_titles(scrape_movie_titles(fetch, parse, url))
|
||||
write(lines, path)
|
||||
return lines
|
||||
@@ -1,40 +0,0 @@
|
||||
"""@brief All constants of the project (URLs, limits, paths, parser name)."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
## Root folder of the repository (two levels above this file's package folder).
|
||||
PROJECT_ROOT: Path = Path(__file__).resolve().parents[2]
|
||||
## Folder with the saved pages the tests and the lecture steps read.
|
||||
FIXTURES_DIR: Path = PROJECT_ROOT / "tests" / "fixtures"
|
||||
## The sample page from the "Parsing HTML and Making Soup" lecture.
|
||||
WEBSITE_HTML_PATH: Path = FIXTURES_DIR / "website.html"
|
||||
|
||||
## Parser handed to BeautifulSoup; it is part of the standard library.
|
||||
HTML_PARSER: str = "html.parser"
|
||||
|
||||
## Seconds to wait for a web server before giving up.
|
||||
REQUEST_TIMEOUT_SECONDS: int = 10
|
||||
## Sent with every request so a site owner can see who asks.
|
||||
USER_AGENT: str = "scrape-movies/0.1 (Udemy 100 Days of Code, Day 45)"
|
||||
|
||||
## The course's static copy of the Hacker News front page.
|
||||
HACKER_NEWS_URL: str = "https://appbrewery.github.io/news.ycombinator.com/"
|
||||
## The live Hacker News front page (scrape it at most once a minute).
|
||||
HACKER_NEWS_LIVE_URL: str = "https://news.ycombinator.com/news"
|
||||
|
||||
## Internet Archive snapshot of the Empire list, as the assignment requires.
|
||||
EMPIRE_WAYBACK_URL: str = (
|
||||
"https://web.archive.org/web/20200518073855/"
|
||||
"https://www.empireonline.com/movies/features/best-movies-2/"
|
||||
)
|
||||
## The live Empire page, an alternative that may have changed its markup.
|
||||
EMPIRE_LIVE_URL: str = "https://www.empireonline.com/movies/features/best-movies-2/"
|
||||
|
||||
## File the movie list is written to.
|
||||
MOVIES_OUTPUT_FILE: str = "movies.txt"
|
||||
|
||||
## The saved copy of the course's Hacker News page, used offline.
|
||||
HACKER_NEWS_FIXTURE_PATH: Path = FIXTURES_DIR / "hacker_news.html"
|
||||
|
||||
## The saved headings of the live Empire page, used offline.
|
||||
EMPIRE_FIXTURE_PATH: Path = FIXTURES_DIR / "empire_live_excerpt.html"
|
||||
@@ -1 +0,0 @@
|
||||
"""@brief Business rules and entities; imports nothing from other layers."""
|
||||
@@ -1,5 +0,0 @@
|
||||
"""@brief Errors the project raises."""
|
||||
|
||||
|
||||
class FetchError(Exception):
|
||||
"""@brief A web page could not be fetched."""
|
||||
@@ -1,15 +0,0 @@
|
||||
"""@brief The Link value object: the text and the address of an anchor."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Link:
|
||||
"""@brief A hyperlink found on a page.
|
||||
|
||||
@param text The visible text of the anchor.
|
||||
@param url The value of its `href` attribute.
|
||||
"""
|
||||
|
||||
text: str
|
||||
url: str
|
||||
@@ -1,22 +0,0 @@
|
||||
"""@brief The movie list rule: ascending order, every line starting with its rank."""
|
||||
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
|
||||
_RANK_PREFIX = re.compile(r"^\d+\) ")
|
||||
|
||||
|
||||
def format_titles(titles: Sequence[str]) -> list[str]:
|
||||
"""@brief Turn a descending page list into an ascending, numbered list.
|
||||
|
||||
Empire lists the movies from 100 down to 1, so the page order is reversed.
|
||||
A title that already starts with its rank, such as `3) Jaws`, is kept as
|
||||
it is; any other title gets `<rank>) ` in front of it.
|
||||
|
||||
@param titles The titles in page order.
|
||||
@return The lines `1) ...`, `2) ...` in ascending order.
|
||||
"""
|
||||
lines: list[str] = []
|
||||
for rank, title in enumerate(reversed(titles), 1):
|
||||
lines.append(title if _RANK_PREFIX.match(title) else f"{rank}) {title}")
|
||||
return lines
|
||||
@@ -1,31 +0,0 @@
|
||||
"""@brief The Story entity (a Hacker News article) and the top-story rule."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Story:
|
||||
"""@brief An article on the Hacker News front page.
|
||||
|
||||
@param title The headline.
|
||||
@param link The address the headline points to.
|
||||
@param upvotes The number of points; 0 when the story shows none.
|
||||
"""
|
||||
|
||||
title: str
|
||||
link: str
|
||||
upvotes: int
|
||||
|
||||
|
||||
def top_story(stories: Sequence[Story]) -> Story | None:
|
||||
"""@brief Find the story with the most upvotes.
|
||||
|
||||
@param stories The stories in page order.
|
||||
@return The first story with the highest score, or None when there are none.
|
||||
"""
|
||||
best: Story | None = None
|
||||
for story in stories:
|
||||
if best is None or story.upvotes > best.upvotes:
|
||||
best = story
|
||||
return best
|
||||
@@ -1 +0,0 @@
|
||||
"""@brief Adapters for the web, BeautifulSoup and files; implements the ports."""
|
||||
@@ -1,55 +0,0 @@
|
||||
"""@brief Reading the movie titles from the Empire "100 Greatest Movies" page."""
|
||||
|
||||
import re
|
||||
|
||||
from bs4 import Tag
|
||||
|
||||
from scrape_movies.infrastructure.soup_basics import make_soup
|
||||
|
||||
## The 2020 page (the assignment's snapshot): `<h3 class="title">100) Title</h3>`.
|
||||
SNAPSHOT_TITLE = "h3.title"
|
||||
## The current page: `<h2><strong>100) Title (year)</strong></h2>`.
|
||||
_RANKED_HEADING = re.compile(r"^\s*\d{1,3}\)")
|
||||
|
||||
|
||||
def parse_titles(html: str) -> list[str]:
|
||||
"""@brief Read the movie titles in page order.
|
||||
|
||||
The assignment's snapshot is read through `h3.title`. When the page has no
|
||||
such tag, as on the current live page, the `h2` headings that start with a
|
||||
rank such as `100)` are used.
|
||||
|
||||
@param html The HTML of the page.
|
||||
@return The titles in page order, each as the page shows it.
|
||||
"""
|
||||
soup = make_soup(html)
|
||||
titles = [tag.get_text().strip() for tag in soup.select(SNAPSHOT_TITLE)]
|
||||
if titles:
|
||||
return titles
|
||||
headings = [tag for tag in soup.find_all("h2") if isinstance(tag, Tag)]
|
||||
return [
|
||||
text
|
||||
for text in (heading.get_text().strip() for heading in headings)
|
||||
if _RANKED_HEADING.match(text)
|
||||
]
|
||||
|
||||
|
||||
def first_title_markup(html: str) -> str:
|
||||
"""@brief Show the markup of the first title, as seen with "Inspect".
|
||||
|
||||
@param html The HTML of the page.
|
||||
@return The indented HTML of the first title tag, or "" when there is none.
|
||||
"""
|
||||
soup = make_soup(html)
|
||||
tag = soup.select_one(SNAPSHOT_TITLE)
|
||||
if tag is None:
|
||||
tag = next(
|
||||
(
|
||||
heading
|
||||
for heading in soup.find_all("h2")
|
||||
if isinstance(heading, Tag)
|
||||
and _RANKED_HEADING.match(heading.get_text())
|
||||
),
|
||||
None,
|
||||
)
|
||||
return str(tag.prettify()).rstrip() if tag is not None else ""
|
||||
@@ -1,81 +0,0 @@
|
||||
"""@brief Reading titles, links and upvotes from a Hacker News page."""
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from scrape_movies.domain.story import Story
|
||||
from scrape_movies.infrastructure.soup_basics import make_soup
|
||||
|
||||
STORY_ROW = "tr.athing"
|
||||
TITLE_LINK = ".titleline > a"
|
||||
SCORE = ".score"
|
||||
|
||||
|
||||
def _upvotes(row: Tag) -> int:
|
||||
"""@brief Read the score shown in the row after a story, 0 when it has none."""
|
||||
following = row.find_next_sibling("tr")
|
||||
score = following.select_one(SCORE) if isinstance(following, Tag) else None
|
||||
if score is None:
|
||||
return 0
|
||||
return int(score.get_text().split()[0])
|
||||
|
||||
|
||||
def _story_rows(soup: BeautifulSoup) -> list[Tag]:
|
||||
return [row for row in soup.select(STORY_ROW) if row.select_one(TITLE_LINK)]
|
||||
|
||||
|
||||
def parse_stories(html: str) -> list[Story]:
|
||||
"""@brief Read every story of a front page.
|
||||
|
||||
@param html The HTML of the page.
|
||||
@return The stories in page order.
|
||||
"""
|
||||
stories: list[Story] = []
|
||||
for row in _story_rows(make_soup(html)):
|
||||
anchor = row.select_one(TITLE_LINK)
|
||||
href = anchor.get("href") if anchor is not None else None
|
||||
if anchor is not None and isinstance(href, str):
|
||||
stories.append(
|
||||
Story(title=anchor.get_text(), link=href, upvotes=_upvotes(row))
|
||||
)
|
||||
return stories
|
||||
|
||||
|
||||
def get_titles(html: str) -> list[str]:
|
||||
"""@brief Return the headlines of a front page.
|
||||
|
||||
@param html The HTML of the page.
|
||||
@return The titles in page order.
|
||||
"""
|
||||
return [story.title for story in parse_stories(html)]
|
||||
|
||||
|
||||
def get_links(html: str) -> list[str]:
|
||||
"""@brief Return the addresses the headlines point to.
|
||||
|
||||
@param html The HTML of the page.
|
||||
@return The links in page order.
|
||||
"""
|
||||
return [story.link for story in parse_stories(html)]
|
||||
|
||||
|
||||
def get_upvotes(html: str) -> list[int]:
|
||||
"""@brief Return the scores as numbers, not text such as "38 points".
|
||||
|
||||
@param html The HTML of the page.
|
||||
@return The upvotes in page order.
|
||||
"""
|
||||
return [story.upvotes for story in parse_stories(html)]
|
||||
|
||||
|
||||
def first_story_markup(html: str) -> str:
|
||||
"""@brief Show the markup of the first story, as seen with "Inspect" in a browser.
|
||||
|
||||
@param html The HTML of the page.
|
||||
@return The indented HTML of its title row and score row, or "" when none.
|
||||
"""
|
||||
rows = _story_rows(make_soup(html))
|
||||
if not rows:
|
||||
return ""
|
||||
following = rows[0].find_next_sibling("tr")
|
||||
parts = [rows[0], following] if isinstance(following, Tag) else [rows[0]]
|
||||
return "\n".join(str(part.prettify()) for part in parts)
|
||||
@@ -1,28 +0,0 @@
|
||||
"""@brief Fetching web pages with Requests."""
|
||||
|
||||
import requests
|
||||
|
||||
from scrape_movies.constants import REQUEST_TIMEOUT_SECONDS, USER_AGENT
|
||||
from scrape_movies.domain.errors import FetchError
|
||||
|
||||
|
||||
def fetch_html(url: str) -> str:
|
||||
"""@brief Download a page with one GET request.
|
||||
|
||||
Sends a User-Agent and gives up after a timeout. Call it once per run:
|
||||
scraping more often than once a minute is not polite.
|
||||
|
||||
@param url The address of the page.
|
||||
@return The HTML text of the response.
|
||||
@throws FetchError When the request fails or the server answers with an error.
|
||||
"""
|
||||
try:
|
||||
response = requests.get(
|
||||
url,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
timeout=REQUEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except requests.RequestException as error:
|
||||
raise FetchError(f"Could not fetch {url}: {error}") from error
|
||||
return str(response.text)
|
||||
@@ -1,15 +0,0 @@
|
||||
"""@brief Writing the movie list to a text file."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def write_movies(lines: Sequence[str], path: Path) -> None:
|
||||
"""@brief Write one line per movie, UTF-8, with a final newline.
|
||||
|
||||
@param lines The lines to write.
|
||||
@param path The file to create or overwrite.
|
||||
"""
|
||||
with path.open("w", encoding="utf-8", newline="\n") as file:
|
||||
for line in lines:
|
||||
file.write(f"{line}\n")
|
||||
@@ -1,107 +0,0 @@
|
||||
"""@brief BeautifulSoup basics from the "Parsing HTML and Making Soup" lectures."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from scrape_movies.constants import HTML_PARSER
|
||||
from scrape_movies.domain.link import Link
|
||||
|
||||
|
||||
def load_html(path: Path) -> str:
|
||||
"""@brief Read an HTML file as text.
|
||||
|
||||
@param path The file to read.
|
||||
@return The content of the file.
|
||||
"""
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def make_soup(html: str) -> BeautifulSoup:
|
||||
"""@brief Parse HTML text into a soup object.
|
||||
|
||||
@param html The HTML document as text.
|
||||
@return The parsed document.
|
||||
"""
|
||||
return BeautifulSoup(html, HTML_PARSER)
|
||||
|
||||
|
||||
def get_title(soup: BeautifulSoup) -> str | None:
|
||||
"""@brief Return the text of the `title` tag.
|
||||
|
||||
@param soup The parsed document.
|
||||
@return The title text, or None when the page has no title.
|
||||
"""
|
||||
if soup.title is None or soup.title.string is None:
|
||||
return None
|
||||
return str(soup.title.string)
|
||||
|
||||
|
||||
def prettify_html(soup: BeautifulSoup) -> str:
|
||||
"""@brief Return the document indented one tag per line.
|
||||
|
||||
@param soup The parsed document.
|
||||
@return The indented HTML.
|
||||
"""
|
||||
return str(soup.prettify())
|
||||
|
||||
|
||||
def find_all_anchors(soup: BeautifulSoup) -> list[Link]:
|
||||
"""@brief Find every anchor tag with its text and address (`find_all`).
|
||||
|
||||
@param soup The parsed document.
|
||||
@return One Link per anchor that has an `href` attribute.
|
||||
"""
|
||||
links: list[Link] = []
|
||||
for anchor in soup.find_all("a"):
|
||||
url = anchor.get("href")
|
||||
if isinstance(url, str):
|
||||
links.append(Link(text=str(anchor.get_text()), url=url))
|
||||
return links
|
||||
|
||||
|
||||
def find_text_by_id(soup: BeautifulSoup, element_id: str) -> str | None:
|
||||
"""@brief Return the text of the first tag with the given id (`find`).
|
||||
|
||||
@param soup The parsed document.
|
||||
@param element_id The value of the `id` attribute.
|
||||
@return The text, or None when no tag has that id.
|
||||
"""
|
||||
tag = soup.find(id=element_id)
|
||||
return str(tag.get_text()) if isinstance(tag, Tag) else None
|
||||
|
||||
|
||||
def find_all_texts_by_class(
|
||||
soup: BeautifulSoup, tag_name: str, class_name: str
|
||||
) -> list[str]:
|
||||
"""@brief Return the text of every tag with the given class (`find_all`).
|
||||
|
||||
The keyword is `class_` because `class` is a reserved word in Python.
|
||||
|
||||
@param soup The parsed document.
|
||||
@param tag_name The tag to look for, for example `h3`.
|
||||
@param class_name The CSS class the tag must have.
|
||||
@return The texts in page order.
|
||||
"""
|
||||
return [str(tag.get_text()) for tag in soup.find_all(tag_name, class_=class_name)]
|
||||
|
||||
|
||||
def select_one_text(soup: BeautifulSoup, selector: str) -> str | None:
|
||||
"""@brief Return the text of the first tag matching a CSS selector.
|
||||
|
||||
@param soup The parsed document.
|
||||
@param selector A CSS selector such as `p a` or `#name`.
|
||||
@return The text, or None when nothing matches.
|
||||
"""
|
||||
tag = soup.select_one(selector)
|
||||
return str(tag.get_text()) if tag is not None else None
|
||||
|
||||
|
||||
def select_texts(soup: BeautifulSoup, selector: str) -> list[str]:
|
||||
"""@brief Return the text of every tag matching a CSS selector.
|
||||
|
||||
@param soup The parsed document.
|
||||
@param selector A CSS selector such as `.heading`.
|
||||
@return The texts in page order.
|
||||
"""
|
||||
return [str(tag.get_text()) for tag in soup.select(selector)]
|
||||
@@ -1 +0,0 @@
|
||||
"""@brief Command-line entry point; calls the use cases."""
|
||||