MIL-001: repository setup and tooling #35
+36
@@ -0,0 +1,36 @@
|
|||||||
|
# Secrets: personal tokens are never part of the project
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
|
||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*.egg-info/
|
||||||
|
build/
|
||||||
|
dist/
|
||||||
|
.eggs/
|
||||||
|
|
||||||
|
# Virtual environments
|
||||||
|
.venv/
|
||||||
|
venv/
|
||||||
|
env/
|
||||||
|
|
||||||
|
# Test, type and lint caches
|
||||||
|
.pytest_cache/
|
||||||
|
.mypy_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
.coverage
|
||||||
|
coverage.xml
|
||||||
|
htmlcov/
|
||||||
|
|
||||||
|
# Doxygen output
|
||||||
|
docs/api/
|
||||||
|
|
||||||
|
# Editors and operating systems
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# AI prompt log (public repository)
|
||||||
|
/docs/ai-prompts/
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=68"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "scrape-movies"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Day 45 of 100 Days of Code: web scraping with Requests and BeautifulSoup (Hacker News and the Empire 100 greatest movies)."
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.13"
|
||||||
|
license = { file = "LICENSE" }
|
||||||
|
dependencies = [
|
||||||
|
"requests>=2.32",
|
||||||
|
"beautifulsoup4>=4.12",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
dev = [
|
||||||
|
"pytest>=8.3",
|
||||||
|
"pytest-cov>=5.0",
|
||||||
|
"ruff>=0.6",
|
||||||
|
"mypy>=1.11",
|
||||||
|
"types-requests>=2.32",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.setuptools.packages.find]
|
||||||
|
where = ["src"]
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
pythonpath = ["src"]
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 88
|
||||||
|
src = ["src", "tests"]
|
||||||
|
extend-exclude = ["framework", ".agents", ".claude"]
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
select = ["E", "F", "I", "N", "UP", "B", "D"]
|
||||||
|
ignore = ["D203", "D213"]
|
||||||
|
|
||||||
|
[tool.ruff.lint.per-file-ignores]
|
||||||
|
"tests/**" = ["D"]
|
||||||
|
|
||||||
|
[tool.mypy]
|
||||||
|
strict = true
|
||||||
|
python_version = "3.13"
|
||||||
|
mypy_path = "src"
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Day 45 scraping exercises: soup basics, Hacker News and 100 movies."""
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Use cases and the ports they need; imports only the domain."""
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
"""@brief All constants of the project (URLs, limits, paths, parser name)."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
## Root folder of the repository (two levels above this file's package folder).
|
||||||
|
PROJECT_ROOT: Path = Path(__file__).resolve().parents[2]
|
||||||
|
## Folder with the saved pages the tests and the lecture steps read.
|
||||||
|
FIXTURES_DIR: Path = PROJECT_ROOT / "tests" / "fixtures"
|
||||||
|
## The sample page from the "Parsing HTML and Making Soup" lecture.
|
||||||
|
WEBSITE_HTML_PATH: Path = FIXTURES_DIR / "website.html"
|
||||||
|
|
||||||
|
## Parser handed to BeautifulSoup; it is part of the standard library.
|
||||||
|
HTML_PARSER: str = "html.parser"
|
||||||
|
|
||||||
|
## Seconds to wait for a web server before giving up.
|
||||||
|
REQUEST_TIMEOUT_SECONDS: int = 10
|
||||||
|
## Sent with every request so a site owner can see who asks.
|
||||||
|
USER_AGENT: str = "scrape-movies/0.1 (Udemy 100 Days of Code, Day 45)"
|
||||||
|
|
||||||
|
## The course's static copy of the Hacker News front page.
|
||||||
|
HACKER_NEWS_URL: str = "https://appbrewery.github.io/news.ycombinator.com/"
|
||||||
|
## The live Hacker News front page (scrape it at most once a minute).
|
||||||
|
HACKER_NEWS_LIVE_URL: str = "https://news.ycombinator.com/news"
|
||||||
|
|
||||||
|
## Internet Archive snapshot of the Empire list, as the assignment requires.
|
||||||
|
EMPIRE_WAYBACK_URL: str = (
|
||||||
|
"https://web.archive.org/web/20200518073855/"
|
||||||
|
"https://www.empireonline.com/movies/features/best-movies-2/"
|
||||||
|
)
|
||||||
|
## The live Empire page, an alternative that may have changed its markup.
|
||||||
|
EMPIRE_LIVE_URL: str = "https://www.empireonline.com/movies/features/best-movies-2/"
|
||||||
|
|
||||||
|
## File the movie list is written to.
|
||||||
|
MOVIES_OUTPUT_FILE: str = "movies.txt"
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Business rules and entities; imports nothing from other layers."""
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Adapters for the web, BeautifulSoup and files; implements the ports."""
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""@brief Command-line entry point; calls the use cases."""
|
||||||
Reference in New Issue
Block a user