MIL-001: repository setup and tooling #35
+36
@@ -0,0 +1,36 @@
|
||||
# Secrets: personal tokens are never part of the project
|
||||
.env
|
||||
.env.*
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
build/
|
||||
dist/
|
||||
.eggs/
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
venv/
|
||||
env/
|
||||
|
||||
# Test, type and lint caches
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
|
||||
# Doxygen output
|
||||
docs/api/
|
||||
|
||||
# Editors and operating systems
|
||||
.idea/
|
||||
.vscode/
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# AI prompt log (public repository)
|
||||
/docs/ai-prompts/
|
||||
@@ -0,0 +1,48 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "scrape-movies"
|
||||
version = "0.1.0"
|
||||
description = "Day 45 of 100 Days of Code: web scraping with Requests and BeautifulSoup (Hacker News and the Empire 100 greatest movies)."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.13"
|
||||
license = { file = "LICENSE" }
|
||||
dependencies = [
|
||||
"requests>=2.32",
|
||||
"beautifulsoup4>=4.12",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.3",
|
||||
"pytest-cov>=5.0",
|
||||
"ruff>=0.6",
|
||||
"mypy>=1.11",
|
||||
"types-requests>=2.32",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["src"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 88
|
||||
src = ["src", "tests"]
|
||||
extend-exclude = ["framework", ".agents", ".claude"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "N", "UP", "B", "D"]
|
||||
ignore = ["D203", "D213"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"tests/**" = ["D"]
|
||||
|
||||
[tool.mypy]
|
||||
strict = true
|
||||
python_version = "3.13"
|
||||
mypy_path = "src"
|
||||
@@ -0,0 +1 @@
|
||||
"""@brief Day 45 scraping exercises: soup basics, Hacker News and 100 movies."""
|
||||
@@ -0,0 +1 @@
|
||||
"""@brief Use cases and the ports they need; imports only the domain."""
|
||||
@@ -0,0 +1,34 @@
|
||||
"""@brief All constants of the project (URLs, limits, paths, parser name)."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
## Root folder of the repository (two levels above this file's package folder).
|
||||
PROJECT_ROOT: Path = Path(__file__).resolve().parents[2]
|
||||
## Folder with the saved pages the tests and the lecture steps read.
|
||||
FIXTURES_DIR: Path = PROJECT_ROOT / "tests" / "fixtures"
|
||||
## The sample page from the "Parsing HTML and Making Soup" lecture.
|
||||
WEBSITE_HTML_PATH: Path = FIXTURES_DIR / "website.html"
|
||||
|
||||
## Parser handed to BeautifulSoup; it is part of the standard library.
|
||||
HTML_PARSER: str = "html.parser"
|
||||
|
||||
## Seconds to wait for a web server before giving up.
|
||||
REQUEST_TIMEOUT_SECONDS: int = 10
|
||||
## Sent with every request so a site owner can see who asks.
|
||||
USER_AGENT: str = "scrape-movies/0.1 (Udemy 100 Days of Code, Day 45)"
|
||||
|
||||
## The course's static copy of the Hacker News front page.
|
||||
HACKER_NEWS_URL: str = "https://appbrewery.github.io/news.ycombinator.com/"
|
||||
## The live Hacker News front page (scrape it at most once a minute).
|
||||
HACKER_NEWS_LIVE_URL: str = "https://news.ycombinator.com/news"
|
||||
|
||||
## Internet Archive snapshot of the Empire list, as the assignment requires.
|
||||
EMPIRE_WAYBACK_URL: str = (
|
||||
"https://web.archive.org/web/20200518073855/"
|
||||
"https://www.empireonline.com/movies/features/best-movies-2/"
|
||||
)
|
||||
## The live Empire page, an alternative that may have changed its markup.
|
||||
EMPIRE_LIVE_URL: str = "https://www.empireonline.com/movies/features/best-movies-2/"
|
||||
|
||||
## File the movie list is written to.
|
||||
MOVIES_OUTPUT_FILE: str = "movies.txt"
|
||||
@@ -0,0 +1 @@
|
||||
"""@brief Business rules and entities; imports nothing from other layers."""
|
||||
@@ -0,0 +1 @@
|
||||
"""@brief Adapters for the web, BeautifulSoup and files; implements the ports."""
|
||||
@@ -0,0 +1 @@
|
||||
"""@brief Command-line entry point; calls the use cases."""
|
||||
Reference in New Issue
Block a user