src/scrape_movies/presentation/cli.py

@@ -1,36 +1,47 @@
-"""@brief Command-line entry point that runs the lecture steps."""
+"""@brief Command-line entry point that runs the lecture steps."""
-
+
-import argparse
+import argparse
-from collections.abc import Sequence
+from collections.abc import Sequence
-
+
-from scrape_movies.presentation import soup_steps
+from scrape_movies.presentation import hacker_news_steps, soup_steps
-
+
-
+
-def build_parser() -> argparse.ArgumentParser:
+def build_parser() -> argparse.ArgumentParser:
-    """@brief Build the argument parser.
+    """@brief Build the argument parser.
-
+
-    @return The parser with one sub-command per lecture.
+    @return The parser with one sub-command per lecture.
-    """
+    """
-    parser = argparse.ArgumentParser(
+    parser = argparse.ArgumentParser(
-        prog="python -m scrape_movies",
+        prog="python -m scrape_movies",
-        description="Day 45 of 100 Days of Code: web scraping exercises.",
+        description="Day 45 of 100 Days of Code: web scraping exercises.",
-    )
+    )
-    commands = parser.add_subparsers(dest="command")
+    commands = parser.add_subparsers(dest="command")
-    soup = commands.add_parser("soup", help="parse the sample page website.html")
+    soup = commands.add_parser("soup", help="parse the sample page website.html")
-    soup.add_argument("step", choices=list(soup_steps.STEPS))
+    soup.add_argument("step", choices=list(soup_steps.STEPS))
-    return parser
+    news = commands.add_parser("hacker-news", help="scrape the Hacker News front page")
-
+    news.add_argument("step", choices=hacker_news_steps.STEPS)
-
+    news.add_argument(
-def main(argv: Sequence[str] | None = None) -> int:
+        "--source",
-    """@brief Run the command given on the command line.
+        choices=hacker_news_steps.SOURCES,
-
+        default="mirror",
-    @param argv The arguments, or None to read `sys.argv`.
+        help="the course mirror (default), the live site or the saved copy",
-    @return The process exit code.
+    )
-    """
+    return parser
-    parser = build_parser()
+
-    args = parser.parse_args(argv)
+
-    if args.command == "soup":
+def main(argv: Sequence[str] | None = None) -> int:
-        print(soup_steps.render_step(args.step))
+    """@brief Run the command given on the command line.
-        return 0
+
-    parser.print_help()
+    @param argv The arguments, or None to read `sys.argv`.
-    return 0
+    @return The process exit code.
+    """
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.command == "soup":
+        print(soup_steps.render_step(args.step))
+        return 0
+    if args.command == "hacker-news":
+        print(hacker_news_steps.render_step(args.step, args.source))
+        return 0
+    parser.print_help()
+    return 0

src/scrape_movies/presentation/hacker_news_steps.py

@@ -0,0 +1,62 @@
+"""@brief Text output of each "Scraping Hacker News" lecture step."""
+
+from scrape_movies.application import use_cases
+from scrape_movies.application.ports import HtmlFetcher
+from scrape_movies.constants import (
+    HACKER_NEWS_FIXTURE_PATH,
+    HACKER_NEWS_LIVE_URL,
+    HACKER_NEWS_URL,
+)
+from scrape_movies.infrastructure import hacker_news, http_client, soup_basics
+
+## Where a step reads the page: the course mirror, the live site or the saved copy.
+SOURCES: tuple[str, ...] = ("mirror", "live", "fixture")
+_URLS: dict[str, str] = {
+    "mirror": HACKER_NEWS_URL,
+    "live": HACKER_NEWS_LIVE_URL,
+    "fixture": str(HACKER_NEWS_FIXTURE_PATH),
+}
+
+
+def _fetcher(source: str) -> HtmlFetcher:
+    if source == "fixture":
+        return lambda path: soup_basics.load_html(HACKER_NEWS_FIXTURE_PATH)
+    return http_client.fetch_html
+
+
+def _numbered(lines: list[str]) -> str:
+    return "\n".join(f"{number}. {line}" for number, line in enumerate(lines, 1))
+
+
+def _run(source: str, step: str) -> str:
+    fetch, url = _fetcher(source), _URLS[source]
+    if step == "markup":
+        return hacker_news.first_story_markup(fetch(url)).rstrip()
+    stories = use_cases.list_stories(fetch, hacker_news.parse_stories, url)
+    if step == "titles":
+        return _numbered([story.title for story in stories])
+    if step == "links":
+        return _numbered([story.link for story in stories])
+    if step == "upvotes":
+        return _numbered([str(story.upvotes) for story in stories])
+    top = use_cases.find_top_story(fetch, hacker_news.parse_stories, url)
+    if top is None:
+        return "No stories found."
+    return f"{top.title}\n{top.link}\n{top.upvotes} points"
+
+
+## The lecture steps by command-line name, in lecture order.
+STEPS: tuple[str, ...] = ("markup", "titles", "links", "upvotes", "top")
+
+
+def render_step(step: str, source: str) -> str:
+    """@brief Run one lecture step.
+
+    @param step One of STEPS.
+    @param source One of SOURCES.
+    @return The text the step prints.
+    @throws KeyError When the step or the source is unknown.
+    """
+    if step not in STEPS:
+        raise KeyError(step)
+    return _run(source, step)

tools/capture_page.py

@@ -0,0 +1,56 @@
+"""@brief Save a screenshot of a web page with a headless Chromium browser.
+
+A headless browser draws into memory, so no desktop window is opened and no
+mouse or keyboard is used. It needs Edge, Chrome or Chromium on the machine.
+"""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+CANDIDATES = (
+    r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
+    r"C:\Program Files\Microsoft\Edge\Application\msedge.exe",
+    r"C:\Program Files\Google\Chrome\Application\chrome.exe",
+    "google-chrome",
+    "chromium",
+    "chromium-browser",
+)
+WINDOW = "1280,1000"
+
+
+def find_browser() -> str:
+    """@brief Locate a Chromium-based browser.
+
+    @return The path or command name of the browser.
+    @throws FileNotFoundError When none is installed.
+    """
+    for candidate in CANDIDATES:
+        if Path(candidate).exists() or shutil.which(candidate):
+            return candidate
+    raise FileNotFoundError("Install Edge, Chrome or Chromium to capture pages.")
+
+
+def capture_page(url: str, target: Path) -> Path:
+    """@brief Render a page and save it as a PNG.
+
+    @param url The page address.
+    @param target The PNG file to write.
+    @return The path of the PNG.
+    """
+    target.parent.mkdir(parents=True, exist_ok=True)
+    subprocess.run(
+        [
+            find_browser(),
+            "--headless=new",
+            "--disable-gpu",
+            "--hide-scrollbars",
+            f"--window-size={WINDOW}",
+            f"--screenshot={target}",
+            url,
+        ],
+        check=True,
+        capture_output=True,
+        timeout=120,
+    )
+    return target

tools/make_screenshots.py

@@ -1,55 +1,72 @@
-"""@brief Run the lecture steps and save one terminal screenshot per step.
+"""@brief Run the lecture steps and save one terminal screenshot per step.
-
+
-Usage: python tools/make_screenshots.py [name-prefix]
+Usage: python tools/make_screenshots.py [name-prefix]
-The images are written to docs/images/ and used by the README and the wiki.
+The images are written to docs/images/ and used by the README and the wiki.
-"""
+"""
-
+
-import subprocess
+import os
-import sys
+import subprocess
-from pathlib import Path
+import sys
-
+from pathlib import Path
-from render_terminal_png import render
+
-
+from capture_page import capture_page
-ROOT = Path(__file__).resolve().parents[1]
+from render_terminal_png import render
-IMAGES = ROOT / "docs" / "images"
+
-
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
-## File name (without .png) and the arguments given to `python -m scrape_movies`.
+
-STEPS: dict[str, list[str]] = {
+ROOT = Path(__file__).resolve().parents[1]
-    "soup-1-title": ["soup", "title"],
+IMAGES = ROOT / "docs" / "images"
-    "soup-2-prettify": ["soup", "prettify"],
+
-    "soup-3-find-all": ["soup", "find-all"],
+## File name (without .png) and the arguments given to `python -m scrape_movies`.
-    "soup-4-find": ["soup", "find"],
+STEPS: dict[str, list[str]] = {
-    "soup-5-select": ["soup", "select"],
+    "soup-1-title": ["soup", "title"],
-}
+    "soup-2-prettify": ["soup", "prettify"],
-
+    "soup-3-find-all": ["soup", "find-all"],
-
+    "soup-4-find": ["soup", "find"],
-def capture(name: str, args: list[str]) -> Path:
+    "soup-5-select": ["soup", "select"],
-    """@brief Run one step and write its screenshot.
+    "hn-2-markup": ["hacker-news", "markup"],
-
+    "hn-3-titles": ["hacker-news", "titles"],
-    @param name The image name without extension.
+    "hn-4-links": ["hacker-news", "links"],
-    @param args The arguments for `python -m scrape_movies`.
+    "hn-5-upvotes": ["hacker-news", "upvotes"],
-    @return The path of the PNG.
+    "hn-6-top": ["hacker-news", "top"],
-    """
+}
-    result = subprocess.run(
+## Screenshots of real web pages: image name and the address of the page.
-        [sys.executable, "-m", "scrape_movies", *args],
+PAGES: dict[str, str] = {
-        cwd=ROOT,
+    "hn-1-page": "https://appbrewery.github.io/news.ycombinator.com/",
-        capture_output=True,
+}
-        text=True,
+
-        encoding="utf-8",
+
-        check=True,
+def capture(name: str, args: list[str]) -> Path:
-    )
+    """@brief Run one step and write its screenshot.
-    target = IMAGES / f"{name}.png"
+
-    render("python -m scrape_movies " + " ".join(args), result.stdout, target)
+    @param name The image name without extension.
-    return target
+    @param args The arguments for `python -m scrape_movies`.
-
+    @return The path of the PNG.
-
+    """
-def main() -> None:
+    result = subprocess.run(
-    """@brief Capture every step whose name starts with the given prefix."""
+        [sys.executable, "-m", "scrape_movies", *args],
-    prefix = sys.argv[1] if len(sys.argv) > 1 else ""
+        cwd=ROOT,
-    for name, args in STEPS.items():
+        env={**os.environ, "PYTHONIOENCODING": "utf-8"},
-        if name.startswith(prefix):
+        capture_output=True,
-            print(capture(name, args))
+        text=True,
-
+        encoding="utf-8",
-
+        check=True,
-if __name__ == "__main__":
+    )
-    main()
+    target = IMAGES / f"{name}.png"
+    render("python -m scrape_movies " + " ".join(args), result.stdout, target)
+    return target
+
+
+def main() -> None:
+    """@brief Capture every step whose name starts with the given prefix."""
+    prefix = sys.argv[1] if len(sys.argv) > 1 else ""
+    for name, args in STEPS.items():
+        if name.startswith(prefix):
+            print(capture(name, args))
+    for name, url in PAGES.items():
+        if name.startswith(prefix):
+            print(capture_page(url, IMAGES / f"{name}.png"))
+
+
+if __name__ == "__main__":
+    main()