src/scrape_movies/presentation/cli.py

@@ -3,7 +3,7 @@
 import argparse
 from collections.abc import Sequence
 
-from scrape_movies.presentation import soup_steps
+from scrape_movies.presentation import hacker_news_steps, soup_steps
 
 
 def build_parser() -> argparse.ArgumentParser:
@@ -18,6 +18,14 @@ def build_parser() -> argparse.ArgumentParser:
     commands = parser.add_subparsers(dest="command")
     soup = commands.add_parser("soup", help="parse the sample page website.html")
     soup.add_argument("step", choices=list(soup_steps.STEPS))
+    news = commands.add_parser("hacker-news", help="scrape the Hacker News front page")
+    news.add_argument("step", choices=hacker_news_steps.STEPS)
+    news.add_argument(
+        "--source",
+        choices=hacker_news_steps.SOURCES,
+        default="mirror",
+        help="the course mirror (default), the live site or the saved copy",
+    )
     return parser
 
 
@@ -32,5 +40,8 @@ def main(argv: Sequence[str] | None = None) -> int:
     if args.command == "soup":
         print(soup_steps.render_step(args.step))
         return 0
+    if args.command == "hacker-news":
+        print(hacker_news_steps.render_step(args.step, args.source))
+        return 0
     parser.print_help()
     return 0

src/scrape_movies/presentation/hacker_news_steps.py

@@ -0,0 +1,62 @@
+"""@brief Text output of each "Scraping Hacker News" lecture step."""
+
+from scrape_movies.application import use_cases
+from scrape_movies.application.ports import HtmlFetcher
+from scrape_movies.constants import (
+    HACKER_NEWS_FIXTURE_PATH,
+    HACKER_NEWS_LIVE_URL,
+    HACKER_NEWS_URL,
+)
+from scrape_movies.infrastructure import hacker_news, http_client, soup_basics
+
+## Where a step reads the page: the course mirror, the live site or the saved copy.
+SOURCES: tuple[str, ...] = ("mirror", "live", "fixture")
+_URLS: dict[str, str] = {
+    "mirror": HACKER_NEWS_URL,
+    "live": HACKER_NEWS_LIVE_URL,
+    "fixture": str(HACKER_NEWS_FIXTURE_PATH),
+}
+
+
+def _fetcher(source: str) -> HtmlFetcher:
+    if source == "fixture":
+        return lambda path: soup_basics.load_html(HACKER_NEWS_FIXTURE_PATH)
+    return http_client.fetch_html
+
+
+def _numbered(lines: list[str]) -> str:
+    return "\n".join(f"{number}. {line}" for number, line in enumerate(lines, 1))
+
+
+def _run(source: str, step: str) -> str:
+    fetch, url = _fetcher(source), _URLS[source]
+    if step == "markup":
+        return hacker_news.first_story_markup(fetch(url)).rstrip()
+    stories = use_cases.list_stories(fetch, hacker_news.parse_stories, url)
+    if step == "titles":
+        return _numbered([story.title for story in stories])
+    if step == "links":
+        return _numbered([story.link for story in stories])
+    if step == "upvotes":
+        return _numbered([str(story.upvotes) for story in stories])
+    top = use_cases.find_top_story(fetch, hacker_news.parse_stories, url)
+    if top is None:
+        return "No stories found."
+    return f"{top.title}\n{top.link}\n{top.upvotes} points"
+
+
+## The lecture steps by command-line name, in lecture order.
+STEPS: tuple[str, ...] = ("markup", "titles", "links", "upvotes", "top")
+
+
+def render_step(step: str, source: str) -> str:
+    """@brief Run one lecture step.
+
+    @param step One of STEPS.
+    @param source One of SOURCES.
+    @return The text the step prints.
+    @throws KeyError When the step or the source is unknown.
+    """
+    if step not in STEPS:
+        raise KeyError(step)
+    return _run(source, step)

tools/capture_page.py

@@ -0,0 +1,56 @@
+"""@brief Save a screenshot of a web page with a headless Chromium browser.
+
+A headless browser draws into memory, so no desktop window is opened and no
+mouse or keyboard is used. It needs Edge, Chrome or Chromium on the machine.
+"""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+CANDIDATES = (
+    r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
+    r"C:\Program Files\Microsoft\Edge\Application\msedge.exe",
+    r"C:\Program Files\Google\Chrome\Application\chrome.exe",
+    "google-chrome",
+    "chromium",
+    "chromium-browser",
+)
+WINDOW = "1280,1000"
+
+
+def find_browser() -> str:
+    """@brief Locate a Chromium-based browser.
+
+    @return The path or command name of the browser.
+    @throws FileNotFoundError When none is installed.
+    """
+    for candidate in CANDIDATES:
+        if Path(candidate).exists() or shutil.which(candidate):
+            return candidate
+    raise FileNotFoundError("Install Edge, Chrome or Chromium to capture pages.")
+
+
+def capture_page(url: str, target: Path) -> Path:
+    """@brief Render a page and save it as a PNG.
+
+    @param url The page address.
+    @param target The PNG file to write.
+    @return The path of the PNG.
+    """
+    target.parent.mkdir(parents=True, exist_ok=True)
+    subprocess.run(
+        [
+            find_browser(),
+            "--headless=new",
+            "--disable-gpu",
+            "--hide-scrollbars",
+            f"--window-size={WINDOW}",
+            f"--screenshot={target}",
+            url,
+        ],
+        check=True,
+        capture_output=True,
+        timeout=120,
+    )
+    return target

tools/make_screenshots.py

@@ -4,12 +4,16 @@ Usage: python tools/make_screenshots.py [name-prefix]
 The images are written to docs/images/ and used by the README and the wiki.
 """
 
+import os
 import subprocess
 import sys
 from pathlib import Path
 
+from capture_page import capture_page
 from render_terminal_png import render
 
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
 ROOT = Path(__file__).resolve().parents[1]
 IMAGES = ROOT / "docs" / "images"
 
@@ -20,6 +24,15 @@ STEPS: dict[str, list[str]] = {
     "soup-3-find-all": ["soup", "find-all"],
     "soup-4-find": ["soup", "find"],
     "soup-5-select": ["soup", "select"],
+    "hn-2-markup": ["hacker-news", "markup"],
+    "hn-3-titles": ["hacker-news", "titles"],
+    "hn-4-links": ["hacker-news", "links"],
+    "hn-5-upvotes": ["hacker-news", "upvotes"],
+    "hn-6-top": ["hacker-news", "top"],
+}
+## Screenshots of real web pages: image name and the address of the page.
+PAGES: dict[str, str] = {
+    "hn-1-page": "https://appbrewery.github.io/news.ycombinator.com/",
 }
 
 
@@ -33,6 +46,7 @@ def capture(name: str, args: list[str]) -> Path:
     result = subprocess.run(
         [sys.executable, "-m", "scrape_movies", *args],
         cwd=ROOT,
+        env={**os.environ, "PYTHONIOENCODING": "utf-8"},
         capture_output=True,
         text=True,
         encoding="utf-8",
@@ -49,6 +63,9 @@ def main() -> None:
     for name, args in STEPS.items():
         if name.startswith(prefix):
             print(capture(name, args))
+    for name, url in PAGES.items():
+        if name.startswith(prefix):
+            print(capture_page(url, IMAGES / f"{name}.png"))
 
 
 if __name__ == "__main__":