src/scrape_movies/application/ports.py

@@ -0,0 +1,10 @@
+"""@brief The ports (interfaces) the use cases need from the outside world."""
+
+from collections.abc import Callable
+
+from scrape_movies.domain.story import Story
+
+## Fetches the HTML text of a page address.
+type HtmlFetcher = Callable[[str], str]
+## Turns Hacker News HTML into stories.
+type StoryParser = Callable[[str], list[Story]]

src/scrape_movies/application/use_cases.py

@@ -0,0 +1,26 @@
+"""@brief Use cases of the scraping exercises."""
+
+from scrape_movies.application.ports import HtmlFetcher, StoryParser
+from scrape_movies.domain.story import Story, top_story
+
+
+def list_stories(fetch: HtmlFetcher, parse: StoryParser, url: str) -> list[Story]:
+    """@brief Fetch a Hacker News page and read its stories.
+
+    @param fetch Fetches the HTML of the page.
+    @param parse Reads the stories from the HTML.
+    @param url The page address.
+    @return The stories in page order.
+    """
+    return parse(fetch(url))
+
+
+def find_top_story(fetch: HtmlFetcher, parse: StoryParser, url: str) -> Story | None:
+    """@brief Find the story with the most upvotes on a Hacker News page.
+
+    @param fetch Fetches the HTML of the page.
+    @param parse Reads the stories from the HTML.
+    @param url The page address.
+    @return The top story, or None when the page has no stories.
+    """
+    return top_story(list_stories(fetch, parse, url))

src/scrape_movies/constants.py

@@ -32,3 +32,6 @@ EMPIRE_LIVE_URL: str = "https://www.empireonline.com/movies/features/best-movies
 
 ## File the movie list is written to.
 MOVIES_OUTPUT_FILE: str = "movies.txt"
+
+## The saved copy of the course's Hacker News page, used offline.
+HACKER_NEWS_FIXTURE_PATH: Path = FIXTURES_DIR / "hacker_news.html"

src/scrape_movies/domain/errors.py

@@ -0,0 +1,5 @@
+"""@brief Errors the project raises."""
+
+
+class FetchError(Exception):
+    """@brief A web page could not be fetched."""

src/scrape_movies/domain/story.py

@@ -0,0 +1,27 @@
+"""@brief The Story entity (a Hacker News article) and the top-story rule."""
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Story:
+    """@brief An article on the Hacker News front page.
+
+    @param title The headline.
+    @param link The address the headline points to.
+    @param upvotes The number of points; 0 when the story shows none.
+    """
+
+    title: str
+    link: str
+    upvotes: int
+
+
+def top_story(stories: Sequence[Story]) -> Story | None:
+    """@brief Find the story with the most upvotes.
+
+    @param stories The stories in page order.
+    @return The first story with the highest score, or None when there are none.
+    """
+    return max(stories, key=lambda story: story.upvotes, default=None)

src/scrape_movies/infrastructure/hacker_news.py

@@ -0,0 +1,81 @@
+"""@brief Reading titles, links and upvotes from a Hacker News page."""
+
+from bs4 import BeautifulSoup, Tag
+
+from scrape_movies.domain.story import Story
+from scrape_movies.infrastructure.soup_basics import make_soup
+
+STORY_ROW = "tr.athing"
+TITLE_LINK = ".titleline > a"
+SCORE = ".score"
+
+
+def _upvotes(row: Tag) -> int:
+    """@brief Read the score shown in the row after a story, 0 when it has none."""
+    following = row.find_next_sibling("tr")
+    score = following.select_one(SCORE) if isinstance(following, Tag) else None
+    if score is None:
+        return 0
+    return int(score.get_text().split()[0])
+
+
+def _story_rows(soup: BeautifulSoup) -> list[Tag]:
+    return [row for row in soup.select(STORY_ROW) if row.select_one(TITLE_LINK)]
+
+
+def parse_stories(html: str) -> list[Story]:
+    """@brief Read every story of a front page.
+
+    @param html The HTML of the page.
+    @return The stories in page order.
+    """
+    stories: list[Story] = []
+    for row in _story_rows(make_soup(html)):
+        anchor = row.select_one(TITLE_LINK)
+        href = anchor.get("href") if anchor is not None else None
+        if anchor is not None and isinstance(href, str):
+            stories.append(
+                Story(title=anchor.get_text(), link=href, upvotes=_upvotes(row))
+            )
+    return stories
+
+
+def get_titles(html: str) -> list[str]:
+    """@brief Return the headlines of a front page.
+
+    @param html The HTML of the page.
+    @return The titles in page order.
+    """
+    return [story.title for story in parse_stories(html)]
+
+
+def get_links(html: str) -> list[str]:
+    """@brief Return the addresses the headlines point to.
+
+    @param html The HTML of the page.
+    @return The links in page order.
+    """
+    return [story.link for story in parse_stories(html)]
+
+
+def get_upvotes(html: str) -> list[int]:
+    """@brief Return the scores as numbers, not text such as "38 points".
+
+    @param html The HTML of the page.
+    @return The upvotes in page order.
+    """
+    return [story.upvotes for story in parse_stories(html)]
+
+
+def first_story_markup(html: str) -> str:
+    """@brief Show the markup of the first story, as seen with "Inspect" in a browser.
+
+    @param html The HTML of the page.
+    @return The indented HTML of its title row and score row, or "" when none.
+    """
+    rows = _story_rows(make_soup(html))
+    if not rows:
+        return ""
+    following = rows[0].find_next_sibling("tr")
+    parts = [rows[0], following] if isinstance(following, Tag) else [rows[0]]
+    return "\n".join(str(part.prettify()) for part in parts)

src/scrape_movies/infrastructure/http_client.py

@@ -0,0 +1,28 @@
+"""@brief Fetching web pages with Requests."""
+
+import requests
+
+from scrape_movies.constants import REQUEST_TIMEOUT_SECONDS, USER_AGENT
+from scrape_movies.domain.errors import FetchError
+
+
+def fetch_html(url: str) -> str:
+    """@brief Download a page with one GET request.
+
+    Sends a User-Agent and gives up after a timeout. Call it once per run:
+    scraping more often than once a minute is not polite.
+
+    @param url The address of the page.
+    @return The HTML text of the response.
+    @throws FetchError When the request fails or the server answers with an error.
+    """
+    try:
+        response = requests.get(
+            url,
+            headers={"User-Agent": USER_AGENT},
+            timeout=REQUEST_TIMEOUT_SECONDS,
+        )
+        response.raise_for_status()
+    except requests.RequestException as error:
+        raise FetchError(f"Could not fetch {url}: {error}") from error
+    return str(response.text)