diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b193b37 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.env +.venv/ +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +*.egg-info/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a759dc9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "framework"] + path = framework + url = ssh://git@git.tirsystem.com:10022/TirSystem/SQA-QC-Framework.git diff --git a/README.md b/README.md index 6109918..4a74ed1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,120 @@ -# 048-scrapping-python-org +# 🚀 048 – Scraping python.org with Selenium +Day 48 of Udemy's *100 Days of Code: The Complete Python Pro Bootcamp*: scrape +the five upcoming events from [python.org](https://www.python.org/) with +Selenium and print them as a dictionary. This is a modernised version of the +original solution. + +## 📚 Table of Contents + +- [Overview](#-overview) +- [Requirements](#-requirements) +- [Setup](#-setup) +- [Run](#-run) +- [Tests](#-tests) +- [License](#-license) +- [Links](#-links) +- [Appendix: Project documents](#-appendix-project-documents) + +## 🧭 Overview + +The program opens python.org in headless Chrome, reads the *Upcoming Events* +widget and prints: + +```python +{0: {'time': '2026-10-29', 'name': 'PyConFR 2026'}, 1: {...}, ..., 4: {...}} +``` + +| Path | Purpose | +| --- | --- | +| `src/scraping_python_org/domain` | The `Event` value object | +| `src/scraping_python_org/application` | The use case and the `EventSource` port | +| `src/scraping_python_org/infrastructure` | Selenium and Chrome adapters | +| `src/scraping_python_org/presentation` | Command line entry point and screenshot capture | +| `tests/` | Unit tests and a headless browser test against a saved page | +| `docs/wiki/` | Wiki in English, Danish and Bahasa Malaysia | + +### The lecture steps + +**1. Open the page.** python.org loads in the browser. + +![Step 1: python.org home page](docs/screenshots/01-python-org-home.png) + +**2. Locate the widget.** Inspect the page: the events sit in the +`.event-widget` element, each in an `li` with a `time` and an `a` tag. + +![Step 2: the Upcoming Events widget](docs/screenshots/02-event-widget.png) + +**3. Select the five list items** with the CSS selector `.event-widget li` +(outlined in red). + +![Step 3: the five list items selected](docs/screenshots/03-event-list-highlighted.png) + +**4. Build the dictionary** with a `for` loop and print it. + +![Step 4: the printed dictionary](docs/screenshots/04-output-dictionary.png) + +What each step teaches is in the [wiki](docs/wiki/Home.md). + +## 📋 Requirements + +- Python 3.11 or newer (developed on 3.13). +- Google Chrome. Selenium Manager downloads a matching driver by itself. + +## 🛠️ Setup + +```bash +python -m venv .venv +source .venv/Scripts/activate # Windows Git Bash; on Linux/macOS: .venv/bin/activate +pip install -e ".[dev]" +``` + +## ▶️ Run + +```bash +python -m scraping_python_org +``` + +Regenerate the screenshots: + +```bash +python -m scraping_python_org.presentation.screenshots +``` + +## 🧪 Tests + +```bash +pytest +``` + +The `browser` test starts headless Chrome on a saved HTML page; skip it with +`pytest -m "not browser"`. + +## 📄 License + +See [LICENSE](./LICENSE). + +## 🔗 Links + +- [Repository][repository] +- [Documentation][documentation] +- [Issue tracker][issues] + +## 📎 Appendix: Project documents + +| Document | What it tells you | +| --- | --- | +| [AGENTS.md][agents] | The rules AI agents follow in this repository | +| [Artifact registry][registry] | The PO language and where artifacts live | +| [Framework README][framework-readme] | The framework this project uses | +| [Framework changelog][framework-changelog] | What changed in each framework release | + +--- + +[repository]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/048-scrapping-python-org +[documentation]: ./docs/wiki/Home.md +[issues]: https://git.tirsystem.com/Tirsvad-Udemy-100-days-of-code/048-scrapping-python-org/issues +[agents]: ./AGENTS.md +[registry]: ./docs/artifact-registry.md +[framework-readme]: ./framework/README.md +[framework-changelog]: ./framework/CHANGELOG.md diff --git a/docs/ai-prompts/session-001/prompts.md b/docs/ai-prompts/session-001/prompts.md new file mode 100644 index 0000000..3f70058 --- /dev/null +++ b/docs/ai-prompts/session-001/prompts.md @@ -0,0 +1,13 @@ +# Session 001 + +## Prompt 1 + +> Day 48 python org. Autonomous mode. Create a Python project (Udemy 100 Days of Code assignment): scrape the five upcoming events from python.org with Selenium into a dictionary `{index: {"time", "name"}}`. We do not make artifacts. Code in src, wiki in docs/wiki in English, Danish and Bahasa Malaysia, a screenshot per lecture step in README and wiki, wiki explains what we gained. Use my solution from github.com/Tirsvad/udemy.python.100-days-of-code (day-48-scrapping-python-org) and modernise it. + +**Outcome:** Plan proposed; PO language set to da. Scraper built under `src/` with tests, README and three-language wiki written. + +## Prompt 2 + +> do not make artifacts. + +**Outcome:** Treated as waiver of plan documents. Work continued; nothing pushed. diff --git a/docs/artifact-registry.md b/docs/artifact-registry.md new file mode 100644 index 0000000..e60a0b2 --- /dev/null +++ b/docs/artifact-registry.md @@ -0,0 +1,56 @@ +# Artifact Registry + +This project's artifact state. Types, short names and `CrossReference +Candidates` come from the framework catalog +(`framework/registry/artifact-catalog.md`); this file only records where +each document lives in *this* project and the next version to use. + +Delete rows for types you don't use. Add a row the first time you create a +document of a type. `Primary File` may contain a glob (e.g. +`docs/use-cases/uc-*/uc.md`); `framework/scripts/find-crossreferences.sh` reads it. + +| Short Name | Artifact Type | Primary File | Next Available Version | +| --- | --- | --- | --- | +| BC | Business Case | docs/business-case.md | 001 | +| SA | Stakeholder Analysis | docs/stakeholder-analysis.md | 001 | + +## Languages + +Set the PO language when the project starts; `project-planning` asks for it +if it is missing. An artifact of a type marked "Written in the PO language" +exists once, in that language, under its normal name (`business-case.md`); the +`artifact` skill ("One file per artifact") has the rule. + +| Setting | Value | +| --- | --- | +| PO language | da | +| PO domain | it | +| High-level register | IT Executive English | +| Technical register | IT Professional English | +| Wiki languages | en and the PO language, at least (`framework/process/wiki.md`) | + +Every artifact of a type marked "Yes" below states its language and domain in +its `Language` and `Domain` Metadata rows; `new-artifact.sh` fills them from +`PO language` and `PO domain`. `Language` is a BCP 47 code (`da`, `en`). +`Domain` is a value from this list; add a row to introduce a domain, so a +reviewer can see at once which professional vocabulary a document uses. + +| Domain | Meaning | +| --- | --- | +| it | Software and IT | +| medical | Healthcare and medical devices | +| construction | Construction and civil engineering | + +| Artifact types | Register | Written in the PO language | +| --- | --- | --- | +| BC, KPI, PP, MIL | IT Executive English | Yes | +| SA, BMC, BPMN, UCD, US, UC, SSD, DM, RA, GOV, DICT | IT Professional English | Yes | +| OC, SD, DCD, ERD, TP, AD, ADR, TM, RC, QC, source code | IT Professional English | No | + +## Notes + +- "Next Available Version" is the zero-padded (3-digit) version to use the + *next* time a new document of that type is created. Increment it only when + a brand-new document is created, not when an existing document's + `## Version History` gets a row. +- `ADR` uses 4 digits (`0001`); `RC` is sequential across all artifact types. diff --git a/docs/screenshots/01-python-org-home.png b/docs/screenshots/01-python-org-home.png new file mode 100644 index 0000000..14763d3 Binary files /dev/null and b/docs/screenshots/01-python-org-home.png differ diff --git a/docs/screenshots/02-event-widget.png b/docs/screenshots/02-event-widget.png new file mode 100644 index 0000000..070c536 Binary files /dev/null and b/docs/screenshots/02-event-widget.png differ diff --git a/docs/screenshots/03-event-list-highlighted.png b/docs/screenshots/03-event-list-highlighted.png new file mode 100644 index 0000000..479173a Binary files /dev/null and b/docs/screenshots/03-event-list-highlighted.png differ diff --git a/docs/screenshots/04-output-dictionary.png b/docs/screenshots/04-output-dictionary.png new file mode 100644 index 0000000..7f06ef3 Binary files /dev/null and b/docs/screenshots/04-output-dictionary.png differ diff --git a/docs/wiki/Home-da.md b/docs/wiki/Home-da.md new file mode 100644 index 0000000..f7e396d --- /dev/null +++ b/docs/wiki/Home-da.md @@ -0,0 +1,21 @@ +[English](Home) · **Dansk** · [Bahasa Malaysia](Home-ms) + +# Dag 48 – Scraping af python.org med Selenium + +I denne lektion bruger vi Selenium til at læse de fem kommende begivenheder på +[python.org](https://www.python.org/) og samle dem i en dictionary: + +```python +{0: {"time": "2026-10-29", "name": "PyConFR 2026"}, 1: {...}, ...} +``` + +## Sider + +- [Lektionens trin](Lecture-Steps-da): de fire trin, hver med et skærmbillede og hvad du får ud af det. +- [Modernisering](Modernisation-da): hvad der er ændret i forhold til den oprindelige løsning, og hvorfor. + +## Hvad du får ud af denne lektion + +- Du læser en websides HTML-struktur med browserens inspektionsværktøj. +- Du vælger locators (`By.CSS_SELECTOR`, `By.TAG_NAME`), der er korte og stabile. +- Du omsætter scrapede elementer til ren, struktureret Python-data. diff --git a/docs/wiki/Home-ms.md b/docs/wiki/Home-ms.md new file mode 100644 index 0000000..7288c5c --- /dev/null +++ b/docs/wiki/Home-ms.md @@ -0,0 +1,22 @@ +[English](Home) · [Dansk](Home-da) · **Bahasa Malaysia** + +# Hari 48 – Mengikis python.org dengan Selenium + +Dalam pelajaran ini kita menggunakan Selenium untuk membaca lima acara akan +datang di [python.org](https://www.python.org/) dan menyusunnya dalam kamus +(dictionary): + +```python +{0: {"time": "2026-10-29", "name": "PyConFR 2026"}, 1: {...}, ...} +``` + +## Halaman + +- [Langkah pelajaran](Lecture-Steps-ms): empat langkah, setiap satu dengan tangkapan skrin dan apa yang anda peroleh. +- [Pemodenan](Modernisation-ms): apa yang berubah daripada penyelesaian asal dan sebabnya. + +## Apa yang anda peroleh daripada pelajaran ini + +- Anda membaca struktur HTML sesebuah laman web dengan alat pemeriksa pelayar. +- Anda memilih locator (`By.CSS_SELECTOR`, `By.TAG_NAME`) yang ringkas dan stabil. +- Anda menukar elemen yang dikikis kepada data Python yang bersih dan tersusun. diff --git a/docs/wiki/Home.md b/docs/wiki/Home.md new file mode 100644 index 0000000..ea3fbeb --- /dev/null +++ b/docs/wiki/Home.md @@ -0,0 +1,21 @@ +**English** · [Dansk](Home-da) · [Bahasa Malaysia](Home-ms) + +# Day 48 – Scraping python.org with Selenium + +In this lecture we use Selenium to read the five upcoming events from +[python.org](https://www.python.org/) and organise them in a dictionary: + +```python +{0: {"time": "2026-10-29", "name": "PyConFR 2026"}, 1: {...}, ...} +``` + +## Pages + +- [Lecture steps](Lecture-Steps): the four steps, each with a screenshot and what you gain from it. +- [Modernisation](Modernisation): what changed from the original solution and why. + +## What you gain in this lecture + +- You read a web page's HTML structure with the browser's inspector. +- You choose locators (`By.CSS_SELECTOR`, `By.TAG_NAME`) that are short and stable. +- You turn scraped elements into clean, structured Python data. diff --git a/docs/wiki/Lecture-Steps-da.md b/docs/wiki/Lecture-Steps-da.md new file mode 100644 index 0000000..9c955b8 --- /dev/null +++ b/docs/wiki/Lecture-Steps-da.md @@ -0,0 +1,43 @@ +[English](Lecture-Steps) · **Dansk** · [Bahasa Malaysia](Lecture-Steps-ms) + +# Lektionens trin + +## Trin 1 – Åbn siden + +![Forsiden på python.org](images/01-python-org-home.png) + +`driver.get("https://www.python.org/")` indlæser siden i Chrome. + +**Hvad du får ud af det:** driveren er din fjernbetjening til en rigtig browser. +Den skal altid lukkes igen, så den moderne kode bruger `try/finally`. + +## Trin 2 – Find widgetten med inspektionsværktøjet + +![Widgetten Upcoming Events](images/02-event-widget.png) + +Højreklik på widgetten og vælg *Inspect*. Begivenhederne ligger i et element med +klassen `event-widget`; hver begivenhed er en `li` med et `time`- og et `a`-tag. + +**Hvad du får ud af det:** du lærer at læse HTML-træet og finde den beholder, +der rummer præcis de data, du vil have. + +## Trin 3 – Vælg de fem listepunkter med en CSS-selector + +![De fem listepunkter markeret med rødt](images/03-event-list-highlighted.png) + +`driver.find_elements(By.CSS_SELECTOR, ".event-widget li")` returnerer de fem +punkter. En klassebaseret selector er kortere og mere stabil end den lange +XPath, man kopierer fra inspektøren, og den rammer ikke urelaterede links. + +**Hvad du får ud af det:** du kender Seleniums locator-strategier og ved, +hvornår CSS-selectors slår XPath. + +## Trin 4 – Byg dictionary'en + +![Den udskrevne dictionary](images/04-output-dictionary.png) + +For hvert punkt læser vi `time` og `a` og gemmer dem under punktets indeks, med +`enumerate` i stedet for en manuel tæller. + +**Hvad du får ud af det:** du omsætter sideelementer til struktureret data: en +dictionary af dictionaries, klar til brug i resten af et program. diff --git a/docs/wiki/Lecture-Steps-ms.md b/docs/wiki/Lecture-Steps-ms.md new file mode 100644 index 0000000..c3d42e0 --- /dev/null +++ b/docs/wiki/Lecture-Steps-ms.md @@ -0,0 +1,43 @@ +[English](Lecture-Steps) · [Dansk](Lecture-Steps-da) · **Bahasa Malaysia** + +# Langkah pelajaran + +## Langkah 1 – Buka laman + +![Laman utama python.org](images/01-python-org-home.png) + +`driver.get("https://www.python.org/")` memuatkan laman dalam Chrome. + +**Apa yang anda peroleh:** driver ialah alat kawalan jauh untuk pelayar sebenar. +Ia mesti ditutup semula, jadi kod moden menggunakan `try/finally`. + +## Langkah 2 – Cari widget dengan alat pemeriksa + +![Widget Upcoming Events](images/02-event-widget.png) + +Klik kanan pada widget dan pilih *Inspect*. Acara berada dalam elemen berkelas +`event-widget`; setiap acara ialah `li` yang mengandungi teg `time` dan `a`. + +**Apa yang anda peroleh:** anda belajar membaca pepohon HTML dan mencari +bekas yang menyimpan data yang tepat seperti yang dikehendaki. + +## Langkah 3 – Pilih lima item senarai dengan pemilih CSS + +![Lima item senarai ditandakan merah](images/03-event-list-highlighted.png) + +`driver.find_elements(By.CSS_SELECTOR, ".event-widget li")` mengembalikan lima +item. Pemilih berasaskan kelas lebih pendek dan lebih stabil daripada XPath +panjang yang disalin dari pemeriksa, dan ia tidak memadankan pautan lain. + +**Apa yang anda peroleh:** anda mengenali strategi locator Selenium dan bila +pemilih CSS lebih baik daripada XPath. + +## Langkah 4 – Bina kamus + +![Kamus yang dicetak](images/04-output-dictionary.png) + +Untuk setiap item kita baca `time` dan `a` lalu simpan di bawah indeks item itu, +dengan `enumerate` dan bukannya pembilang manual. + +**Apa yang anda peroleh:** anda menukar elemen halaman kepada data tersusun: +kamus dalam kamus, sedia digunakan dalam bahagian program yang lain. diff --git a/docs/wiki/Lecture-Steps.md b/docs/wiki/Lecture-Steps.md new file mode 100644 index 0000000..ace700a --- /dev/null +++ b/docs/wiki/Lecture-Steps.md @@ -0,0 +1,44 @@ +**English** · [Dansk](Lecture-Steps-da) · [Bahasa Malaysia](Lecture-Steps-ms) + +# Lecture steps + +## Step 1 – Open the page + +![python.org home page](images/01-python-org-home.png) + +`driver.get("https://www.python.org/")` loads the page in Chrome. + +**What you gain:** the driver is your remote control for a real browser. It +must always be closed again, so the modern code wraps it in `try/finally`. + +## Step 2 – Locate the widget with the inspector + +![The Upcoming Events widget](images/02-event-widget.png) + +Right-click the widget and choose *Inspect*. The events live inside an element +with the class `event-widget`; each event is an `li` holding a `time` and an +`a` tag. + +**What you gain:** you learn to read the HTML tree and find the container that +holds exactly the data you want. + +## Step 3 – Select the five list items with a CSS selector + +![The five list items outlined in red](images/03-event-list-highlighted.png) + +`driver.find_elements(By.CSS_SELECTOR, ".event-widget li")` returns the five +items. A class-based selector is shorter and more stable than the long XPath +copied from the inspector, and it does not match unrelated links. + +**What you gain:** you know the Selenium locator strategies and when CSS +selectors beat XPath. + +## Step 4 – Build the dictionary + +![The printed dictionary](images/04-output-dictionary.png) + +For every item we read `time` and `a` and store them under the item's index, +with `enumerate` instead of a manual counter. + +**What you gain:** you turn page elements into structured data: a dictionary +of dictionaries, ready to use in the rest of a program. diff --git a/docs/wiki/Modernisation-da.md b/docs/wiki/Modernisation-da.md new file mode 100644 index 0000000..2ee32d7 --- /dev/null +++ b/docs/wiki/Modernisation-da.md @@ -0,0 +1,16 @@ +[English](Modernisation) · **Dansk** · [Bahasa Malaysia](Modernisation-ms) + +# Modernisering + +| Oprindelig løsning | Moderne løsning | Hvorfor | +| --- | --- | --- | +| Én `main.py` | Lag i `src/scraping_python_org` (domain, application, infrastructure, presentation) | Forretningsreglerne afhænger ikke af Selenium; use casen testes med en fake | +| Lang absolut XPath | CSS-selector `.event-widget li` | Kort, læsbar, tåler ændringer i layoutet | +| Manuel tæller `c` | `enumerate` | Mindre kode, ingen fejl med off-by-one | +| `detach=True` efterfulgt af `driver.quit()` | Headless Chrome i en context manager, der altid lukker | Ingen efterladte browservinduer, kører uden skrivebord | +| `selenium` uden version i `requirements.txt` | `pyproject.toml` med minimumsversioner | Reproducerbare installationer | +| Ingen tests | Enhedstests og en headless browsertest på en gemt side | Ændringer er trygge | +| Ingen typeannotationer | `mypy --strict`, `ruff` | Fejl findes før kørsel | + +**Hvad du får ud af det:** en scraper, du kan ændre med ro i sindet; samme idé +som i lektionen, men bygget som et rigtigt projekt. diff --git a/docs/wiki/Modernisation-ms.md b/docs/wiki/Modernisation-ms.md new file mode 100644 index 0000000..e41df60 --- /dev/null +++ b/docs/wiki/Modernisation-ms.md @@ -0,0 +1,16 @@ +[English](Modernisation) · [Dansk](Modernisation-da) · **Bahasa Malaysia** + +# Pemodenan + +| Penyelesaian asal | Penyelesaian moden | Sebab | +| --- | --- | --- | +| Satu `main.py` | Lapisan dalam `src/scraping_python_org` (domain, application, infrastructure, presentation) | Peraturan perniagaan tidak bergantung pada Selenium; use case diuji dengan palsu (fake) | +| XPath mutlak yang panjang | Pemilih CSS `.event-widget li` | Pendek, mudah dibaca, tahan perubahan reka letak | +| Pembilang manual `c` | `enumerate` | Kod lebih sedikit, tiada ralat off-by-one | +| `detach=True` diikuti `driver.quit()` | Chrome headless dalam context manager yang sentiasa menutup | Tiada tetingkap pelayar tertinggal, berjalan tanpa desktop | +| `selenium` tanpa versi dalam `requirements.txt` | `pyproject.toml` dengan versi minimum | Pemasangan yang boleh diulang | +| Tiada ujian | Ujian unit dan ujian pelayar headless pada halaman tersimpan | Perubahan selamat | +| Tiada petunjuk jenis | `mypy --strict`, `ruff` | Ralat ditemui sebelum dijalankan | + +**Apa yang anda peroleh:** pengikis yang boleh diubah dengan yakin; idea yang +sama seperti dalam pelajaran, tetapi dibina seperti projek sebenar. diff --git a/docs/wiki/Modernisation.md b/docs/wiki/Modernisation.md new file mode 100644 index 0000000..9d4e7f1 --- /dev/null +++ b/docs/wiki/Modernisation.md @@ -0,0 +1,16 @@ +**English** · [Dansk](Modernisation-da) · [Bahasa Malaysia](Modernisation-ms) + +# Modernisation + +| Original solution | Modern solution | Why | +| --- | --- | --- | +| One `main.py` | Layers in `src/scraping_python_org` (domain, application, infrastructure, presentation) | Business rules do not depend on Selenium; the use case is tested with a fake | +| Long absolute XPath | CSS selector `.event-widget li` | Short, readable, survives layout changes | +| Manual counter `c` | `enumerate` | Less code, no off-by-one risk | +| `detach=True` followed by `driver.quit()` | Headless Chrome in a context manager that always quits | No stray browser windows, runs without a desktop | +| Unpinned `selenium` in `requirements.txt` | `pyproject.toml` with minimum versions | Reproducible installs | +| No tests | Unit tests plus a headless browser test on a saved page | Changes are safe | +| No type hints | `mypy --strict`, `ruff` | Errors found before running | + +**What you gain:** a scraper you can change with confidence, the same idea +as the lecture but built the way a real project is. diff --git a/docs/wiki/_Sidebar.md b/docs/wiki/_Sidebar.md new file mode 100644 index 0000000..009aa7a --- /dev/null +++ b/docs/wiki/_Sidebar.md @@ -0,0 +1,17 @@ +**English** · [Dansk](Home-da) · [Bahasa Malaysia](Home-ms) + +- [Home](Home) +- [Lecture steps](Lecture-Steps) +- [Modernisation](Modernisation) + +**Dansk** + +- [Hjem](Home-da) +- [Lektionens trin](Lecture-Steps-da) +- [Modernisering](Modernisation-da) + +**Bahasa Malaysia** + +- [Utama](Home-ms) +- [Langkah pelajaran](Lecture-Steps-ms) +- [Pemodenan](Modernisation-ms) diff --git a/docs/wiki/images/01-python-org-home.png b/docs/wiki/images/01-python-org-home.png new file mode 100644 index 0000000..14763d3 Binary files /dev/null and b/docs/wiki/images/01-python-org-home.png differ diff --git a/docs/wiki/images/02-event-widget.png b/docs/wiki/images/02-event-widget.png new file mode 100644 index 0000000..070c536 Binary files /dev/null and b/docs/wiki/images/02-event-widget.png differ diff --git a/docs/wiki/images/03-event-list-highlighted.png b/docs/wiki/images/03-event-list-highlighted.png new file mode 100644 index 0000000..479173a Binary files /dev/null and b/docs/wiki/images/03-event-list-highlighted.png differ diff --git a/docs/wiki/images/04-output-dictionary.png b/docs/wiki/images/04-output-dictionary.png new file mode 100644 index 0000000..7f06ef3 Binary files /dev/null and b/docs/wiki/images/04-output-dictionary.png differ diff --git a/framework b/framework new file mode 160000 index 0000000..7c27919 --- /dev/null +++ b/framework @@ -0,0 +1 @@ +Subproject commit 7c2791939c42c18ec444bce245e9a3958b242cae diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..aa778b9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["hatchling>=1.27"] +build-backend = "hatchling.build" + +[project] +name = "scraping-python-org" +version = "0.1.0" +description = "Scrape the upcoming events from python.org with Selenium." +requires-python = ">=3.12" +dependencies = ["selenium>=4.25"] + +[dependency-groups] +dev = ["pytest>=8.3", "ruff>=0.8", "mypy>=1.13"] + +[tool.hatch.build.targets.wheel] +packages = ["src/scraping_python_org"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] +markers = ["browser: needs a headless Chrome installed"] + +[tool.ruff] +line-length = 100 +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "D", "ANN", "SIM", "T20"] +ignore = ["D203", "D213"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["D"] +"src/scraping_python_org/presentation/cli.py" = ["T201"] + +[tool.mypy] +strict = true +mypy_path = "src" diff --git a/src/scraping_python_org/__init__.py b/src/scraping_python_org/__init__.py new file mode 100644 index 0000000..257e53a --- /dev/null +++ b/src/scraping_python_org/__init__.py @@ -0,0 +1 @@ +"""Scrape the upcoming events from python.org.""" diff --git a/src/scraping_python_org/__main__.py b/src/scraping_python_org/__main__.py new file mode 100644 index 0000000..5593172 --- /dev/null +++ b/src/scraping_python_org/__main__.py @@ -0,0 +1,5 @@ +"""Entry point for ``python -m scraping_python_org``.""" + +from scraping_python_org.presentation.cli import main + +raise SystemExit(main()) diff --git a/src/scraping_python_org/application/__init__.py b/src/scraping_python_org/application/__init__.py new file mode 100644 index 0000000..b5bca45 --- /dev/null +++ b/src/scraping_python_org/application/__init__.py @@ -0,0 +1 @@ +"""Use cases and the ports they need.""" diff --git a/src/scraping_python_org/application/event_source.py b/src/scraping_python_org/application/event_source.py new file mode 100644 index 0000000..6eab7f6 --- /dev/null +++ b/src/scraping_python_org/application/event_source.py @@ -0,0 +1,13 @@ +"""Port for anything that can supply upcoming events.""" + +from typing import Protocol + +from scraping_python_org.domain.event import Event + + +class EventSource(Protocol): + """Supplies upcoming events in page order.""" + + def fetch_events(self) -> list[Event]: + """Return the upcoming events.""" + ... diff --git a/src/scraping_python_org/application/get_upcoming_events.py b/src/scraping_python_org/application/get_upcoming_events.py new file mode 100644 index 0000000..da0a3f3 --- /dev/null +++ b/src/scraping_python_org/application/get_upcoming_events.py @@ -0,0 +1,13 @@ +"""Use case: list the upcoming events as a numbered dictionary.""" + +from scraping_python_org.application.event_source import EventSource + +DEFAULT_LIMIT = 5 + + +def get_upcoming_events( + source: EventSource, limit: int = DEFAULT_LIMIT +) -> dict[int, dict[str, str]]: + """Return up to ``limit`` events as ``{index: {"time": ..., "name": ...}}``.""" + events = source.fetch_events()[:limit] + return {index: event.to_dict() for index, event in enumerate(events)} diff --git a/src/scraping_python_org/domain/__init__.py b/src/scraping_python_org/domain/__init__.py new file mode 100644 index 0000000..db6f68a --- /dev/null +++ b/src/scraping_python_org/domain/__init__.py @@ -0,0 +1 @@ +"""Business rules; no input or output.""" diff --git a/src/scraping_python_org/domain/event.py b/src/scraping_python_org/domain/event.py new file mode 100644 index 0000000..d1357a6 --- /dev/null +++ b/src/scraping_python_org/domain/event.py @@ -0,0 +1,15 @@ +"""The Event value object.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class Event: + """An upcoming event with its date text and name.""" + + time: str + name: str + + def to_dict(self) -> dict[str, str]: + """Return the event as a plain dictionary.""" + return {"time": self.time, "name": self.name} diff --git a/src/scraping_python_org/infrastructure/__init__.py b/src/scraping_python_org/infrastructure/__init__.py new file mode 100644 index 0000000..e6f8d5b --- /dev/null +++ b/src/scraping_python_org/infrastructure/__init__.py @@ -0,0 +1 @@ +"""Drivers and adapters for outside technology.""" diff --git a/src/scraping_python_org/infrastructure/chrome.py b/src/scraping_python_org/infrastructure/chrome.py new file mode 100644 index 0000000..00b8ef9 --- /dev/null +++ b/src/scraping_python_org/infrastructure/chrome.py @@ -0,0 +1,22 @@ +"""Chrome WebDriver lifecycle.""" + +from collections.abc import Iterator +from contextlib import contextmanager + +from selenium import webdriver + +WINDOW_SIZE = "1280,1000" + + +@contextmanager +def chrome_driver(headless: bool = True) -> Iterator[webdriver.Chrome]: + """Yield a Chrome driver (found by Selenium Manager) and always quit it.""" + options = webdriver.ChromeOptions() + if headless: + options.add_argument("--headless=new") + options.add_argument(f"--window-size={WINDOW_SIZE}") + driver = webdriver.Chrome(options=options) + try: + yield driver + finally: + driver.quit() diff --git a/src/scraping_python_org/infrastructure/selenium_event_source.py b/src/scraping_python_org/infrastructure/selenium_event_source.py new file mode 100644 index 0000000..37ab442 --- /dev/null +++ b/src/scraping_python_org/infrastructure/selenium_event_source.py @@ -0,0 +1,30 @@ +"""Selenium adapter for the EventSource port.""" + +from selenium.webdriver.common.by import By +from selenium.webdriver.remote.webdriver import WebDriver + +from scraping_python_org.domain.event import Event + +EVENT_WIDGET_SELECTOR = ".event-widget" +EVENT_ITEM_SELECTOR = f"{EVENT_WIDGET_SELECTOR} li" + + +class SeleniumEventSource: + """Reads the events from the event widget of a page.""" + + def __init__(self, driver: WebDriver, url: str) -> None: + """Remember the driver and the page to read.""" + self._driver = driver + self._url = url + + def fetch_events(self) -> list[Event]: + """Open the page and return one Event per widget list item.""" + self._driver.get(self._url) + items = self._driver.find_elements(By.CSS_SELECTOR, EVENT_ITEM_SELECTOR) + return [ + Event( + time=item.find_element(By.TAG_NAME, "time").text, + name=item.find_element(By.TAG_NAME, "a").text, + ) + for item in items + ] diff --git a/src/scraping_python_org/presentation/__init__.py b/src/scraping_python_org/presentation/__init__.py new file mode 100644 index 0000000..fb5dca6 --- /dev/null +++ b/src/scraping_python_org/presentation/__init__.py @@ -0,0 +1 @@ +"""Entry points and the composition root.""" diff --git a/src/scraping_python_org/presentation/cli.py b/src/scraping_python_org/presentation/cli.py new file mode 100644 index 0000000..54190c5 --- /dev/null +++ b/src/scraping_python_org/presentation/cli.py @@ -0,0 +1,16 @@ +"""Command line entry point and composition root.""" + +from scraping_python_org.application.get_upcoming_events import get_upcoming_events +from scraping_python_org.infrastructure.chrome import chrome_driver +from scraping_python_org.infrastructure.selenium_event_source import SeleniumEventSource + +PYTHON_ORG_URL = "https://www.python.org/" + + +def main() -> int: + """Scrape the upcoming events from python.org and print them.""" + with chrome_driver() as driver: + source = SeleniumEventSource(driver, PYTHON_ORG_URL) + events = get_upcoming_events(source) + print(events) + return 0 diff --git a/src/scraping_python_org/presentation/screenshots.py b/src/scraping_python_org/presentation/screenshots.py new file mode 100644 index 0000000..4ca1be7 --- /dev/null +++ b/src/scraping_python_org/presentation/screenshots.py @@ -0,0 +1,67 @@ +"""Capture one screenshot per lecture step into docs/screenshots/. + +Run with ``python -m scraping_python_org.presentation.screenshots``. +""" + +import html +import logging +from pathlib import Path +from urllib.parse import quote + +from selenium.webdriver.common.by import By +from selenium.webdriver.remote.webdriver import WebDriver + +from scraping_python_org.application.get_upcoming_events import get_upcoming_events +from scraping_python_org.infrastructure.chrome import chrome_driver +from scraping_python_org.infrastructure.selenium_event_source import ( + EVENT_ITEM_SELECTOR, + EVENT_WIDGET_SELECTOR, + SeleniumEventSource, +) +from scraping_python_org.presentation.cli import PYTHON_ORG_URL + +logger = logging.getLogger(__name__) + +OUTPUT_DIR = Path(__file__).resolve().parents[3] / "docs" / "screenshots" +HIGHLIGHT_SCRIPT = ( + "document.querySelectorAll(arguments[0])" + ".forEach(li => { li.style.outline = '3px solid red'; });" +) +OUTPUT_PAGE = ( + "" + "
{text}
" + "" +) + + +def _render_text(driver: WebDriver, text: str, path: Path) -> None: + """Screenshot ``text`` rendered in a 
 block."""
+    page = OUTPUT_PAGE.format(text=html.escape(text))
+    driver.get("data:text/html;charset=utf-8," + quote(page))
+    driver.find_element(By.TAG_NAME, "pre").screenshot(str(path))
+
+
+def capture(output_dir: Path = OUTPUT_DIR) -> list[Path]:
+    """Write the four lecture screenshots and return their paths."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    home = output_dir / "01-python-org-home.png"
+    widget = output_dir / "02-event-widget.png"
+    highlighted = output_dir / "03-event-list-highlighted.png"
+    output = output_dir / "04-output-dictionary.png"
+    with chrome_driver() as driver:
+        events = get_upcoming_events(SeleniumEventSource(driver, PYTHON_ORG_URL))
+        driver.save_screenshot(str(home))
+        driver.find_element(By.CSS_SELECTOR, EVENT_WIDGET_SELECTOR).screenshot(str(widget))
+        driver.execute_script(HIGHLIGHT_SCRIPT, EVENT_ITEM_SELECTOR)
+        driver.find_element(By.CSS_SELECTOR, EVENT_WIDGET_SELECTOR).screenshot(str(highlighted))
+        _render_text(driver, str(events), output)
+    paths = [home, widget, highlighted, output]
+    for path in paths:
+        logger.info("wrote %s", path)
+    return paths
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    capture()
diff --git a/tests/application/__init__.py b/tests/application/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/application/test_get_upcoming_events.py b/tests/application/test_get_upcoming_events.py
new file mode 100644
index 0000000..a21f31f
--- /dev/null
+++ b/tests/application/test_get_upcoming_events.py
@@ -0,0 +1,35 @@
+from scraping_python_org.application.get_upcoming_events import get_upcoming_events
+from scraping_python_org.domain.event import Event
+
+
+class FakeEventSource:
+    def __init__(self, events: list[Event]) -> None:
+        self._events = events
+
+    def fetch_events(self) -> list[Event]:
+        return self._events
+
+
+def _events(count: int) -> list[Event]:
+    return [Event(f"11-{i:02d}", f"Event {i}") for i in range(count)]
+
+
+def test_get_upcoming_events_numbers_events_from_zero() -> None:
+    result = get_upcoming_events(FakeEventSource(_events(2)))
+
+    assert result == {
+        0: {"time": "11-00", "name": "Event 0"},
+        1: {"time": "11-01", "name": "Event 1"},
+    }
+
+
+def test_get_upcoming_events_limits_to_five_by_default() -> None:
+    assert list(get_upcoming_events(FakeEventSource(_events(8)))) == [0, 1, 2, 3, 4]
+
+
+def test_get_upcoming_events_honours_custom_limit() -> None:
+    assert len(get_upcoming_events(FakeEventSource(_events(8)), limit=3)) == 3
+
+
+def test_get_upcoming_events_returns_empty_dict_without_events() -> None:
+    assert get_upcoming_events(FakeEventSource([])) == {}
diff --git a/tests/domain/__init__.py b/tests/domain/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/domain/test_event.py b/tests/domain/test_event.py
new file mode 100644
index 0000000..ad26c22
--- /dev/null
+++ b/tests/domain/test_event.py
@@ -0,0 +1,15 @@
+import dataclasses
+
+import pytest
+
+from scraping_python_org.domain.event import Event
+
+
+def test_to_dict_returns_time_and_name() -> None:
+    assert Event("2026-11-02", "PyCon").to_dict() == {"time": "2026-11-02", "name": "PyCon"}
+
+
+def test_event_is_immutable() -> None:
+    event = Event("2026-11-02", "PyCon")
+    with pytest.raises(dataclasses.FrozenInstanceError):
+        event.name = "Other"  # type: ignore[misc]
diff --git a/tests/fixtures/python_org_events.html b/tests/fixtures/python_org_events.html
new file mode 100644
index 0000000..142913f
--- /dev/null
+++ b/tests/fixtures/python_org_events.html
@@ -0,0 +1,22 @@
+
+
+Welcome to Python.org (fixture)
+
+

+  

+  

+    

+      
>>> Upcoming Events

+      
More

+      
+    

+  

+

+
+
diff --git a/tests/infrastructure/__init__.py b/tests/infrastructure/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/infrastructure/test_selenium_event_source.py b/tests/infrastructure/test_selenium_event_source.py
new file mode 100644
index 0000000..8b245d3
--- /dev/null
+++ b/tests/infrastructure/test_selenium_event_source.py
@@ -0,0 +1,30 @@
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+from selenium import webdriver
+
+from scraping_python_org.infrastructure.chrome import chrome_driver
+from scraping_python_org.infrastructure.selenium_event_source import SeleniumEventSource
+
+FIXTURE = Path(__file__).resolve().parents[1] / "fixtures" / "python_org_events.html"
+
+
+@pytest.fixture(scope="module")
+def driver() -> Iterator[webdriver.Chrome]:
+    with chrome_driver() as chrome:
+        yield chrome
+
+
+@pytest.mark.browser
+def test_fetch_events_reads_widget_items_only(driver: webdriver.Chrome) -> None:
+    events = SeleniumEventSource(driver, FIXTURE.as_uri()).fetch_events()
+
+    assert [event.name for event in events] == [
+        "PyCon Alpha",
+        "PyData Beta",
+        "Django Gamma",
+        "EuroPython Delta",
+        "PyLadies Epsilon",
+    ]
+    assert events[0].time == "2026-11-02"