From e06b886c28a77ac5fed0b0fb12ce2ea2d810b78c Mon Sep 17 00:00:00 2001 From: j2 Date: Sun, 11 Oct 2026 23:16:22 -0500 Subject: [PATCH] Add dialogue detection and coloring, some tests for attribution but little success --- .github/workflows/ci.yml | 17 +- .gitignore | 2 + Makefile | 2 + README.md | 104 +- TODO.md | 26 +- book-processing/attribution.py | 435 +++++++ book-processing/epub_to_chapters.py | 126 +- book-processing/label_dialogue.py | 321 +++++ book-processing/requirements.txt | 5 + .../attribution/chapter-1/Chapter 1.json | 337 +++++ .../attribution/chapter-1/Chapter 1.rsvp | Bin 0 -> 457 bytes .../attribution/chapter-1/Chapter 1.txt | 191 +++ .../prologue/Prologue- To Live.json | 1089 +++++++++++++++++ .../prologue/Prologue- To Live.rsvp | Bin 0 -> 3549 bytes .../prologue/Prologue- To Live.txt | 618 ++++++++++ book-processing/tests/test_attribution.py | 289 +++++ .../tests/test_epub_to_chapters.py | 74 +- dialogue-labels.json | 337 +++++ docs/history/PLAN.md | 80 ++ firmware/include/Board.h | 5 +- firmware/include/Library.h | 3 +- firmware/include/Palette.h | 29 + firmware/include/Reader.h | 3 + firmware/include/Settings.h | 1 + firmware/src/Board.cpp | 19 +- firmware/src/Library.cpp | 63 +- firmware/src/Reader.cpp | 21 +- firmware/src/Settings.cpp | 5 + firmware/src/main.cpp | 29 +- 29 files changed, 4075 insertions(+), 156 deletions(-) create mode 100644 book-processing/attribution.py create mode 100644 book-processing/label_dialogue.py create mode 100644 book-processing/tests/fixtures/attribution/chapter-1/Chapter 1.json create mode 100644 book-processing/tests/fixtures/attribution/chapter-1/Chapter 1.rsvp create mode 100644 book-processing/tests/fixtures/attribution/chapter-1/Chapter 1.txt create mode 100644 book-processing/tests/fixtures/attribution/prologue/Prologue- To Live.json create mode 100644 book-processing/tests/fixtures/attribution/prologue/Prologue- To Live.rsvp create mode 100644 book-processing/tests/fixtures/attribution/prologue/Prologue- To Live.txt create mode 100644 book-processing/tests/test_attribution.py create mode 100644 dialogue-labels.json create mode 100644 docs/history/PLAN.md create mode 100644 firmware/include/Palette.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 804cea4..df467d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,27 +7,20 @@ on: jobs: converter: runs-on: ubuntu-latest - defaults: - run: - working-directory: book-processing steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' - - run: python -m pip install -r requirements.txt - - run: python -m unittest discover -v - - run: python epub_to_chapters.py examples/rsvp-timing-sample.epub /tmp/timing-sample + python-version: '3.11' + - run: make test + - run: make convert firmware: runs-on: ubuntu-latest - defaults: - run: - working-directory: firmware steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.11' - run: python -m pip install platformio - - run: pio run + - run: make build diff --git a/.gitignore b/.gitignore index cc97458..6cb6037 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # PlatformIO build output and local editor settings. .pio/ +/.platformio/ .vscode/ *.bin *.elf @@ -16,6 +17,7 @@ htmlcov/ # Generated reader content and local source books. chapters/ +/Wind and Truth/ *.epub !book-processing/examples/rsvp-timing-sample.epub diff --git a/Makefile b/Makefile index 505b765..6b5f96b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ PYTHON ?= python3 PIO ?= pio +PLATFORMIO_CORE_DIR ?= $(CURDIR)/.platformio +export PLATFORMIO_CORE_DIR UPLOAD_PORT ?= VENV := .venv DEPS_STAMP := $(VENV)/.requirements-installed diff --git a/README.md b/README.md index ad433c6..5f40ec6 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ # T-Display S3 Pro RSVP Reader -An offline-first RSVP reader for the LilyGO T-Display S3 Pro. It renders one -white word on a black, landscape display and never starts Wi-Fi or Bluetooth. +An offline RSVP reader for the LilyGO T-Display S3 Pro. It renders one word at +a time on a black, landscape display and never starts Wi-Fi or Bluetooth. +Detected dialogue appears grey when Colors is on; other words appear white. ## What you need - A LilyGO T-Display S3 Pro and a **USB-C data cable**. A charge-only cable cannot upload firmware. -- A computer with internet access for the first build, Python 3, and a terminal. +- A computer with internet access for the first build, Python 3.11, and a terminal. - A microSD card formatted as FAT32. The reader uses this as its book library. - This repository, cloned or downloaded to your computer. @@ -67,13 +68,16 @@ operating systems: .txt`. The reader reads and caches this file for navigation; +it opens only the selected section file. +Chapter titles such as `TOC` and `CONVERSION_REPORT` are valid because their +files live in `content/`. For an older card, move each section's `.txt` and +`.rsvp` files into `content/`, leaving `TOC.txt` and book metadata at the root. The resulting card must look like this: @@ -82,12 +86,14 @@ microSD root/ └── books/ └── Example Book/ ├── TOC.txt - ├── Dedication.txt - ├── Dedication.rsvp - ├── Chapter 1.txt - ├── Chapter 1.rsvp - ├── Chapter 2.txt - └── Chapter 2.rsvp + ├── CONVERSION_REPORT.txt + └── content/ + ├── Dedication.txt + ├── Dedication.rsvp + ├── Chapter 1.txt + ├── Chapter 1.rsvp + ├── Chapter 2.txt + └── Chapter 2.rsvp ``` For the example above, `TOC.txt` contains: @@ -98,22 +104,24 @@ Chapter 1 Chapter 2 ``` -Generate this layout from an EPUB with the included converter: +From the repository root, install the Python dependencies and convert an EPUB: ```bash -./.venv/bin/python -m pip install -r book-processing/requirements.txt +make deps ./.venv/bin/python book-processing/epub_to_chapters.py book.epub "Example Book" ``` -For a quick end-to-end check, the repository includes a two-chapter sample -with EPUB styles, punctuation, and repeated invented names: +For a quick end-to-end check, convert the included two-chapter sample: ```bash -./.venv/bin/python book-processing/epub_to_chapters.py book-processing/examples/rsvp-timing-sample.epub "Timing Sample" +make convert ``` -The output directory must be empty unless `--overwrite` is supplied. Copy the -resulting `Example Book` directory into `/books` on the card. A section is +The sample command prints its temporary output directory. For your own book, +the output directory must be empty unless `--overwrite` is supplied. On +overwrite, the converter replaces generated chapter files in `content/` and +the root TOC and conversion report while preserving other root metadata. Copy +the resulting `Example Book` directory into `/books` on the card. A section is limited to 150,000 words; its source text and `.rsvp` sidecar may each be up to 1 MiB. The converter writes `CONVERSION_REPORT.txt` in the book directory. If unsupported non-ASCII @@ -125,8 +133,12 @@ chosen substitutions. The included display font is best for printable ASCII. The converter also writes a compact `
.rsvp` file alongside every section. Its versioned, delta-encoded records preserve italic and bold EPUB -markup—including semantic tags, CSS classes, and inline CSS—and reading-timing -metadata. The fingerprint covers the source `.txt` bytes, before the device +markup—including semantic tags, CSS classes, and inline CSS—reading-timing +metadata, and a word color flag. Quoted dialogue and wholly italic or uppercase +punctuated clauses receive the `Other` flag; speaker attribution is not used +during conversion. With coloring enabled, the device shows those words in grey +and other words in white. The reader menu has a persisted **Colors: On/Off** +setting. The fingerprint covers the source `.txt` bytes, before the device applies display-only punctuation normalization. During conversion, the desktop-only English vocabulary classifier marks each unfamiliar word. It combines a dictionary with word-frequency data and recognizes normal English @@ -141,25 +153,38 @@ bold records select matching reader-font variants; the device never performs dictionary or SD-card lookups while advancing words. The converter requires the desktop-only `english-words` and `wordfreq` -packages listed in `book-processing/requirements.txt`; the reader does not need either -dependency. Reconvert a book to create v2 sidecars with unfamiliar-word -timing. Existing v1 style-only sidecars remain readable but do not contain -timing metadata. +packages listed in `book-processing/requirements.txt`; the reader does not +need either dependency. Reconvert a book to create v3 sidecars with dialogue coloring. +Existing v1/v2 sidecars remain readable; v1 has no timing metadata, and neither +older version has word color flags. + +Speaker attribution remains an evaluation tool and is not part of EPUB +conversion. `book-processing/attribution.py` contains experimental rules, +including Maverick Coref for pronouns in speech tags. The default `make test` +uses a stub model and the labeled Prologue and Chapter 1 fixtures. To run the +optional evaluation with the downloaded model, use: + +```bash +RUN_MAVERICK_EVAL=1 make test +``` + +That first run downloads the LitBank model and tokenizer data. Its results do +not change the generated book or sidecars. Eject the card safely and insert it in the board before powering the board. ## 3. Build the firmware -Open a terminal in the `firmware` directory: +From the repository root, build through the Makefile: ```bash -cd /path/to/tds3p-rsvp-reader/firmware -pio run +make build ``` On the first run, PlatformIO downloads the pinned ESP32 platform, compiler, -and the TFT_eSPI, SensorLib, and XPowersLib dependencies. A successful build -ends with `SUCCESS` and creates the firmware under `.pio/build/rsvp_reader/`. +and the TFT_eSPI, SensorLib, and XPowersLib dependencies into the repository's +`.platformio/` cache. A successful build ends with `SUCCESS` and creates the +firmware under `firmware/.pio/build/rsvp_reader/`. The project targets the T-Display S3 Pro's 16 MB flash and 8 MB OPI PSRAM. It defaults to the v1.1 16-step backlight driver. If your board is the original @@ -188,13 +213,13 @@ on macOS, and `COM3` on Windows. Note the port belonging to the board. Let PlatformIO select the port automatically: ```bash -pio run -t upload +make flash ``` If it selects the wrong port or does not find one, supply it explicitly: ```bash -pio run -t upload --upload-port /dev/ttyACM0 +make flash UPLOAD_PORT=/dev/ttyACM0 ``` Replace `/dev/ttyACM0` with the port from the previous step. On Windows, for @@ -243,8 +268,9 @@ The reader saves the selected section, paragraph-level position, WPM, brightness, and font settings in the ESP32's nonvolatile storage. On the next boot or wake it opens the same section's title card and remains paused; pressing BOOT begins at the saved paragraph. It opens the -saved section directly, then falls back to the first TOC-listed section if the -card has changed. +saved section directly, maps a saved path from the old flat book layout to +`content/` when possible, then falls back to the first TOC-listed section if +the card has changed. When continuous reading reaches the end of a section, it opens the next section's title card and pauses. Press BOOT to begin that @@ -261,17 +287,17 @@ cloud account, or synchronization feature. - **Permission denied on Linux:** complete the `dialout` group step and log out and back in. - **`NO BOOKS`:** reformat the card FAT32, create `/books/`, and add a - `TOC.txt` plus the listed `.txt` section files. Confirm the card is fully + `TOC.txt` plus the listed `.txt` section files under `content/`. Confirm the card is fully seated. Dot-prefixed folders such as `.Trash` are ignored. - **Black display after boot:** press GPIO12 to wake it if it has slept; otherwise reflash and check whether the v1.0 backlight build flag is needed. -- **Build fails before compilation:** rerun `pio run` with internet access so +- **Build fails before compilation:** rerun `make build` with internet access so PlatformIO can download the required packages. ## Limits of the first MVP - It reads sections up to 150,000 words and 1 MiB from - `/books//
.txt`, in the order declared by that book's + `/books//content/
.txt`, in the order declared by that book's `TOC.txt`. - The built-in display font is best for printable ASCII text. UTF-8 text files are accepted, but accented/non-Latin characters need a Unicode font addition diff --git a/TODO.md b/TODO.md index 531b34d..81cda8f 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,10 @@ -# TODO +# Project status and next steps + +The converter writes TOC and book metadata at the book root, with chapter text +and v3 RSVP sidecars in `content/`. It detects dialogue and marks it `Other`; +the firmware displays those words in grey when Colors is on. Named speaker +attribution and Maverick Coref remain outside the conversion flow and are used +only in optional evaluation. ## Reading profiles @@ -9,24 +15,18 @@ profile that preserves the current cadence. Replace fixed 2×/3× sidecar word-delay values with categorical timing tags. Map each category to a configurable delay in reader settings, so changing a delay does not require regenerating sidecars. Version the format change and -keep existing v1/v2 sidecars readable. +keep existing v1/v2/v3 sidecars readable. ## Dialogue and character colors -Add optional color rendering for dialogue and character names. The converter -should detect conventional quotation marks and styled thought/dialogue spans, -then attribute only high-confidence spans to a canonical entity. Store the -entity identity, never a color, in a compact v3 RSVP sidecar; load a separate, -editable per-book entity map that assigns each entity a hue from 30 through 330 +Attribute only high-confidence spans to a canonical entity and load a separate, editable +per-book entity map that assigns each named entity a hue from 30 through 330 in 30-degree HSL steps at 100% saturation and 75% lightness. Include aliases so descriptions such as "mysterious man" can later resolve to a named entity, and report unresolved or ambiguous attribution for review rather than guessing. -Keep the sidecar sparse: unassigned words must have no sidecar record at all, -not a record with an entity ID of zero or another null sentinel. Extend a -record only when it already exists for style/timing metadata or when an entity -assignment requires one. Add a persisted reader setting to toggle coloring -without requiring reconversion. +Keep later entity assignments sparse: unassigned words have no sidecar record +unless style or timing metadata requires one. ## Add anti-aliased reader fonts @@ -41,7 +41,7 @@ previews. 3.5, and 4. 2. Place the files in a dedicated SD-card directory, for example `/fonts/