Even Mer VP

.gitignore

@@ -16,6 +16,7 @@ htmlcov/
 # Generated reader content and local source books.
 chapters/
 *.epub
+!examples/rsvp-timing-sample.epub
 
 # Filesystem metadata that may appear when preparing an SD card.
 .DS_Store

README.md

@@ -103,6 +103,13 @@ python3 -m pip install -r requirements.txt
 python3 epub_to_chapters.py book.epub "Example Book"
 ```
 
+For a quick end-to-end check, the repository includes a two-chapter sample
+with EPUB styles, punctuation, and repeated invented names:
+
+```bash
+python3 epub_to_chapters.py examples/rsvp-timing-sample.epub "Timing Sample"
+```
+
 Copy the resulting `Example Book` directory into `/books` on the card. A
 section may be up to 1.5 MiB. The converter writes `CONVERSION_REPORT.txt` in
 the book directory. It records any unsupported non-ASCII characters left after
@@ -113,12 +120,22 @@ Unicode-capable font is available. The included display font is best for
 printable ASCII.
 
 The converter also writes a compact `<section>.rsvp` file alongside every
-section. Its versioned records preserve italic and bold EPUB markup—including
+section. Its versioned, delta-encoded records preserve italic and bold EPUB
-semantic tags, CSS classes, and inline CSS—using delta-encoded word indexes,
+markup—including semantic tags, CSS classes, and inline CSS—and reading-timing
-and include a reserved timing-modifier field for a future reading-timing pass.
+metadata. During conversion, the desktop-only English dictionary marks each
-The reader validates the sidecar's fingerprint against its `.txt` file and
+word not in that dictionary. The first ten appearances of each marked word in
-safely ignores it if it is missing, stale, or malformed. Valid italic and bold
+the book, plus its first appearance in every section, use a 3× delay. Later
-records select matching reader-font variants.
+appearances use a 2× delay. The reader uses the strongest applicable delay, so
+sentence and paragraph pauses are never compounded with unfamiliar-word
+timing. The reader validates the sidecar's fingerprint against its `.txt` file
+and safely ignores it if it is missing, stale, or malformed. Valid italic and
+bold records select matching reader-font variants; the device never performs
+dictionary or SD-card lookups while advancing words.
+
+The converter requires the desktop-only `english-words` package listed in
+`requirements.txt`; the reader does not need this dependency. Reconvert a book
+to create v2 sidecars with unfamiliar-word timing. Existing v1 style-only
+sidecars remain readable but do not contain timing metadata.
 
 Eject the card safely and insert it in the board before powering the board.
 
@@ -131,9 +148,9 @@ cd /path/to/tds3p-rsvp-reader
 pio run
 ```
 
-On the first run, PlatformIO downloads the pinned ESP32 platform, compiler, and
+On the first run, PlatformIO downloads the pinned ESP32 platform, compiler,
-TFT_eSPI dependency. A successful build ends with `SUCCESS` and creates the
+and the TFT_eSPI, SensorLib, and XPowersLib dependencies. A successful build
-firmware under `.pio/build/rsvp_reader/`.
+ends with `SUCCESS` and creates the firmware under `.pio/build/rsvp_reader/`.
 
 The project targets the T-Display S3 Pro's 16 MB flash and 8 MB OPI PSRAM. It
 defaults to the v1.1 16-step backlight driver. If your board is the original

TODO.md

@@ -25,23 +25,3 @@ previews.
    during normal word advances.
 6. Verify that chapter-title and `The End` cards use the same selected smooth
    font and remain aligned with normal words.
-
-## Preprocess unfamiliar-word timing
-
-Avoid dictionary lookups while reading. The existing chapter sidecars already
-carry sparse italic/bold metadata and reserve a timing-modifier field. Extend
-the desktop preprocessor to populate that field before books are copied to the
-SD card.
-
-1. Store only marked words: delta-encoded word indexes plus a small extra-delay
-   class (for example, 25%, 50%, or 100%). This keeps metadata sparse rather
-   than recording timing for every token.
-2. Have the preprocessor inspect the whole book to flag first appearances of
-   uncommon words, likely proper nouns, non-English terms, and user-supplied
-   vocabulary. Keep language models and dictionaries on the desktop, not the
-   device.
-3. At runtime, compare the current word
-   index against the next marked index; do not perform SD or dictionary lookups
-   during word advancement.
-4. Combine an extra-delay class with existing punctuation and long-word timing
-   using the strongest applicable delay, rather than multiplying every rule.

epub_to_chapters.py

@@ -115,10 +115,17 @@ STYLE_ITALIC = 0x01
 STYLE_BOLD = 0x02
 STYLE_MARKER = "\x01"
 SIDECAR_MAGIC = b"RSVP"
-SIDECAR_VERSION = 1
+# Version 2 activates timing modifiers. Version 1 sidecars remain readable by
+# the firmware, but their reserved timing bytes are required to be zero.
+SIDECAR_VERSION = 2
 SIDECAR_HEADER = struct.Struct("<4sBBQI")
 CSS_RULE = re.compile(r"([^{}]+)\{([^{}]*)\}")
 CSS_CLASS = re.compile(r"\.([A-Za-z_-][A-Za-z0-9_-]*)")
+WORD_LOOKUP_KEY = re.compile(r"^[^A-Za-z]*([A-Za-z]+(?:['-][A-Za-z]+)*)[^A-Za-z]*$")
+
+TIMING_NONE = 0
+TIMING_DOUBLE = 2
+TIMING_TRIPLE = 3
 
 DISPLAY_PUNCTUATION = str.maketrans({
     "\ufeff": "",      # UTF-8 byte-order mark.
@@ -312,12 +319,75 @@ def decode_styled_text(text):
         for style in character_styles[match.start():match.end()]:
             flags |= style
         if flags:
-            # The timing-modifier byte is reserved for a future sidecar version.
             style_records.append((word_index, flags, 0))
 
     return plain_text, style_records
 
 
+def lookup_word_key(token):
+    """Return a lower-case dictionary key for one RSVP token, if it has one."""
+
+    match = WORD_LOOKUP_KEY.fullmatch(token)
+    return match.group(1).lower() if match else None
+
+
+def load_english_dictionary():
+    """Load the desktop-only English dictionary required for timing analysis."""
+
+    try:
+        from english_words import get_english_words_set
+    except ModuleNotFoundError as exc:
+        raise RuntimeError(
+            "Unfamiliar-word timing needs english-words. Install converter "
+            "dependencies with: python3 -m pip install -r requirements.txt"
+        ) from exc
+
+    # web2 is a general-purpose English dictionary. Proper nouns and invented
+    # words are normally absent, which deliberately makes them qualify.
+    return get_english_words_set(["web2"], lower=True)
+
+
+def add_unfamiliar_word_timing(chapters, english_words):
+    """Merge book-wide unfamiliar-word timing into chapter sidecar records.
+
+    A word outside the supplied English dictionary receives a triple delay for
+    its first ten appearances in the book and on its first appearance in every
+    chapter. All later appearances receive a double delay. Timing is stored
+    sparsely alongside any EPUB style metadata already present for that token.
+    """
+
+    global_occurrences = Counter()
+
+    for chapter in chapters:
+        records = {
+            index: [style_flags, timing_modifier]
+            for index, style_flags, timing_modifier in chapter["style_records"]
+        }
+        chapter_occurrences = set()
+
+        for word_index, match in enumerate(re.finditer(r"\S+", chapter["text"])):
+            key = lookup_word_key(match.group())
+            if key is None or key in english_words:
+                continue
+
+            prior_occurrences = global_occurrences[key]
+            first_in_chapter = key not in chapter_occurrences
+            timing_modifier = (
+                TIMING_TRIPLE
+                if prior_occurrences < 10 or first_in_chapter
+                else TIMING_DOUBLE
+            )
+            record = records.setdefault(word_index, [0, TIMING_NONE])
+            record[1] = max(record[1], timing_modifier)
+            global_occurrences[key] += 1
+            chapter_occurrences.add(key)
+
+        chapter["style_records"] = [
+            (word_index, style_flags, timing_modifier)
+            for word_index, (style_flags, timing_modifier) in sorted(records.items())
+        ]
+
+
 def source_fingerprint(text):
     """Return a stable 64-bit FNV-1a fingerprint of UTF-8 chapter text."""
 
@@ -343,7 +413,7 @@ def encode_varint(value):
 
 
 def write_style_sidecar(output_file, text, style_records):
-    """Write a validated, extensible style-only sidecar beside a chapter."""
+    """Write validated sparse style and timing metadata beside a chapter."""
 
     payload = bytearray(SIDECAR_HEADER.pack(
         SIDECAR_MAGIC,
@@ -356,7 +426,9 @@ def write_style_sidecar(output_file, text, style_records):
     for record_number, (word_index, style_flags, timing_modifier) in enumerate(style_records):
         if word_index < 0 or style_flags & ~(STYLE_ITALIC | STYLE_BOLD):
             raise ValueError("invalid style sidecar record")
-        if style_flags == 0 or not 0 <= timing_modifier <= 0xFF:
+        if timing_modifier not in {TIMING_NONE, TIMING_DOUBLE, TIMING_TRIPLE}:
+            raise ValueError("invalid style sidecar record")
+        if style_flags == 0 and timing_modifier == TIMING_NONE:
             raise ValueError("invalid style sidecar record")
         if record_number and word_index <= previous_index:
             raise ValueError("style records must have ascending word indexes")
@@ -1364,6 +1436,12 @@ def convert_epub(epub_path, output_dir):
 
         sys.exit(1)
 
+    try:
+        add_unfamiliar_word_timing(chapters, load_english_dictionary())
+    except RuntimeError as exc:
+        print(f"Error: {exc}")
+        sys.exit(1)
+
     print(
         f"Chapters to write: {len(chapters)}"
     )

include/Reader.h

@@ -37,8 +37,10 @@ class Reader {
   std::vector<StyleToken> styleTokens_;
   std::vector<uint32_t> paragraphStarts_;
   uint32_t index_ = 0;
+  size_t nextStyleToken_ = 0;
   bool sidecarValid_ = false;
 
   bool loadSidecar(const std::vector<uint8_t>& sidecar);
-  uint8_t styleFlagsAt(uint32_t wordIndex) const;
+  void seekStyleToken();
+  const StyleToken* currentStyleToken() const;
 };

platformio.ini

@@ -14,3 +14,5 @@ build_flags =
   -DDISABLE_ALL_LIBRARY_WARNINGS
 lib_deps =
   TFT_eSPI
+  lewisxhe/SensorLib@0.4.0
+  lewisxhe/XPowersLib@0.2.9

requirements.txt

@@ -1,2 +1,3 @@
 ebooklib
 beautifulsoup4
+english-words==2.0.2

src/Reader.cpp

@@ -9,7 +9,10 @@ namespace {
 constexpr uint8_t kStyleItalic = 0x01;
 constexpr uint8_t kStyleBold = 0x02;
 constexpr uint8_t kKnownStyleFlags = kStyleItalic | kStyleBold;
-constexpr uint8_t kSidecarVersion = 1;
+constexpr uint8_t kSidecarVersion1 = 1;
+constexpr uint8_t kSidecarVersion2 = 2;
+constexpr uint8_t kTimingDouble = 2;
+constexpr uint8_t kTimingTriple = 3;
 constexpr size_t kSidecarHeaderBytes = 18;
 
 uint32_t readUint32Le(const std::vector<uint8_t>& bytes, size_t offset) {
@@ -122,6 +125,7 @@ bool Reader::load(const String& book, const std::vector<uint8_t>& sidecar) {
   styleTokens_.clear();
   paragraphStarts_.clear();
   index_ = 0;
+  nextStyleToken_ = 0;
   sidecarValid_ = false;
 
   bool inWord = false;
@@ -161,10 +165,12 @@ bool Reader::loadSidecar(const std::vector<uint8_t>& sidecar) {
   if (sidecar.size() < kSidecarHeaderBytes ||
       sidecar[0] != 'R' || sidecar[1] != 'S' ||
       sidecar[2] != 'V' || sidecar[3] != 'P' ||
-      sidecar[4] != kSidecarVersion || sidecar[5] != 0 ||
+      (sidecar[4] != kSidecarVersion1 && sidecar[4] != kSidecarVersion2) ||
+      sidecar[5] != 0 ||
       readUint64Le(sidecar, 6) != sourceFingerprint(book_)) {
     return false;
   }
+  const uint8_t sidecarVersion = sidecar[4];
 
   const uint32_t recordCount = readUint32Le(sidecar, 14);
   if (recordCount > tokens_.size()) return false;
@@ -187,8 +193,12 @@ bool Reader::loadSidecar(const std::vector<uint8_t>& sidecar) {
     }
     const uint8_t styleFlags = sidecar[offset++];
     const uint8_t timingModifier = sidecar[offset++];
-    if (wordIndex >= tokens_.size() || styleFlags == 0 ||
+    if (wordIndex >= tokens_.size() ||
-        (styleFlags & ~kKnownStyleFlags) != 0) {
+        (styleFlags & ~kKnownStyleFlags) != 0 ||
+        (timingModifier != 0 && timingModifier != kTimingDouble &&
+         timingModifier != kTimingTriple) ||
+        (sidecarVersion == kSidecarVersion1 && timingModifier != 0) ||
+        (styleFlags == 0 && timingModifier == 0)) {
       return false;
     }
     records.push_back({wordIndex, styleFlags, timingModifier});
@@ -197,6 +207,7 @@ bool Reader::loadSidecar(const std::vector<uint8_t>& sidecar) {
 
   if (offset != sidecar.size()) return false;
   styleTokens_.swap(records);
+  nextStyleToken_ = 0;
   return true;
 }
 
@@ -204,6 +215,7 @@ void Reader::restoreAtParagraphStart(uint32_t savedWord) {
   if (tokens_.empty()) return;
   index_ = savedWord < tokens_.size() ? savedWord : tokens_.size() - 1;
   index_ = paragraphStarts_[tokens_[index_].paragraph];
+  seekStyleToken();
 }
 
 const String Reader::currentWord() const {
@@ -215,6 +227,10 @@ const String Reader::currentWord() const {
 bool Reader::advance() {
   if (index_ + 1 >= tokens_.size()) return false;
   ++index_;
+  while (nextStyleToken_ < styleTokens_.size() &&
+         styleTokens_[nextStyleToken_].index < index_) {
+    ++nextStyleToken_;
+  }
   return true;
 }
 
@@ -223,9 +239,11 @@ bool Reader::previousParagraph() {
   const uint32_t paragraph = tokens_[index_].paragraph;
   if (paragraph == 0) {
     index_ = 0;
+    seekStyleToken();
     return false;
   }
   index_ = paragraphStarts_[paragraph - 1];
+  seekStyleToken();
   return true;
 }
 
@@ -234,12 +252,14 @@ bool Reader::nextParagraph() {
   const uint32_t paragraph = tokens_[index_].paragraph;
   if (paragraph + 1 >= paragraphStarts_.size()) return false;
   index_ = paragraphStarts_[paragraph + 1];
+  seekStyleToken();
   return true;
 }
 
 bool Reader::lastParagraph() {
   if (paragraphStarts_.empty()) return false;
   index_ = paragraphStarts_.back();
+  seekStyleToken();
   return true;
 }
 
@@ -255,17 +275,26 @@ uint8_t Reader::progressPercent() const {
                               lastIndex);
 }
 
-uint8_t Reader::currentStyleFlags() const { return styleFlagsAt(index_); }
+void Reader::seekStyleToken() {
-
-uint8_t Reader::styleFlagsAt(uint32_t wordIndex) const {
   const auto record = std::lower_bound(
-      styleTokens_.begin(), styleTokens_.end(), wordIndex,
+      styleTokens_.begin(), styleTokens_.end(), index_,
       [](const StyleToken& token, uint32_t index) {
         return token.index < index;
       });
-  return record != styleTokens_.end() && record->index == wordIndex
+  nextStyleToken_ = record - styleTokens_.begin();
-             ? record->styleFlags
+}
-             : 0;
+
+const StyleToken* Reader::currentStyleToken() const {
+  if (nextStyleToken_ >= styleTokens_.size() ||
+      styleTokens_[nextStyleToken_].index != index_) {
+    return nullptr;
+  }
+  return &styleTokens_[nextStyleToken_];
+}
+
+uint8_t Reader::currentStyleFlags() const {
+  const StyleToken* token = currentStyleToken();
+  return token ? token->styleFlags : 0;
 }
 
 bool Reader::sidecarValid() const { return sidecarValid_; }
@@ -289,5 +318,9 @@ uint32_t Reader::delayMs(uint16_t wpm) const {
   } else if (trailingLineBreaks == 1) {
     multiplier = max(multiplier, static_cast<uint8_t>(2));
   }
+  const StyleToken* styleToken = currentStyleToken();
+  if (styleToken) {
+    multiplier = max(multiplier, styleToken->timingModifier);
+  }
   return delay * multiplier;
 }

tests/test_epub_to_chapters.py

@@ -10,6 +10,9 @@ from epub_to_chapters import (
     SIDECAR_VERSION,
     STYLE_BOLD,
     STYLE_ITALIC,
+    TIMING_DOUBLE,
+    TIMING_TRIPLE,
+    add_unfamiliar_word_timing,
     decode_styled_text,
     document_root,
     html_to_text,
@@ -136,6 +139,19 @@ class StyleSidecarTests(unittest.TestCase):
             ]),
         )
 
+    def test_writes_timing_only_records(self):
+        text = "Blarenptnip"
+
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            sidecar_file = Path(temporary_directory) / "Chapter 1.rsvp"
+            write_style_sidecar(sidecar_file, text, [(0, 0, TIMING_TRIPLE)])
+            data = sidecar_file.read_bytes()
+
+        self.assertEqual(
+            data[SIDECAR_HEADER.size:],
+            bytes([0, 0, TIMING_TRIPLE]),
+        )
+
     def test_extracts_styles_from_css_classes_and_inline_css(self):
         class FakeItem:
             def __init__(self, name, content):
@@ -184,5 +200,39 @@ class StyleSidecarTests(unittest.TestCase):
         )
 
 
+class UnfamiliarWordTimingTests(unittest.TestCase):
+    def test_marks_first_ten_globally_and_first_word_in_each_chapter(self):
+        chapters = [
+            {
+                "text": " ".join(["Blarenptnip"] * 11),
+                "style_records": [(0, STYLE_ITALIC, 0)],
+            },
+            {
+                "text": "Blarenptnip",
+                "style_records": [],
+            },
+        ]
+
+        add_unfamiliar_word_timing(chapters, {"the", "reader"})
+
+        self.assertEqual(
+            chapters[0]["style_records"],
+            [(0, STYLE_ITALIC, TIMING_TRIPLE)] +
+            [(index, 0, TIMING_TRIPLE) for index in range(1, 10)] +
+            [(10, 0, TIMING_DOUBLE)],
+        )
+        self.assertEqual(chapters[1]["style_records"], [(0, 0, TIMING_TRIPLE)])
+
+    def test_english_words_and_non_words_are_not_marked(self):
+        chapters = [{
+            "text": "The reader has 123 pages.",
+            "style_records": [],
+        }]
+
+        add_unfamiliar_word_timing(chapters, {"the", "reader", "has", "pages"})
+
+        self.assertEqual(chapters[0]["style_records"], [])
+
+
 if __name__ == "__main__":
     unittest.main()