diff --git a/.gitignore b/.gitignore index af0ec00..f279553 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ htmlcov/ # Generated reader content and local source books. chapters/ *.epub +!examples/rsvp-timing-sample.epub # Filesystem metadata that may appear when preparing an SD card. .DS_Store diff --git a/README.md b/README.md index 3bad09e..d4fb08c 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,13 @@ python3 -m pip install -r requirements.txt python3 epub_to_chapters.py book.epub "Example Book" ``` +For a quick end-to-end check, the repository includes a two-chapter sample +with EPUB styles, punctuation, and repeated invented names: + +```bash +python3 epub_to_chapters.py examples/rsvp-timing-sample.epub "Timing Sample" +``` + Copy the resulting `Example Book` directory into `/books` on the card. A section may be up to 1.5 MiB. The converter writes `CONVERSION_REPORT.txt` in the book directory. It records any unsupported non-ASCII characters left after @@ -113,12 +120,22 @@ Unicode-capable font is available. The included display font is best for printable ASCII. The converter also writes a compact `
.rsvp` file alongside every -section. Its versioned records preserve italic and bold EPUB markup—including -semantic tags, CSS classes, and inline CSS—using delta-encoded word indexes, -and include a reserved timing-modifier field for a future reading-timing pass. -The reader validates the sidecar's fingerprint against its `.txt` file and -safely ignores it if it is missing, stale, or malformed. Valid italic and bold -records select matching reader-font variants. +section. Its versioned, delta-encoded records preserve italic and bold EPUB +markup—including semantic tags, CSS classes, and inline CSS—and reading-timing +metadata. During conversion, the desktop-only English dictionary marks each +word not in that dictionary. The first ten appearances of each marked word in +the book, plus its first appearance in every section, use a 3× delay. Later +appearances use a 2× delay. The reader uses the strongest applicable delay, so +sentence and paragraph pauses are never compounded with unfamiliar-word +timing. The reader validates the sidecar's fingerprint against its `.txt` file +and safely ignores it if it is missing, stale, or malformed. Valid italic and +bold records select matching reader-font variants; the device never performs +dictionary or SD-card lookups while advancing words. + +The converter requires the desktop-only `english-words` package listed in +`requirements.txt`; the reader does not need this dependency. Reconvert a book +to create v2 sidecars with unfamiliar-word timing. Existing v1 style-only +sidecars remain readable but do not contain timing metadata. Eject the card safely and insert it in the board before powering the board. @@ -131,9 +148,9 @@ cd /path/to/tds3p-rsvp-reader pio run ``` -On the first run, PlatformIO downloads the pinned ESP32 platform, compiler, and -TFT_eSPI dependency. A successful build ends with `SUCCESS` and creates the -firmware under `.pio/build/rsvp_reader/`. +On the first run, PlatformIO downloads the pinned ESP32 platform, compiler, +and the TFT_eSPI, SensorLib, and XPowersLib dependencies. A successful build +ends with `SUCCESS` and creates the firmware under `.pio/build/rsvp_reader/`. The project targets the T-Display S3 Pro's 16 MB flash and 8 MB OPI PSRAM. It defaults to the v1.1 16-step backlight driver. If your board is the original diff --git a/TODO.md b/TODO.md index 1b5d481..cff1a97 100644 --- a/TODO.md +++ b/TODO.md @@ -25,23 +25,3 @@ previews. during normal word advances. 6. Verify that chapter-title and `The End` cards use the same selected smooth font and remain aligned with normal words. - -## Preprocess unfamiliar-word timing - -Avoid dictionary lookups while reading. The existing chapter sidecars already -carry sparse italic/bold metadata and reserve a timing-modifier field. Extend -the desktop preprocessor to populate that field before books are copied to the -SD card. - -1. Store only marked words: delta-encoded word indexes plus a small extra-delay - class (for example, 25%, 50%, or 100%). This keeps metadata sparse rather - than recording timing for every token. -2. Have the preprocessor inspect the whole book to flag first appearances of - uncommon words, likely proper nouns, non-English terms, and user-supplied - vocabulary. Keep language models and dictionaries on the desktop, not the - device. -3. At runtime, compare the current word - index against the next marked index; do not perform SD or dictionary lookups - during word advancement. -4. Combine an extra-delay class with existing punctuation and long-word timing - using the strongest applicable delay, rather than multiplying every rule. diff --git a/epub_to_chapters.py b/epub_to_chapters.py index 2e6019b..3f6541d 100644 --- a/epub_to_chapters.py +++ b/epub_to_chapters.py @@ -115,10 +115,17 @@ STYLE_ITALIC = 0x01 STYLE_BOLD = 0x02 STYLE_MARKER = "\x01" SIDECAR_MAGIC = b"RSVP" -SIDECAR_VERSION = 1 +# Version 2 activates timing modifiers. Version 1 sidecars remain readable by +# the firmware, but their reserved timing bytes are required to be zero. +SIDECAR_VERSION = 2 SIDECAR_HEADER = struct.Struct("<4sBBQI") CSS_RULE = re.compile(r"([^{}]+)\{([^{}]*)\}") CSS_CLASS = re.compile(r"\.([A-Za-z_-][A-Za-z0-9_-]*)") +WORD_LOOKUP_KEY = re.compile(r"^[^A-Za-z]*([A-Za-z]+(?:['-][A-Za-z]+)*)[^A-Za-z]*$") + +TIMING_NONE = 0 +TIMING_DOUBLE = 2 +TIMING_TRIPLE = 3 DISPLAY_PUNCTUATION = str.maketrans({ "\ufeff": "", # UTF-8 byte-order mark. @@ -312,12 +319,75 @@ def decode_styled_text(text): for style in character_styles[match.start():match.end()]: flags |= style if flags: - # The timing-modifier byte is reserved for a future sidecar version. style_records.append((word_index, flags, 0)) return plain_text, style_records +def lookup_word_key(token): + """Return a lower-case dictionary key for one RSVP token, if it has one.""" + + match = WORD_LOOKUP_KEY.fullmatch(token) + return match.group(1).lower() if match else None + + +def load_english_dictionary(): + """Load the desktop-only English dictionary required for timing analysis.""" + + try: + from english_words import get_english_words_set + except ModuleNotFoundError as exc: + raise RuntimeError( + "Unfamiliar-word timing needs english-words. Install converter " + "dependencies with: python3 -m pip install -r requirements.txt" + ) from exc + + # web2 is a general-purpose English dictionary. Proper nouns and invented + # words are normally absent, which deliberately makes them qualify. + return get_english_words_set(["web2"], lower=True) + + +def add_unfamiliar_word_timing(chapters, english_words): + """Merge book-wide unfamiliar-word timing into chapter sidecar records. + + A word outside the supplied English dictionary receives a triple delay for + its first ten appearances in the book and on its first appearance in every + chapter. All later appearances receive a double delay. Timing is stored + sparsely alongside any EPUB style metadata already present for that token. + """ + + global_occurrences = Counter() + + for chapter in chapters: + records = { + index: [style_flags, timing_modifier] + for index, style_flags, timing_modifier in chapter["style_records"] + } + chapter_occurrences = set() + + for word_index, match in enumerate(re.finditer(r"\S+", chapter["text"])): + key = lookup_word_key(match.group()) + if key is None or key in english_words: + continue + + prior_occurrences = global_occurrences[key] + first_in_chapter = key not in chapter_occurrences + timing_modifier = ( + TIMING_TRIPLE + if prior_occurrences < 10 or first_in_chapter + else TIMING_DOUBLE + ) + record = records.setdefault(word_index, [0, TIMING_NONE]) + record[1] = max(record[1], timing_modifier) + global_occurrences[key] += 1 + chapter_occurrences.add(key) + + chapter["style_records"] = [ + (word_index, style_flags, timing_modifier) + for word_index, (style_flags, timing_modifier) in sorted(records.items()) + ] + + def source_fingerprint(text): """Return a stable 64-bit FNV-1a fingerprint of UTF-8 chapter text.""" @@ -343,7 +413,7 @@ def encode_varint(value): def write_style_sidecar(output_file, text, style_records): - """Write a validated, extensible style-only sidecar beside a chapter.""" + """Write validated sparse style and timing metadata beside a chapter.""" payload = bytearray(SIDECAR_HEADER.pack( SIDECAR_MAGIC, @@ -356,7 +426,9 @@ def write_style_sidecar(output_file, text, style_records): for record_number, (word_index, style_flags, timing_modifier) in enumerate(style_records): if word_index < 0 or style_flags & ~(STYLE_ITALIC | STYLE_BOLD): raise ValueError("invalid style sidecar record") - if style_flags == 0 or not 0 <= timing_modifier <= 0xFF: + if timing_modifier not in {TIMING_NONE, TIMING_DOUBLE, TIMING_TRIPLE}: + raise ValueError("invalid style sidecar record") + if style_flags == 0 and timing_modifier == TIMING_NONE: raise ValueError("invalid style sidecar record") if record_number and word_index <= previous_index: raise ValueError("style records must have ascending word indexes") @@ -1364,6 +1436,12 @@ def convert_epub(epub_path, output_dir): sys.exit(1) + try: + add_unfamiliar_word_timing(chapters, load_english_dictionary()) + except RuntimeError as exc: + print(f"Error: {exc}") + sys.exit(1) + print( f"Chapters to write: {len(chapters)}" ) diff --git a/examples/rsvp-timing-sample.epub b/examples/rsvp-timing-sample.epub new file mode 100644 index 0000000..5c0c463 Binary files /dev/null and b/examples/rsvp-timing-sample.epub differ diff --git a/include/Reader.h b/include/Reader.h index ac0a49a..9cf9171 100644 --- a/include/Reader.h +++ b/include/Reader.h @@ -37,8 +37,10 @@ class Reader { std::vector styleTokens_; std::vector paragraphStarts_; uint32_t index_ = 0; + size_t nextStyleToken_ = 0; bool sidecarValid_ = false; bool loadSidecar(const std::vector& sidecar); - uint8_t styleFlagsAt(uint32_t wordIndex) const; + void seekStyleToken(); + const StyleToken* currentStyleToken() const; }; diff --git a/platformio.ini b/platformio.ini index 4d5da69..bb20777 100644 --- a/platformio.ini +++ b/platformio.ini @@ -14,3 +14,5 @@ build_flags = -DDISABLE_ALL_LIBRARY_WARNINGS lib_deps = TFT_eSPI + lewisxhe/SensorLib@0.4.0 + lewisxhe/XPowersLib@0.2.9 diff --git a/requirements.txt b/requirements.txt index a22f248..f1abb10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ ebooklib beautifulsoup4 +english-words==2.0.2 diff --git a/src/Reader.cpp b/src/Reader.cpp index 2611695..5731012 100644 --- a/src/Reader.cpp +++ b/src/Reader.cpp @@ -9,7 +9,10 @@ namespace { constexpr uint8_t kStyleItalic = 0x01; constexpr uint8_t kStyleBold = 0x02; constexpr uint8_t kKnownStyleFlags = kStyleItalic | kStyleBold; -constexpr uint8_t kSidecarVersion = 1; +constexpr uint8_t kSidecarVersion1 = 1; +constexpr uint8_t kSidecarVersion2 = 2; +constexpr uint8_t kTimingDouble = 2; +constexpr uint8_t kTimingTriple = 3; constexpr size_t kSidecarHeaderBytes = 18; uint32_t readUint32Le(const std::vector& bytes, size_t offset) { @@ -122,6 +125,7 @@ bool Reader::load(const String& book, const std::vector& sidecar) { styleTokens_.clear(); paragraphStarts_.clear(); index_ = 0; + nextStyleToken_ = 0; sidecarValid_ = false; bool inWord = false; @@ -161,10 +165,12 @@ bool Reader::loadSidecar(const std::vector& sidecar) { if (sidecar.size() < kSidecarHeaderBytes || sidecar[0] != 'R' || sidecar[1] != 'S' || sidecar[2] != 'V' || sidecar[3] != 'P' || - sidecar[4] != kSidecarVersion || sidecar[5] != 0 || + (sidecar[4] != kSidecarVersion1 && sidecar[4] != kSidecarVersion2) || + sidecar[5] != 0 || readUint64Le(sidecar, 6) != sourceFingerprint(book_)) { return false; } + const uint8_t sidecarVersion = sidecar[4]; const uint32_t recordCount = readUint32Le(sidecar, 14); if (recordCount > tokens_.size()) return false; @@ -187,8 +193,12 @@ bool Reader::loadSidecar(const std::vector& sidecar) { } const uint8_t styleFlags = sidecar[offset++]; const uint8_t timingModifier = sidecar[offset++]; - if (wordIndex >= tokens_.size() || styleFlags == 0 || - (styleFlags & ~kKnownStyleFlags) != 0) { + if (wordIndex >= tokens_.size() || + (styleFlags & ~kKnownStyleFlags) != 0 || + (timingModifier != 0 && timingModifier != kTimingDouble && + timingModifier != kTimingTriple) || + (sidecarVersion == kSidecarVersion1 && timingModifier != 0) || + (styleFlags == 0 && timingModifier == 0)) { return false; } records.push_back({wordIndex, styleFlags, timingModifier}); @@ -197,6 +207,7 @@ bool Reader::loadSidecar(const std::vector& sidecar) { if (offset != sidecar.size()) return false; styleTokens_.swap(records); + nextStyleToken_ = 0; return true; } @@ -204,6 +215,7 @@ void Reader::restoreAtParagraphStart(uint32_t savedWord) { if (tokens_.empty()) return; index_ = savedWord < tokens_.size() ? savedWord : tokens_.size() - 1; index_ = paragraphStarts_[tokens_[index_].paragraph]; + seekStyleToken(); } const String Reader::currentWord() const { @@ -215,6 +227,10 @@ const String Reader::currentWord() const { bool Reader::advance() { if (index_ + 1 >= tokens_.size()) return false; ++index_; + while (nextStyleToken_ < styleTokens_.size() && + styleTokens_[nextStyleToken_].index < index_) { + ++nextStyleToken_; + } return true; } @@ -223,9 +239,11 @@ bool Reader::previousParagraph() { const uint32_t paragraph = tokens_[index_].paragraph; if (paragraph == 0) { index_ = 0; + seekStyleToken(); return false; } index_ = paragraphStarts_[paragraph - 1]; + seekStyleToken(); return true; } @@ -234,12 +252,14 @@ bool Reader::nextParagraph() { const uint32_t paragraph = tokens_[index_].paragraph; if (paragraph + 1 >= paragraphStarts_.size()) return false; index_ = paragraphStarts_[paragraph + 1]; + seekStyleToken(); return true; } bool Reader::lastParagraph() { if (paragraphStarts_.empty()) return false; index_ = paragraphStarts_.back(); + seekStyleToken(); return true; } @@ -255,17 +275,26 @@ uint8_t Reader::progressPercent() const { lastIndex); } -uint8_t Reader::currentStyleFlags() const { return styleFlagsAt(index_); } - -uint8_t Reader::styleFlagsAt(uint32_t wordIndex) const { +void Reader::seekStyleToken() { const auto record = std::lower_bound( - styleTokens_.begin(), styleTokens_.end(), wordIndex, + styleTokens_.begin(), styleTokens_.end(), index_, [](const StyleToken& token, uint32_t index) { return token.index < index; }); - return record != styleTokens_.end() && record->index == wordIndex - ? record->styleFlags - : 0; + nextStyleToken_ = record - styleTokens_.begin(); +} + +const StyleToken* Reader::currentStyleToken() const { + if (nextStyleToken_ >= styleTokens_.size() || + styleTokens_[nextStyleToken_].index != index_) { + return nullptr; + } + return &styleTokens_[nextStyleToken_]; +} + +uint8_t Reader::currentStyleFlags() const { + const StyleToken* token = currentStyleToken(); + return token ? token->styleFlags : 0; } bool Reader::sidecarValid() const { return sidecarValid_; } @@ -289,5 +318,9 @@ uint32_t Reader::delayMs(uint16_t wpm) const { } else if (trailingLineBreaks == 1) { multiplier = max(multiplier, static_cast(2)); } + const StyleToken* styleToken = currentStyleToken(); + if (styleToken) { + multiplier = max(multiplier, styleToken->timingModifier); + } return delay * multiplier; } diff --git a/tests/test_epub_to_chapters.py b/tests/test_epub_to_chapters.py index cedcb03..e2eb1f6 100644 --- a/tests/test_epub_to_chapters.py +++ b/tests/test_epub_to_chapters.py @@ -10,6 +10,9 @@ from epub_to_chapters import ( SIDECAR_VERSION, STYLE_BOLD, STYLE_ITALIC, + TIMING_DOUBLE, + TIMING_TRIPLE, + add_unfamiliar_word_timing, decode_styled_text, document_root, html_to_text, @@ -136,6 +139,19 @@ class StyleSidecarTests(unittest.TestCase): ]), ) + def test_writes_timing_only_records(self): + text = "Blarenptnip" + + with tempfile.TemporaryDirectory() as temporary_directory: + sidecar_file = Path(temporary_directory) / "Chapter 1.rsvp" + write_style_sidecar(sidecar_file, text, [(0, 0, TIMING_TRIPLE)]) + data = sidecar_file.read_bytes() + + self.assertEqual( + data[SIDECAR_HEADER.size:], + bytes([0, 0, TIMING_TRIPLE]), + ) + def test_extracts_styles_from_css_classes_and_inline_css(self): class FakeItem: def __init__(self, name, content): @@ -184,5 +200,39 @@ class StyleSidecarTests(unittest.TestCase): ) +class UnfamiliarWordTimingTests(unittest.TestCase): + def test_marks_first_ten_globally_and_first_word_in_each_chapter(self): + chapters = [ + { + "text": " ".join(["Blarenptnip"] * 11), + "style_records": [(0, STYLE_ITALIC, 0)], + }, + { + "text": "Blarenptnip", + "style_records": [], + }, + ] + + add_unfamiliar_word_timing(chapters, {"the", "reader"}) + + self.assertEqual( + chapters[0]["style_records"], + [(0, STYLE_ITALIC, TIMING_TRIPLE)] + + [(index, 0, TIMING_TRIPLE) for index in range(1, 10)] + + [(10, 0, TIMING_DOUBLE)], + ) + self.assertEqual(chapters[1]["style_records"], [(0, 0, TIMING_TRIPLE)]) + + def test_english_words_and_non_words_are_not_marked(self): + chapters = [{ + "text": "The reader has 123 pages.", + "style_records": [], + }] + + add_unfamiliar_word_timing(chapters, {"the", "reader", "has", "pages"}) + + self.assertEqual(chapters[0]["style_records"], []) + + if __name__ == "__main__": unittest.main()