From b977f07297b536a0ad48ec8510aefe61cea23711 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 23 Sep 2026 01:28:09 -0500 Subject: [PATCH] Shrink token from 16b to 8b, improving memory usage --- README.md | 10 ++++++---- TODO.md | 15 ++++++++------- epub_to_chapters.py | 32 ++++++++++---------------------- include/Config.h | 10 +++++++--- include/Reader.h | 5 +++-- src/Reader.cpp | 30 ++++++++++++++++++++---------- src/main.cpp | 2 +- tests/test_epub_to_chapters.py | 14 ++++++++++++++ 8 files changed, 69 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index a0e9642..2c31442 100644 --- a/README.md +++ b/README.md @@ -111,8 +111,9 @@ with EPUB styles, punctuation, and repeated invented names: ``` The output directory must be empty unless `--overwrite` is supplied. Copy the -resulting `Example Book` directory into `/books` on the card. A -section and its `.rsvp` sidecar may each be up to 1 MiB. The converter writes +resulting `Example Book` directory into `/books` on the card. A section is +limited to 150,000 words; its source text and `.rsvp` sidecar may each be up +to 1 MiB. The converter writes `CONVERSION_REPORT.txt` in the book directory. It records any unsupported non-ASCII characters left after normalizing common punctuation, ligatures, accents, whitespace, fractions, and @@ -255,8 +256,9 @@ cloud account, or synchronization feature. ## Limits of the first MVP -- It reads sections up to 1 MiB from `/books//
.txt`, in the - order declared by that book's `TOC.txt`. +- It reads sections up to 150,000 words and 1 MiB from + `/books//
.txt`, in the order declared by that book's + `TOC.txt`. - The built-in display font is best for printable ASCII text. UTF-8 text files are accepted, but accented/non-Latin characters need a Unicode font addition before they will render correctly. diff --git a/TODO.md b/TODO.md index 258194a..504aef0 100644 --- a/TODO.md +++ b/TODO.md @@ -14,16 +14,17 @@ profile that preserves the current cadence. ## Converter preflight and safe section splitting -Add a `--check` mode that reports section and sidecar sizes, filename issues, -and unsupported characters without writing output. Offer optional splitting at -paragraph boundaries for sections that exceed the device's 1 MiB file limit. +Add a `--check` mode that reports section word counts, section and sidecar +sizes, filename issues, and unsupported characters without writing output. +Offer optional splitting at paragraph boundaries for sections that exceed the +device's 150,000-word or 1 MiB file limits. ## Memory-limit hardware validation -Stress-test the 1 MiB section and sidecar limits on real hardware, record PSRAM -headroom, and lower the supported limit if token indexing or sidecar metadata -leaves insufficient memory for reliable reading. Include adversarial short-word -and dense-sidecar inputs, not just ordinary prose. +Stress-test the 150,000-word and 1 MiB section/sidecar limits on real hardware, +record PSRAM headroom, and lower the supported limit if token indexing or +sidecar metadata leaves insufficient memory for reliable reading. Include +adversarial short-word and dense-sidecar inputs, not just ordinary prose. ## Reading continuity preference diff --git a/epub_to_chapters.py b/epub_to_chapters.py index 030c1fc..8ab2411 100644 --- a/epub_to_chapters.py +++ b/epub_to_chapters.py @@ -91,24 +91,6 @@ BLOCK_TAGS = { "h6", } -# These are useful TOC entries but generally aren't chapters. -FRONT_MATTER = { - "contents", - "table of contents", - "toc", - "cover", - "title page", - "copyright", - "copyright page", - "dedication", - "epigraph", - "acknowledgments", - "acknowledgements", - "about the author", - "also by", - "colophon", -} - FAT_FORBIDDEN_FILENAME_CHARACTERS = re.compile(r'[<>:"/\\|?*\x00-\x1f]') STYLE_ITALIC = 0x01 @@ -123,8 +105,10 @@ CSS_RULE = re.compile(r"([^{}]+)\{([^{}]*)\}") CSS_CLASS = re.compile(r"\.([A-Za-z_-][A-Za-z0-9_-]*)") WORD_LOOKUP_KEY = re.compile(r"^[^A-Za-z]*([A-Za-z]+(?:['-][A-Za-z]+)*)[^A-Za-z]*$") MAX_SECTION_BYTES = 1024 * 1024 -MAX_TOKEN_BYTES = 256 * 1024 +MAX_SECTION_WORDS = 150_000 +MAX_TOKEN_BYTES = 65_535 MAX_SECTION_NAME_BYTES = 200 +RESERVED_SECTION_NAMES = {"toc", "conversion_report"} TIMING_NONE = 0 TIMING_DOUBLE = 2 @@ -335,9 +319,13 @@ def lookup_word_key(token): def validate_token_lengths(text): - """Reject a malformed section containing an impractically long token.""" + """Reject sections whose token index would exceed the device memory budget.""" for word_number, match in enumerate(re.finditer(r"\S+", text), start=1): + if word_number > MAX_SECTION_WORDS: + raise ValueError( + f"section exceeds the {MAX_SECTION_WORDS} word limit" + ) if len(match.group().encode("utf-8")) > MAX_TOKEN_BYTES: raise ValueError( f"token {word_number} exceeds the {MAX_TOKEN_BYTES} byte limit" @@ -1397,7 +1385,7 @@ def convert_epub(epub_path, output_dir, overwrite=False): toc_entries = [] prepared_sections = [] - used_names = set() + used_names = set(RESERVED_SECTION_NAMES) for number, chapter in enumerate( chapters, @@ -1418,7 +1406,7 @@ def convert_epub(epub_path, output_dir, overwrite=False): try: validate_token_lengths(chapter["text"]) except ValueError as exc: - print(f"Error: {title!r} contains an invalid token: {exc}") + print(f"Error: {title!r} exceeds reader limits: {exc}") sys.exit(1) sidecar_bytes = style_sidecar_bytes( chapter["text"], chapter["style_records"] diff --git a/include/Config.h b/include/Config.h index 36f0615..8f2072c 100644 --- a/include/Config.h +++ b/include/Config.h @@ -24,9 +24,13 @@ constexpr uint8_t kButtonNext = 16; constexpr char kBooksDirectory[] = "/books"; // Source sections and their optional RSVP sidecars are loaded into memory. constexpr size_t kMaxSectionBytes = 1024 * 1024; -// A single token can be a long URL or identifier, but should never consume a -// material fraction of the reader's working memory. -constexpr uint32_t kMaxTokenBytes = 256 * 1024; +// Each token occupies eight bytes. Limiting a section to 150,000 tokens keeps +// its token index near 1.2 MiB, leaving PSRAM headroom for source text, +// sidecar metadata, and the display runtime. +constexpr uint32_t kMaxSectionWords = 150000; +// Token lengths are stored in uint16_t. A 64 KiB token is already much longer +// than the display can usefully render. +constexpr uint16_t kMaxTokenBytes = UINT16_MAX; constexpr uint16_t kDefaultWpm = 300; constexpr uint16_t kMinimumWpm = 100; diff --git a/include/Reader.h b/include/Reader.h index be7d277..c8778f2 100644 --- a/include/Reader.h +++ b/include/Reader.h @@ -5,10 +5,10 @@ struct Token { uint32_t offset; - uint32_t length; - uint32_t paragraph; + uint16_t length; uint8_t trailingLineBreaks; }; +static_assert(sizeof(Token) == 8, "Token storage must remain memory-budgeted"); struct StyleToken { uint32_t index; @@ -42,6 +42,7 @@ class Reader { uint64_t sourceFingerprint_ = 0; bool loadSidecar(const std::vector& sidecar); + uint32_t paragraphStartFor(uint32_t wordIndex) const; void seekStyleToken(); const StyleToken* currentStyleToken() const; }; diff --git a/src/Reader.cpp b/src/Reader.cpp index d70fe03..271d1f8 100644 --- a/src/Reader.cpp +++ b/src/Reader.cpp @@ -154,13 +154,14 @@ bool Reader::load(const String& book, const std::vector& sidecar) { } if (whitespace && inWord) { const uint32_t wordLength = i - wordStart; - if (wordLength > config::kMaxTokenBytes) { + if (wordLength > config::kMaxTokenBytes || + tokens_.size() >= config::kMaxSectionWords) { tokens_.clear(); paragraphStarts_.clear(); return false; } - tokens_.push_back({wordStart, wordLength, - static_cast(paragraphStarts_.size() - 1), 0}); + tokens_.push_back( + {wordStart, static_cast(wordLength), 0}); inWord = false; } if (c == '\n') { @@ -225,7 +226,7 @@ bool Reader::loadSidecar(const std::vector& sidecar) { void Reader::restoreAtParagraphStart(uint32_t savedWord) { if (tokens_.empty()) return; index_ = savedWord < tokens_.size() ? savedWord : tokens_.size() - 1; - index_ = paragraphStarts_[tokens_[index_].paragraph]; + index_ = paragraphStartFor(index_); seekStyleToken(); } @@ -247,22 +248,25 @@ bool Reader::advance() { bool Reader::previousParagraph() { if (tokens_.empty()) return false; - const uint32_t paragraph = tokens_[index_].paragraph; - if (paragraph == 0) { + const auto nextParagraph = std::upper_bound( + paragraphStarts_.begin(), paragraphStarts_.end(), index_); + const auto currentParagraph = std::prev(nextParagraph); + if (currentParagraph == paragraphStarts_.begin()) { index_ = 0; seekStyleToken(); return false; } - index_ = paragraphStarts_[paragraph - 1]; + index_ = *std::prev(currentParagraph); seekStyleToken(); return true; } bool Reader::nextParagraph() { if (tokens_.empty()) return false; - const uint32_t paragraph = tokens_[index_].paragraph; - if (paragraph + 1 >= paragraphStarts_.size()) return false; - index_ = paragraphStarts_[paragraph + 1]; + const auto nextParagraph = std::upper_bound( + paragraphStarts_.begin(), paragraphStarts_.end(), index_); + if (nextParagraph == paragraphStarts_.end()) return false; + index_ = *nextParagraph; seekStyleToken(); return true; } @@ -286,6 +290,12 @@ uint8_t Reader::progressPercent() const { lastIndex); } +uint32_t Reader::paragraphStartFor(uint32_t wordIndex) const { + const auto nextParagraph = std::upper_bound( + paragraphStarts_.begin(), paragraphStarts_.end(), wordIndex); + return *std::prev(nextParagraph); +} + void Reader::seekStyleToken() { const auto record = std::lower_bound( styleTokens_.begin(), styleTokens_.end(), index_, diff --git a/src/main.cpp b/src/main.cpp index d71e63e..edbde9d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -503,7 +503,7 @@ void loop() { handleTouch(); handleInput(); - if (playing && millis() >= nextWordMs) { + if (playing && static_cast(millis() - nextWordMs) >= 0) { if (!reader.advance()) { String nextChapterPath; if (library.nextChapter(state.bookPath, nextChapterPath) && diff --git a/tests/test_epub_to_chapters.py b/tests/test_epub_to_chapters.py index 24bd54e..4fa4a42 100644 --- a/tests/test_epub_to_chapters.py +++ b/tests/test_epub_to_chapters.py @@ -13,6 +13,7 @@ from epub_to_chapters import ( TIMING_DOUBLE, TIMING_TRIPLE, MAX_TOKEN_BYTES, + MAX_SECTION_WORDS, add_unfamiliar_word_timing, decode_styled_text, document_root, @@ -67,6 +68,15 @@ class DisplayTextNormalizationTests(unittest.TestCase): self.assertLessEqual(len(name.encode("utf-8")), 200) + def test_section_names_do_not_collide_with_generated_files(self): + used_names = {"toc", "conversion_report"} + + self.assertEqual(output_section_name("TOC", 1, used_names), "TOC (2)") + self.assertEqual( + output_section_name("CONVERSION_REPORT", 2, used_names), + "CONVERSION_REPORT (2)", + ) + class TokenValidationTests(unittest.TestCase): def test_allows_a_token_at_the_configured_byte_limit(self): @@ -76,6 +86,10 @@ class TokenValidationTests(unittest.TestCase): with self.assertRaises(ValueError): validate_token_lengths("a" * (MAX_TOKEN_BYTES + 1)) + def test_rejects_sections_with_too_many_words(self): + with self.assertRaises(ValueError): + validate_token_lengths("a " * (MAX_SECTION_WORDS + 1)) + class ConversionReportTests(unittest.TestCase): def test_reports_remaining_non_ascii_characters_by_generated_file(self):