Shrink token from 16b to 8b, improving memory usage

README.md

@@ -111,8 +111,9 @@ with EPUB styles, punctuation, and repeated invented names:
 ```
 
 The output directory must be empty unless `--overwrite` is supplied. Copy the
-resulting `Example Book` directory into `/books` on the card. A
-section and its `.rsvp` sidecar may each be up to 1 MiB. The converter writes
+resulting `Example Book` directory into `/books` on the card. A section is
+limited to 150,000 words; its source text and `.rsvp` sidecar may each be up
+to 1 MiB. The converter writes
 `CONVERSION_REPORT.txt` in the book directory. It records any unsupported
 non-ASCII characters left after
 normalizing common punctuation, ligatures, accents, whitespace, fractions, and
@@ -255,8 +256,9 @@ cloud account, or synchronization feature.
 
 ## Limits of the first MVP
 
-- It reads sections up to 1 MiB from `/books/<book>/<section>.txt`, in the
-  order declared by that book's `TOC.txt`.
+- It reads sections up to 150,000 words and 1 MiB from
+  `/books/<book>/<section>.txt`, in the order declared by that book's
+  `TOC.txt`.
 - The built-in display font is best for printable ASCII text. UTF-8 text files
   are accepted, but accented/non-Latin characters need a Unicode font addition
   before they will render correctly.

TODO.md

@@ -14,16 +14,17 @@ profile that preserves the current cadence.
 
 ## Converter preflight and safe section splitting
 
-Add a `--check` mode that reports section and sidecar sizes, filename issues,
-and unsupported characters without writing output. Offer optional splitting at
-paragraph boundaries for sections that exceed the device's 1 MiB file limit.
+Add a `--check` mode that reports section word counts, section and sidecar
+sizes, filename issues, and unsupported characters without writing output.
+Offer optional splitting at paragraph boundaries for sections that exceed the
+device's 150,000-word or 1 MiB file limits.
 
 ## Memory-limit hardware validation
 
-Stress-test the 1 MiB section and sidecar limits on real hardware, record PSRAM
-headroom, and lower the supported limit if token indexing or sidecar metadata
-leaves insufficient memory for reliable reading. Include adversarial short-word
-and dense-sidecar inputs, not just ordinary prose.
+Stress-test the 150,000-word and 1 MiB section/sidecar limits on real hardware,
+record PSRAM headroom, and lower the supported limit if token indexing or
+sidecar metadata leaves insufficient memory for reliable reading. Include
+adversarial short-word and dense-sidecar inputs, not just ordinary prose.
 
 ## Reading continuity preference
 

epub_to_chapters.py

@@ -91,24 +91,6 @@ BLOCK_TAGS = {
     "h6",
 }
 
-# These are useful TOC entries but generally aren't chapters.
-FRONT_MATTER = {
-    "contents",
-    "table of contents",
-    "toc",
-    "cover",
-    "title page",
-    "copyright",
-    "copyright page",
-    "dedication",
-    "epigraph",
-    "acknowledgments",
-    "acknowledgements",
-    "about the author",
-    "also by",
-    "colophon",
-}
-
 FAT_FORBIDDEN_FILENAME_CHARACTERS = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
 
 STYLE_ITALIC = 0x01
@@ -123,8 +105,10 @@ CSS_RULE = re.compile(r"([^{}]+)\{([^{}]*)\}")
 CSS_CLASS = re.compile(r"\.([A-Za-z_-][A-Za-z0-9_-]*)")
 WORD_LOOKUP_KEY = re.compile(r"^[^A-Za-z]*([A-Za-z]+(?:['-][A-Za-z]+)*)[^A-Za-z]*$")
 MAX_SECTION_BYTES = 1024 * 1024
-MAX_TOKEN_BYTES = 256 * 1024
+MAX_SECTION_WORDS = 150_000
+MAX_TOKEN_BYTES = 65_535
 MAX_SECTION_NAME_BYTES = 200
+RESERVED_SECTION_NAMES = {"toc", "conversion_report"}
 
 TIMING_NONE = 0
 TIMING_DOUBLE = 2
@@ -335,9 +319,13 @@ def lookup_word_key(token):
 
 
 def validate_token_lengths(text):
-    """Reject a malformed section containing an impractically long token."""
+    """Reject sections whose token index would exceed the device memory budget."""
 
     for word_number, match in enumerate(re.finditer(r"\S+", text), start=1):
+        if word_number > MAX_SECTION_WORDS:
+            raise ValueError(
+                f"section exceeds the {MAX_SECTION_WORDS} word limit"
+            )
         if len(match.group().encode("utf-8")) > MAX_TOKEN_BYTES:
             raise ValueError(
                 f"token {word_number} exceeds the {MAX_TOKEN_BYTES} byte limit"
@@ -1397,7 +1385,7 @@ def convert_epub(epub_path, output_dir, overwrite=False):
 
     toc_entries = []
     prepared_sections = []
-    used_names = set()
+    used_names = set(RESERVED_SECTION_NAMES)
 
     for number, chapter in enumerate(
         chapters,
@@ -1418,7 +1406,7 @@ def convert_epub(epub_path, output_dir, overwrite=False):
         try:
             validate_token_lengths(chapter["text"])
         except ValueError as exc:
-            print(f"Error: {title!r} contains an invalid token: {exc}")
+            print(f"Error: {title!r} exceeds reader limits: {exc}")
             sys.exit(1)
         sidecar_bytes = style_sidecar_bytes(
             chapter["text"], chapter["style_records"]

include/Config.h

@@ -24,9 +24,13 @@ constexpr uint8_t kButtonNext = 16;
 constexpr char kBooksDirectory[] = "/books";
 // Source sections and their optional RSVP sidecars are loaded into memory.
 constexpr size_t kMaxSectionBytes = 1024 * 1024;
-// A single token can be a long URL or identifier, but should never consume a
-// material fraction of the reader's working memory.
-constexpr uint32_t kMaxTokenBytes = 256 * 1024;
+// Each token occupies eight bytes. Limiting a section to 150,000 tokens keeps
+// its token index near 1.2 MiB, leaving PSRAM headroom for source text,
+// sidecar metadata, and the display runtime.
+constexpr uint32_t kMaxSectionWords = 150000;
+// Token lengths are stored in uint16_t. A 64 KiB token is already much longer
+// than the display can usefully render.
+constexpr uint16_t kMaxTokenBytes = UINT16_MAX;
 
 constexpr uint16_t kDefaultWpm = 300;
 constexpr uint16_t kMinimumWpm = 100;

include/Reader.h

@@ -5,10 +5,10 @@
 
 struct Token {
   uint32_t offset;
-  uint32_t length;
-  uint32_t paragraph;
+  uint16_t length;
   uint8_t trailingLineBreaks;
 };
+static_assert(sizeof(Token) == 8, "Token storage must remain memory-budgeted");
 
 struct StyleToken {
   uint32_t index;
@@ -42,6 +42,7 @@ class Reader {
   uint64_t sourceFingerprint_ = 0;
 
   bool loadSidecar(const std::vector<uint8_t>& sidecar);
+  uint32_t paragraphStartFor(uint32_t wordIndex) const;
   void seekStyleToken();
   const StyleToken* currentStyleToken() const;
 };

src/Reader.cpp

@@ -154,13 +154,14 @@ bool Reader::load(const String& book, const std::vector<uint8_t>& sidecar) {
     }
     if (whitespace && inWord) {
       const uint32_t wordLength = i - wordStart;
-      if (wordLength > config::kMaxTokenBytes) {
+      if (wordLength > config::kMaxTokenBytes ||
+          tokens_.size() >= config::kMaxSectionWords) {
         tokens_.clear();
         paragraphStarts_.clear();
         return false;
       }
-      tokens_.push_back({wordStart, wordLength,
-                         static_cast<uint32_t>(paragraphStarts_.size() - 1), 0});
+      tokens_.push_back(
+          {wordStart, static_cast<uint16_t>(wordLength), 0});
       inWord = false;
     }
     if (c == '\n') {
@@ -225,7 +226,7 @@ bool Reader::loadSidecar(const std::vector<uint8_t>& sidecar) {
 void Reader::restoreAtParagraphStart(uint32_t savedWord) {
   if (tokens_.empty()) return;
   index_ = savedWord < tokens_.size() ? savedWord : tokens_.size() - 1;
-  index_ = paragraphStarts_[tokens_[index_].paragraph];
+  index_ = paragraphStartFor(index_);
   seekStyleToken();
 }
 
@@ -247,22 +248,25 @@ bool Reader::advance() {
 
 bool Reader::previousParagraph() {
   if (tokens_.empty()) return false;
-  const uint32_t paragraph = tokens_[index_].paragraph;
-  if (paragraph == 0) {
+  const auto nextParagraph = std::upper_bound(
+      paragraphStarts_.begin(), paragraphStarts_.end(), index_);
+  const auto currentParagraph = std::prev(nextParagraph);
+  if (currentParagraph == paragraphStarts_.begin()) {
     index_ = 0;
     seekStyleToken();
     return false;
   }
-  index_ = paragraphStarts_[paragraph - 1];
+  index_ = *std::prev(currentParagraph);
   seekStyleToken();
   return true;
 }
 
 bool Reader::nextParagraph() {
   if (tokens_.empty()) return false;
-  const uint32_t paragraph = tokens_[index_].paragraph;
-  if (paragraph + 1 >= paragraphStarts_.size()) return false;
-  index_ = paragraphStarts_[paragraph + 1];
+  const auto nextParagraph = std::upper_bound(
+      paragraphStarts_.begin(), paragraphStarts_.end(), index_);
+  if (nextParagraph == paragraphStarts_.end()) return false;
+  index_ = *nextParagraph;
   seekStyleToken();
   return true;
 }
@@ -286,6 +290,12 @@ uint8_t Reader::progressPercent() const {
                               lastIndex);
 }
 
+uint32_t Reader::paragraphStartFor(uint32_t wordIndex) const {
+  const auto nextParagraph = std::upper_bound(
+      paragraphStarts_.begin(), paragraphStarts_.end(), wordIndex);
+  return *std::prev(nextParagraph);
+}
+
 void Reader::seekStyleToken() {
   const auto record = std::lower_bound(
       styleTokens_.begin(), styleTokens_.end(), index_,

src/main.cpp

@@ -503,7 +503,7 @@ void loop() {
   handleTouch();
   handleInput();
 
-  if (playing && millis() >= nextWordMs) {
+  if (playing && static_cast<int32_t>(millis() - nextWordMs) >= 0) {
     if (!reader.advance()) {
       String nextChapterPath;
       if (library.nextChapter(state.bookPath, nextChapterPath) &&

tests/test_epub_to_chapters.py

@@ -13,6 +13,7 @@ from epub_to_chapters import (
     TIMING_DOUBLE,
     TIMING_TRIPLE,
     MAX_TOKEN_BYTES,
+    MAX_SECTION_WORDS,
     add_unfamiliar_word_timing,
     decode_styled_text,
     document_root,
@@ -67,6 +68,15 @@ class DisplayTextNormalizationTests(unittest.TestCase):
 
         self.assertLessEqual(len(name.encode("utf-8")), 200)
 
+    def test_section_names_do_not_collide_with_generated_files(self):
+        used_names = {"toc", "conversion_report"}
+
+        self.assertEqual(output_section_name("TOC", 1, used_names), "TOC (2)")
+        self.assertEqual(
+            output_section_name("CONVERSION_REPORT", 2, used_names),
+            "CONVERSION_REPORT (2)",
+        )
+
 
 class TokenValidationTests(unittest.TestCase):
     def test_allows_a_token_at_the_configured_byte_limit(self):
@@ -76,6 +86,10 @@ class TokenValidationTests(unittest.TestCase):
         with self.assertRaises(ValueError):
             validate_token_lengths("a" * (MAX_TOKEN_BYTES + 1))
 
+    def test_rejects_sections_with_too_many_words(self):
+        with self.assertRaises(ValueError):
+            validate_token_lengths("a " * (MAX_SECTION_WORDS + 1))
+
 
 class ConversionReportTests(unittest.TestCase):
     def test_reports_remaining_non_ascii_characters_by_generated_file(self):