update unique word classification

README.md

@@ -70,8 +70,10 @@ operating systems: <https://docs.platformio.org/en/latest/core/installation/meth
    `TOC.txt`, UTF-8 text files for its sections, and matching `.rsvp` sidecars
    generated by the converter.
 
-`TOC.txt` controls both section order and title-card text. Each non-empty line
+`TOC.txt` is the reader's source of truth for section order, membership, and
-names a section and its file is that name plus `.txt`.
+title-card text. Each non-empty line names a section and its file is that name
+plus `.txt`. The reader reads and caches this file when it needs navigation
+order; it opens only the selected section file.
 
 The resulting card must look like this:
 
@@ -126,8 +128,10 @@ section. Its versioned, delta-encoded records preserve italic and bold EPUB
 markup—including semantic tags, CSS classes, and inline CSS—and reading-timing
 metadata. The fingerprint covers the source `.txt` bytes, before the device
 applies display-only punctuation normalization. During conversion, the
-desktop-only English dictionary marks each word not in that dictionary. The
+desktop-only English vocabulary classifier marks each unfamiliar word. It
-first ten appearances of each marked word in
+combines a dictionary with word-frequency data and recognizes normal English
+inflections, contractions, possessives, and hyphenated compounds. The first
+ten appearances of each marked word in
 the book, plus its first appearance in every section, use a 3× delay. Later
 appearances use a 2× delay. The reader uses the strongest applicable delay, so
 sentence and paragraph pauses are never compounded with unfamiliar-word
@@ -136,10 +140,11 @@ and safely ignores it if it is missing, stale, or malformed. Valid italic and
 bold records select matching reader-font variants; the device never performs
 dictionary or SD-card lookups while advancing words.
 
-The converter requires the desktop-only `english-words` package listed in
+The converter requires the desktop-only `english-words` and `wordfreq`
-`requirements.txt`; the reader does not need this dependency. Reconvert a book
+packages listed in `requirements.txt`; the reader does not need either
-to create v2 sidecars with unfamiliar-word timing. Existing v1 style-only
+dependency. Reconvert a book to create v2 sidecars with unfamiliar-word
-sidecars remain readable but do not contain timing metadata.
+timing. Existing v1 style-only sidecars remain readable but do not contain
+timing metadata.
 
 Eject the card safely and insert it in the board before powering the board.
 
@@ -197,7 +202,7 @@ example, use `--upload-port COM3`.
 
 After a successful upload, the display shows the current section title and is
 **paused**. If it shows `NO BOOKS`, check that the card is inserted, FAT32, and
-contains at least one `/books/<book>/TOC.txt` with a listed section file.
+contains at least one `/books/<book>/TOC.txt` with a listed section.
 
 If the board is not detected or an upload repeatedly fails, force download mode:
 
@@ -227,9 +232,9 @@ GPIO12 wakes it.
 The reader saves the selected section, paragraph-level position, WPM,
 brightness, and font settings in the ESP32's nonvolatile storage. On the next
 boot or wake it opens the same section's title card and remains paused;
-pressing the touch-edge Home button begins at the saved paragraph. It validates
+pressing the touch-edge Home button begins at the saved paragraph. It opens the
-the saved section against its `TOC.txt` before opening it, then falls back to
+saved section directly, then falls back to the first TOC-listed section if the
-the first available book if the card has changed.
+card has changed.
 
 When continuous reading reaches the end of a section, it opens the next
 section's title card and pauses. Press the touch-edge Home button to begin that

epub_to_chapters.py

@@ -314,10 +314,112 @@ def decode_styled_text(text):
 def lookup_word_key(token):
     """Return a lower-case dictionary key for one RSVP token, if it has one."""
 
-    match = WORD_LOOKUP_KEY.fullmatch(token)
+    match = WORD_LOOKUP_KEY.fullmatch(token.replace("\u2019", "'"))
     return match.group(1).lower() if match else None
 
 
+IRREGULAR_WORD_BASES = {
+    "ain't": "be",
+    "can't": "can",
+    "did": "do",
+    "does": "do",
+    "had": "have",
+    "has": "have",
+    "says": "say",
+    "shan't": "shall",
+    "won't": "will",
+}
+
+CONTRACTION_SUBJECTS = {
+    "he", "here", "how", "i", "it", "let", "she", "that", "there", "they",
+    "we", "what", "when", "where", "who", "why", "you",
+}
+MINIMUM_ENGLISH_ZIPF_FREQUENCY = 2.0
+
+
+class EnglishVocabulary:
+    """Membership view combining a dictionary with common-word frequency."""
+
+    def __init__(self, words, zipf_frequency):
+        self.words = words
+        self.zipf_frequency = zipf_frequency
+
+    def __contains__(self, word):
+        return (
+            word in self.words or
+            self.zipf_frequency(word, "en") >= MINIMUM_ENGLISH_ZIPF_FREQUENCY
+        )
+
+
+def english_word_bases(word):
+    """Yield plausible dictionary forms for a normalized English word.
+
+    ``english-words``' web2 list contains many stems but omits common
+    inflections and contractions.  These conservative transformations prevent
+    those ordinary forms from being mistaken for unfamiliar vocabulary without
+    turning an unknown stem into a known word.
+    """
+
+    stems = [word]
+    if word.endswith("n't"):
+        stems.append(word[:-3])
+    elif word.endswith(("'d", "'ll", "'m", "'re", "'s", "'ve")):
+        stems.append(word[:-2])
+    elif word.endswith("s'"):
+        stems.append(word[:-1])
+
+    for stem in stems:
+        yield stem
+        irregular_base = IRREGULAR_WORD_BASES.get(stem)
+        if irregular_base:
+            yield irregular_base
+
+        if stem.endswith("ies") and len(stem) > 3:
+            yield stem[:-3] + "y"
+        if stem.endswith("ing") and len(stem) > 4:
+            root = stem[:-3]
+            yield root
+            yield root + "e"
+            if len(root) > 2 and root[-1] == root[-2]:
+                yield root[:-1]
+        if stem.endswith("ed") and len(stem) > 3:
+            root = stem[:-2]
+            yield root
+            yield stem[:-1]
+            if len(root) > 2 and root[-1] == root[-2]:
+                yield root[:-1]
+        if stem.endswith(("er", "est")) and len(stem) > 3:
+            suffix_length = 2 if stem.endswith("er") else 3
+            root = stem[:-suffix_length]
+            yield root
+            if root.endswith("i"):
+                yield root[:-1] + "y"
+            if len(root) > 2 and root[-1] == root[-2]:
+                yield root[:-1]
+        if stem.endswith("es") and len(stem) > 3:
+            yield stem[:-2]
+            yield stem[:-1]
+        elif stem.endswith("s") and len(stem) > 2:
+            yield stem[:-1]
+
+
+def is_familiar_english_word(word, english_words):
+    """Return whether a lookup key is familiar English or a normal variant."""
+
+    if (
+        word.endswith(("'d", "'ll", "'m", "'re", "'s", "'ve")) and
+        word[:-2] in CONTRACTION_SUBJECTS
+    ):
+        return True
+    if "-" in word:
+        components = word.split("-")
+        return all(
+            component and is_familiar_english_word(component, english_words)
+            for component in components
+        )
+    return any(candidate in english_words for candidate in english_word_bases(word))
+
+
 def validate_token_lengths(text):
     """Reject sections whose token index would exceed the device memory budget."""
 
@@ -337,15 +439,20 @@ def load_english_dictionary():
 
     try:
         from english_words import get_english_words_set
+        from wordfreq import zipf_frequency
     except ModuleNotFoundError as exc:
         raise RuntimeError(
-            "Unfamiliar-word timing needs english-words. Install converter "
+            "Unfamiliar-word timing needs english-words and wordfreq. Install converter "
             "dependencies with: python3 -m pip install -r requirements.txt"
         ) from exc
 
     # web2 is a general-purpose English dictionary. Proper nouns and invented
-    # words are normally absent, which deliberately makes them qualify.
+    # words are normally absent, which deliberately makes them qualify. The
-    return get_english_words_set(["web2"], lower=True)
+    # frequency fallback fills web2's gaps for ordinary inflections and words.
+    return EnglishVocabulary(
+        get_english_words_set(["web2"], lower=True),
+        zipf_frequency,
+    )
 
 
 def add_unfamiliar_word_timing(chapters, english_words):
@@ -368,7 +475,7 @@ def add_unfamiliar_word_timing(chapters, english_words):
 
         for word_index, match in enumerate(re.finditer(r"\S+", chapter["text"])):
             key = lookup_word_key(match.group())
-            if key is None or key in english_words:
+            if key is None or is_familiar_english_word(key, english_words):
                 continue
 
             prior_occurrences = global_occurrences[key]

include/Library.h

@@ -18,14 +18,12 @@ class Library {
  private:
   bool ensureBooks() const;
   bool ensureChapters(const String& directoryPath) const;
-  bool isListedChapter(const String& chapterPath) const;
   bool firstChapter(const String& directoryPath, String& chapterPath) const;
   bool loadFile(const String& path, String& contents) const;
   bool firstBook(String& path) const;
 
   mutable bool booksCached_ = false;
   mutable std::vector<String> books_;
-  mutable std::vector<String> firstChapters_;
   mutable String chapterDirectory_;
   mutable std::vector<String> chapters_;
   mutable std::vector<String> chapterTitles_;

include/tft_setup.h

@@ -0,0 +1,34 @@
+#pragma once
+
+// LilyGO T-Display-S3-Pro: ST7796, 222 x 480, SPI.
+// TFT_eSPI automatically includes this project-local file before its default
+// setup, keeping the board configuration independent of external libraries.
+#define USER_SETUP_LOADED
+#define USER_SETUP_ID 214
+
+#define ST7796_DRIVER
+#define TFT_WIDTH 222
+#define TFT_HEIGHT 480
+#define CGRAM_OFFSET
+#define TFT_RGB_ORDER TFT_BGR
+#define TFT_INVERSION_ON
+
+#define TFT_MISO 8
+#define TFT_MOSI 17
+#define TFT_SCLK 18
+#define TFT_CS 39
+#define TFT_DC 9
+#define TFT_RST 47
+#define TFT_BL 48
+#define TFT_BACKLIGHT_ON HIGH
+
+#define LOAD_GLCD
+#define LOAD_FONT2
+#define LOAD_FONT4
+#define LOAD_FONT6
+#define LOAD_FONT7
+#define LOAD_FONT8
+#define LOAD_GFXFF
+#define SMOOTH_FONT
+
+#define SPI_FREQUENCY 40000000

platformio.ini

@@ -11,6 +11,7 @@ board_build.partitions = partitions.csv
 build_flags =
   -DARDUINO_USB_CDC_ON_BOOT=1
   -DDISABLE_ALL_LIBRARY_WARNINGS
+  -include$PROJECT_DIR/include/tft_setup.h
 lib_deps =
   TFT_eSPI@2.5.43
   lewisxhe/SensorLib@0.4.0

requirements.txt

@@ -1,3 +1,4 @@
 ebooklib==0.20
 beautifulsoup4==4.15.0
 english-words==2.0.2
+wordfreq==3.1.1

src/Library.cpp

@@ -22,8 +22,9 @@ bool Library::loadLastOrFirst(String& path, String& contents) const {
   Settings settings;
   settings.begin();
   const SavedState state = settings.load();
-  if (isChapterPath(state.bookPath) && isListedChapter(state.bookPath) &&
+  // Loading the saved file verifies that it still exists. Avoid walking every
-      loadFile(state.bookPath, contents)) {
+  // TOC entry at boot; that turns a resume into one SD-card open per chapter.
+  if (isChapterPath(state.bookPath) && loadFile(state.bookPath, contents)) {
     path = state.bookPath;
     return true;
   }
@@ -31,7 +32,7 @@ bool Library::loadLastOrFirst(String& path, String& contents) const {
 }
 
 bool Library::load(const String& path, String& contents) const {
-  return isChapterPath(path) && isListedChapter(path) && loadFile(path, contents);
+  return isChapterPath(path) && loadFile(path, contents);
 }
 
 bool Library::loadSidecar(const String& chapterPath,
@@ -108,45 +109,15 @@ bool listChapters(const String& path, std::vector<String>& chapters,
     title.trim();
     if (title.isEmpty() || title.indexOf('/') >= 0 || title.indexOf('\\') >= 0) continue;
 
-    const String chapterPath = path + "/" + title + ".txt";
+    // TOC.txt is the source of truth for both chapter order and membership.
-    File chapter = SD.open(chapterPath, FILE_READ);
+    // Opening every listed file here makes navigation scale with book length.
-    const bool validChapter = chapter && !chapter.isDirectory();
+    chapters.push_back(path + "/" + title + ".txt");
-    chapter.close();
-    if (!validChapter) continue;
-
-    chapters.push_back(chapterPath);
     titles.push_back(title);
   }
   toc.close();
   return !chapters.empty();
 }
 
-bool firstListedChapter(const String& path, String& chapterPath) {
-  File toc = SD.open(path + "/TOC.txt", FILE_READ);
-  if (!toc || toc.isDirectory()) return false;
-
-  while (toc.available()) {
-    String title = toc.readStringUntil('\n');
-    title.replace("\r", "");
-    title.trim();
-    if (title.isEmpty() || title.indexOf('/') >= 0 || title.indexOf('\\') >= 0) {
-      continue;
-    }
-
-    const String candidate = path + "/" + title + ".txt";
-    File chapter = SD.open(candidate, FILE_READ);
-    const bool validChapter = chapter && !chapter.isDirectory();
-    chapter.close();
-    if (validChapter) {
-      chapterPath = candidate;
-      toc.close();
-      return true;
-    }
-  }
-  toc.close();
-  return false;
-}
-
 bool cycle(const std::vector<String>& choices, const String& current,
            int direction, String& choice) {
   if (choices.empty()) return false;
@@ -240,10 +211,7 @@ bool Library::ensureBooks() const {
 
   booksCached_ = true;
   books_.clear();
-  firstChapters_.clear();
+  return listDirectories(config::kBooksDirectory, books_);
-  if (!listDirectories(config::kBooksDirectory, books_)) return false;
-  firstChapters_.resize(books_.size());
-  return true;
 }
 
 bool Library::ensureChapters(const String& directoryPath) const {
@@ -255,29 +223,9 @@ bool Library::ensureChapters(const String& directoryPath) const {
   return listChapters(chapterDirectory_, chapters_, chapterTitles_);
 }
 
-bool Library::isListedChapter(const String& chapterPath) const {
-  const String directoryPath = parentPath(chapterPath);
-  if (directoryPath.isEmpty() || !ensureChapters(directoryPath)) return false;
-  for (const String& path : chapters_) {
-    if (path == chapterPath) return true;
-  }
-  return false;
-}
-
 bool Library::firstChapter(const String& directoryPath, String& chapterPath) const {
-  size_t bookIndex = books_.size();
+  if (!ensureChapters(directoryPath)) return false;
-  for (size_t index = 0; index < books_.size(); ++index) {
+  chapterPath = chapters_.front();
-    if (books_[index] == directoryPath) {
-      bookIndex = index;
-      break;
-    }
-  }
-  if (bookIndex < firstChapters_.size() && !firstChapters_[bookIndex].isEmpty()) {
-    chapterPath = firstChapters_[bookIndex];
-    return true;
-  }
-  if (!firstListedChapter(directoryPath, chapterPath)) return false;
-  if (bookIndex < firstChapters_.size()) firstChapters_[bookIndex] = chapterPath;
   return true;
 }
 

src/main.cpp

@@ -349,8 +349,9 @@ void confirmMenuSelection() {
 
 void beginTouch() {
   touch.setPins(config::kTouchReset, config::kTouchInterrupt);
-  touchAvailable = touch.begin(Wire, CST226SE_SLAVE_ADDRESS,
+  // Board::begin() has already initialized the shared I2C bus on these pins.
-                               config::kI2cSda, config::kI2cScl);
+  // Passing -1 prevents the touch library from trying to reassign them.
+  touchAvailable = touch.begin(Wire, CST226SE_SLAVE_ADDRESS, -1, -1);
   if (!touchAvailable) return;
 
   configureTouchLandscape();
@@ -367,9 +368,7 @@ void configureTouchLandscape() {
 
 void handleTouch() {
   if (!touchAvailable) return;
-  int16_t x;
+  touch.getTouchPoints();  // Polling delivers home-button callbacks.
-  int16_t y;
-  touch.getPoint(&x, &y);  // Polling delivers home-button callbacks.
   if (touchPlayPauseRequested) {
     touchPlayPauseRequested = false;
     confirmMenuSelection();

tests/test_epub_to_chapters.py

@@ -15,6 +15,7 @@ from epub_to_chapters import (
     TIMING_TRIPLE,
     MAX_TOKEN_BYTES,
     MAX_SECTION_WORDS,
+    EnglishVocabulary,
     add_unfamiliar_word_timing,
     decode_styled_text,
     document_root,
@@ -324,6 +325,35 @@ class UnfamiliarWordTimingTests(unittest.TestCase):
 
         self.assertEqual(chapters[0]["style_records"], [])
 
+    def test_common_variants_of_known_words_are_not_marked(self):
+        chapters = [{
+            "text": (
+                "asked walked eyes using he'd couldn't didn't hasn't that's "
+                "father's stories thicker life-threatening well-dressed "
+                "Blarenptnip"
+            ),
+            "style_records": [],
+        }]
+        english_words = {
+            "ask", "walk", "eye", "use", "he", "could", "do", "have",
+            "that", "father", "story", "thick", "life", "threaten", "well",
+            "dress",
+        }
+
+        add_unfamiliar_word_timing(chapters, english_words)
+
+        self.assertEqual(chapters[0]["style_records"], [(14, 0, TIMING_TRIPLE)])
+
+    def test_frequency_fallback_accepts_common_word_missing_from_web2(self):
+        vocabulary = EnglishVocabulary(
+            {"reader"},
+            lambda word, language: 2.1 if word == "mudflat" else 0,
+        )
+
+        self.assertIn("reader", vocabulary)
+        self.assertIn("mudflat", vocabulary)
+        self.assertNotIn("blarenptnip", vocabulary)
+
 
 class SameDocumentTocTests(unittest.TestCase):
     def test_extracts_each_fragment_when_two_toc_entries_share_a_document(self):