diff --git a/README.md b/README.md index 6dbf5b8..d2a51b8 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,10 @@ operating systems: /TOC.txt` with a listed section file. +contains at least one `/books//TOC.txt` with a listed section. If the board is not detected or an upload repeatedly fails, force download mode: @@ -227,9 +232,9 @@ GPIO12 wakes it. The reader saves the selected section, paragraph-level position, WPM, brightness, and font settings in the ESP32's nonvolatile storage. On the next boot or wake it opens the same section's title card and remains paused; -pressing the touch-edge Home button begins at the saved paragraph. It validates -the saved section against its `TOC.txt` before opening it, then falls back to -the first available book if the card has changed. +pressing the touch-edge Home button begins at the saved paragraph. It opens the +saved section directly, then falls back to the first TOC-listed section if the +card has changed. When continuous reading reaches the end of a section, it opens the next section's title card and pauses. Press the touch-edge Home button to begin that diff --git a/epub_to_chapters.py b/epub_to_chapters.py index b1023ab..684465c 100644 --- a/epub_to_chapters.py +++ b/epub_to_chapters.py @@ -314,10 +314,112 @@ def decode_styled_text(text): def lookup_word_key(token): """Return a lower-case dictionary key for one RSVP token, if it has one.""" - match = WORD_LOOKUP_KEY.fullmatch(token) + match = WORD_LOOKUP_KEY.fullmatch(token.replace("\u2019", "'")) return match.group(1).lower() if match else None +IRREGULAR_WORD_BASES = { + "ain't": "be", + "can't": "can", + "did": "do", + "does": "do", + "had": "have", + "has": "have", + "says": "say", + "shan't": "shall", + "won't": "will", +} + +CONTRACTION_SUBJECTS = { + "he", "here", "how", "i", "it", "let", "she", "that", "there", "they", + "we", "what", "when", "where", "who", "why", "you", +} +MINIMUM_ENGLISH_ZIPF_FREQUENCY = 2.0 + + +class EnglishVocabulary: + """Membership view combining a dictionary with common-word frequency.""" + + def __init__(self, words, zipf_frequency): + self.words = words + self.zipf_frequency = zipf_frequency + + def __contains__(self, word): + return ( + word in self.words or + self.zipf_frequency(word, "en") >= MINIMUM_ENGLISH_ZIPF_FREQUENCY + ) + + +def english_word_bases(word): + """Yield plausible dictionary forms for a normalized English word. + + ``english-words``' web2 list contains many stems but omits common + inflections and contractions. These conservative transformations prevent + those ordinary forms from being mistaken for unfamiliar vocabulary without + turning an unknown stem into a known word. + """ + + stems = [word] + if word.endswith("n't"): + stems.append(word[:-3]) + elif word.endswith(("'d", "'ll", "'m", "'re", "'s", "'ve")): + stems.append(word[:-2]) + elif word.endswith("s'"): + stems.append(word[:-1]) + + for stem in stems: + yield stem + irregular_base = IRREGULAR_WORD_BASES.get(stem) + if irregular_base: + yield irregular_base + + if stem.endswith("ies") and len(stem) > 3: + yield stem[:-3] + "y" + if stem.endswith("ing") and len(stem) > 4: + root = stem[:-3] + yield root + yield root + "e" + if len(root) > 2 and root[-1] == root[-2]: + yield root[:-1] + if stem.endswith("ed") and len(stem) > 3: + root = stem[:-2] + yield root + yield stem[:-1] + if len(root) > 2 and root[-1] == root[-2]: + yield root[:-1] + if stem.endswith(("er", "est")) and len(stem) > 3: + suffix_length = 2 if stem.endswith("er") else 3 + root = stem[:-suffix_length] + yield root + if root.endswith("i"): + yield root[:-1] + "y" + if len(root) > 2 and root[-1] == root[-2]: + yield root[:-1] + if stem.endswith("es") and len(stem) > 3: + yield stem[:-2] + yield stem[:-1] + elif stem.endswith("s") and len(stem) > 2: + yield stem[:-1] + + +def is_familiar_english_word(word, english_words): + """Return whether a lookup key is familiar English or a normal variant.""" + + if ( + word.endswith(("'d", "'ll", "'m", "'re", "'s", "'ve")) and + word[:-2] in CONTRACTION_SUBJECTS + ): + return True + if "-" in word: + components = word.split("-") + return all( + component and is_familiar_english_word(component, english_words) + for component in components + ) + return any(candidate in english_words for candidate in english_word_bases(word)) + + def validate_token_lengths(text): """Reject sections whose token index would exceed the device memory budget.""" @@ -337,15 +439,20 @@ def load_english_dictionary(): try: from english_words import get_english_words_set + from wordfreq import zipf_frequency except ModuleNotFoundError as exc: raise RuntimeError( - "Unfamiliar-word timing needs english-words. Install converter " + "Unfamiliar-word timing needs english-words and wordfreq. Install converter " "dependencies with: python3 -m pip install -r requirements.txt" ) from exc # web2 is a general-purpose English dictionary. Proper nouns and invented - # words are normally absent, which deliberately makes them qualify. - return get_english_words_set(["web2"], lower=True) + # words are normally absent, which deliberately makes them qualify. The + # frequency fallback fills web2's gaps for ordinary inflections and words. + return EnglishVocabulary( + get_english_words_set(["web2"], lower=True), + zipf_frequency, + ) def add_unfamiliar_word_timing(chapters, english_words): @@ -368,7 +475,7 @@ def add_unfamiliar_word_timing(chapters, english_words): for word_index, match in enumerate(re.finditer(r"\S+", chapter["text"])): key = lookup_word_key(match.group()) - if key is None or key in english_words: + if key is None or is_familiar_english_word(key, english_words): continue prior_occurrences = global_occurrences[key] diff --git a/include/Library.h b/include/Library.h index c250eff..3077f0f 100644 --- a/include/Library.h +++ b/include/Library.h @@ -18,14 +18,12 @@ class Library { private: bool ensureBooks() const; bool ensureChapters(const String& directoryPath) const; - bool isListedChapter(const String& chapterPath) const; bool firstChapter(const String& directoryPath, String& chapterPath) const; bool loadFile(const String& path, String& contents) const; bool firstBook(String& path) const; mutable bool booksCached_ = false; mutable std::vector books_; - mutable std::vector firstChapters_; mutable String chapterDirectory_; mutable std::vector chapters_; mutable std::vector chapterTitles_; diff --git a/include/tft_setup.h b/include/tft_setup.h new file mode 100644 index 0000000..7f7defc --- /dev/null +++ b/include/tft_setup.h @@ -0,0 +1,34 @@ +#pragma once + +// LilyGO T-Display-S3-Pro: ST7796, 222 x 480, SPI. +// TFT_eSPI automatically includes this project-local file before its default +// setup, keeping the board configuration independent of external libraries. +#define USER_SETUP_LOADED +#define USER_SETUP_ID 214 + +#define ST7796_DRIVER +#define TFT_WIDTH 222 +#define TFT_HEIGHT 480 +#define CGRAM_OFFSET +#define TFT_RGB_ORDER TFT_BGR +#define TFT_INVERSION_ON + +#define TFT_MISO 8 +#define TFT_MOSI 17 +#define TFT_SCLK 18 +#define TFT_CS 39 +#define TFT_DC 9 +#define TFT_RST 47 +#define TFT_BL 48 +#define TFT_BACKLIGHT_ON HIGH + +#define LOAD_GLCD +#define LOAD_FONT2 +#define LOAD_FONT4 +#define LOAD_FONT6 +#define LOAD_FONT7 +#define LOAD_FONT8 +#define LOAD_GFXFF +#define SMOOTH_FONT + +#define SPI_FREQUENCY 40000000 diff --git a/platformio.ini b/platformio.ini index c16717d..b663ec3 100644 --- a/platformio.ini +++ b/platformio.ini @@ -11,6 +11,7 @@ board_build.partitions = partitions.csv build_flags = -DARDUINO_USB_CDC_ON_BOOT=1 -DDISABLE_ALL_LIBRARY_WARNINGS + -include$PROJECT_DIR/include/tft_setup.h lib_deps = TFT_eSPI@2.5.43 lewisxhe/SensorLib@0.4.0 diff --git a/requirements.txt b/requirements.txt index f332b82..c74bf67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ ebooklib==0.20 beautifulsoup4==4.15.0 english-words==2.0.2 +wordfreq==3.1.1 diff --git a/src/Library.cpp b/src/Library.cpp index 061d070..57ef43a 100644 --- a/src/Library.cpp +++ b/src/Library.cpp @@ -22,8 +22,9 @@ bool Library::loadLastOrFirst(String& path, String& contents) const { Settings settings; settings.begin(); const SavedState state = settings.load(); - if (isChapterPath(state.bookPath) && isListedChapter(state.bookPath) && - loadFile(state.bookPath, contents)) { + // Loading the saved file verifies that it still exists. Avoid walking every + // TOC entry at boot; that turns a resume into one SD-card open per chapter. + if (isChapterPath(state.bookPath) && loadFile(state.bookPath, contents)) { path = state.bookPath; return true; } @@ -31,7 +32,7 @@ bool Library::loadLastOrFirst(String& path, String& contents) const { } bool Library::load(const String& path, String& contents) const { - return isChapterPath(path) && isListedChapter(path) && loadFile(path, contents); + return isChapterPath(path) && loadFile(path, contents); } bool Library::loadSidecar(const String& chapterPath, @@ -108,45 +109,15 @@ bool listChapters(const String& path, std::vector& chapters, title.trim(); if (title.isEmpty() || title.indexOf('/') >= 0 || title.indexOf('\\') >= 0) continue; - const String chapterPath = path + "/" + title + ".txt"; - File chapter = SD.open(chapterPath, FILE_READ); - const bool validChapter = chapter && !chapter.isDirectory(); - chapter.close(); - if (!validChapter) continue; - - chapters.push_back(chapterPath); + // TOC.txt is the source of truth for both chapter order and membership. + // Opening every listed file here makes navigation scale with book length. + chapters.push_back(path + "/" + title + ".txt"); titles.push_back(title); } toc.close(); return !chapters.empty(); } -bool firstListedChapter(const String& path, String& chapterPath) { - File toc = SD.open(path + "/TOC.txt", FILE_READ); - if (!toc || toc.isDirectory()) return false; - - while (toc.available()) { - String title = toc.readStringUntil('\n'); - title.replace("\r", ""); - title.trim(); - if (title.isEmpty() || title.indexOf('/') >= 0 || title.indexOf('\\') >= 0) { - continue; - } - - const String candidate = path + "/" + title + ".txt"; - File chapter = SD.open(candidate, FILE_READ); - const bool validChapter = chapter && !chapter.isDirectory(); - chapter.close(); - if (validChapter) { - chapterPath = candidate; - toc.close(); - return true; - } - } - toc.close(); - return false; -} - bool cycle(const std::vector& choices, const String& current, int direction, String& choice) { if (choices.empty()) return false; @@ -240,10 +211,7 @@ bool Library::ensureBooks() const { booksCached_ = true; books_.clear(); - firstChapters_.clear(); - if (!listDirectories(config::kBooksDirectory, books_)) return false; - firstChapters_.resize(books_.size()); - return true; + return listDirectories(config::kBooksDirectory, books_); } bool Library::ensureChapters(const String& directoryPath) const { @@ -255,29 +223,9 @@ bool Library::ensureChapters(const String& directoryPath) const { return listChapters(chapterDirectory_, chapters_, chapterTitles_); } -bool Library::isListedChapter(const String& chapterPath) const { - const String directoryPath = parentPath(chapterPath); - if (directoryPath.isEmpty() || !ensureChapters(directoryPath)) return false; - for (const String& path : chapters_) { - if (path == chapterPath) return true; - } - return false; -} - bool Library::firstChapter(const String& directoryPath, String& chapterPath) const { - size_t bookIndex = books_.size(); - for (size_t index = 0; index < books_.size(); ++index) { - if (books_[index] == directoryPath) { - bookIndex = index; - break; - } - } - if (bookIndex < firstChapters_.size() && !firstChapters_[bookIndex].isEmpty()) { - chapterPath = firstChapters_[bookIndex]; - return true; - } - if (!firstListedChapter(directoryPath, chapterPath)) return false; - if (bookIndex < firstChapters_.size()) firstChapters_[bookIndex] = chapterPath; + if (!ensureChapters(directoryPath)) return false; + chapterPath = chapters_.front(); return true; } diff --git a/src/main.cpp b/src/main.cpp index 42a5585..b6c86c0 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -349,8 +349,9 @@ void confirmMenuSelection() { void beginTouch() { touch.setPins(config::kTouchReset, config::kTouchInterrupt); - touchAvailable = touch.begin(Wire, CST226SE_SLAVE_ADDRESS, - config::kI2cSda, config::kI2cScl); + // Board::begin() has already initialized the shared I2C bus on these pins. + // Passing -1 prevents the touch library from trying to reassign them. + touchAvailable = touch.begin(Wire, CST226SE_SLAVE_ADDRESS, -1, -1); if (!touchAvailable) return; configureTouchLandscape(); @@ -367,9 +368,7 @@ void configureTouchLandscape() { void handleTouch() { if (!touchAvailable) return; - int16_t x; - int16_t y; - touch.getPoint(&x, &y); // Polling delivers home-button callbacks. + touch.getTouchPoints(); // Polling delivers home-button callbacks. if (touchPlayPauseRequested) { touchPlayPauseRequested = false; confirmMenuSelection(); diff --git a/tests/test_epub_to_chapters.py b/tests/test_epub_to_chapters.py index b8bb0f1..4748896 100644 --- a/tests/test_epub_to_chapters.py +++ b/tests/test_epub_to_chapters.py @@ -15,6 +15,7 @@ from epub_to_chapters import ( TIMING_TRIPLE, MAX_TOKEN_BYTES, MAX_SECTION_WORDS, + EnglishVocabulary, add_unfamiliar_word_timing, decode_styled_text, document_root, @@ -324,6 +325,35 @@ class UnfamiliarWordTimingTests(unittest.TestCase): self.assertEqual(chapters[0]["style_records"], []) + def test_common_variants_of_known_words_are_not_marked(self): + chapters = [{ + "text": ( + "asked walked eyes using he'd couldn't didn't hasn't that's " + "father's stories thicker life-threatening well-dressed " + "Blarenptnip" + ), + "style_records": [], + }] + english_words = { + "ask", "walk", "eye", "use", "he", "could", "do", "have", + "that", "father", "story", "thick", "life", "threaten", "well", + "dress", + } + + add_unfamiliar_word_timing(chapters, english_words) + + self.assertEqual(chapters[0]["style_records"], [(14, 0, TIMING_TRIPLE)]) + + def test_frequency_fallback_accepts_common_word_missing_from_web2(self): + vocabulary = EnglishVocabulary( + {"reader"}, + lambda word, language: 2.1 if word == "mudflat" else 0, + ) + + self.assertIn("reader", vocabulary) + self.assertIn("mudflat", vocabulary) + self.assertNotIn("blarenptnip", vocabulary) + class SameDocumentTocTests(unittest.TestCase): def test_extracts_each_fragment_when_two_toc_entries_share_a_document(self):