Better unsupported character conversion

README.md

@@ -114,13 +114,12 @@ The output directory must be empty unless `--overwrite` is supplied. Copy the
 resulting `Example Book` directory into `/books` on the card. A section is
 limited to 150,000 words; its source text and `.rsvp` sidecar may each be up
 to 1 MiB. The converter writes
-`CONVERSION_REPORT.txt` in the book directory. It records any unsupported
+`CONVERSION_REPORT.txt` in the book directory. If unsupported non-ASCII
-non-ASCII characters left after
+characters remain after normalizing common punctuation, ligatures, accents,
-normalizing common punctuation, ligatures, accents, whitespace, fractions, and
+whitespace, fractions, and ornamental breaks, the converter asks once for an
-ornamental breaks. Review that report before copying the book: transliterate or
+ASCII replacement for each character. The replacement is used everywhere that
-replace reported characters deliberately, or defer the book until a
+character occurs; press Enter for a blank replacement. The report records the
-Unicode-capable font is available. The included display font is best for
+chosen substitutions. The included display font is best for printable ASCII.
-printable ASCII.
 
 The converter also writes a compact `<section>.rsvp` file alongside every
 section. Its versioned, delta-encoded records preserve italic and bold EPUB

epub_to_chapters.py

@@ -485,7 +485,70 @@ def remaining_non_ascii(text):
     return Counter(character for character in text if ord(character) > 127)
 
 
-def write_conversion_report(output_dir, toc_entries, written_sections):
+def apply_substitutions(text, substitutions):
+    """Replace each unsupported character using its chosen ASCII text."""
+
+    return "".join(substitutions.get(character, character) for character in text)
+
+
+def prompt_for_substitutions(characters, input_fn=input, output_fn=print):
+    """Ask once for the printable-ASCII replacement of each character."""
+
+    substitutions = {}
+    for character in sorted(characters, key=ord):
+        codepoint = f"U+{ord(character):04X}"
+        name = unicodedata.name(character, "UNNAMED")
+        count = characters[character]
+        prompt = (
+            f"Replace {codepoint} {name} ({count} occurrence"
+            f"{'s' if count != 1 else ''}; {character!r}) with ASCII "
+            "text, or press Enter to remove it: "
+        )
+        while True:
+            try:
+                replacement = input_fn(prompt)
+            except EOFError as exc:
+                raise RuntimeError(
+                    f"input ended before a replacement was supplied for {codepoint}"
+                ) from exc
+            if all(32 <= ord(replacement_character) <= 126
+                   for replacement_character in replacement):
+                substitutions[character] = replacement
+                break
+            output_fn("Replacement must contain printable ASCII characters only.")
+    return substitutions
+
+
+def substitute_unsupported_characters(chapters, input_fn=input, output_fn=print):
+    """Interactively make generated section titles and text ASCII-safe.
+
+    Text is changed before style records and timing metadata are generated, so
+    replacements that add or remove characters cannot invalidate word indexes.
+    """
+
+    characters = Counter()
+    for chapter in chapters:
+        chapter["title"] = normalize_display_text(chapter["title"] or "")
+        chapter["text"] = normalize_text(chapter["text"])
+        characters.update(remaining_non_ascii(chapter["title"]))
+        characters.update(remaining_non_ascii(chapter["text"]))
+
+    if not characters:
+        return {}
+
+    output_fn()
+    output_fn("Unsupported characters need an ASCII replacement for this book.")
+    substitutions = prompt_for_substitutions(characters, input_fn, output_fn)
+    for chapter in chapters:
+        chapter["title"] = apply_substitutions(chapter["title"], substitutions)
+        chapter["text"] = normalize_text(
+            apply_substitutions(chapter["text"], substitutions)
+        )
+    return substitutions
+
+
+def write_conversion_report(output_dir, toc_entries, written_sections,
+                            substitutions=None):
     """Write a report for unsupported characters left in generated text."""
 
     occurrences = defaultdict(Counter)
@@ -506,6 +569,18 @@ def write_conversion_report(output_dir, toc_entries, written_sections):
         "The reader's current font supports printable ASCII only.",
     ]
 
+    if substitutions:
+        lines.extend([
+            "",
+            "User substitutions applied to all occurrences:",
+        ])
+        for character in sorted(substitutions, key=ord):
+            codepoint = f"U+{ord(character):04X}"
+            name = unicodedata.name(character, "UNNAMED")
+            lines.append(
+                f"{codepoint} {name} -> {substitutions[character]!r}"
+            )
+
     if not occurrences:
         lines.extend([
             "",
@@ -1344,6 +1419,12 @@ def convert_epub(epub_path, output_dir, overwrite=False):
             class_styles,
         )
 
+    try:
+        substitutions = substitute_unsupported_characters(chapters)
+    except RuntimeError as exc:
+        print(f"Error: {exc}")
+        sys.exit(1)
+
     for chapter in chapters:
         text, style_records = decode_styled_text(
             normalize_text(chapter["text"])
@@ -1439,6 +1520,7 @@ def convert_epub(epub_path, output_dir, overwrite=False):
         output_dir,
         toc_entries,
         written_sections,
+        substitutions,
     )
 
     print()

src/main.cpp

@@ -256,6 +256,7 @@ void togglePlayback() {
 }
 
 void navigatePreviousPaused() {
+  bool positionChanged = false;
   switch (readerView) {
     case ReaderView::kChapterTitle: {
       String previousChapterPath;
@@ -264,12 +265,15 @@ void navigatePreviousPaused() {
         reader.lastParagraph();
         readerView = ReaderView::kWord;
         showReaderScreen();
+        positionChanged = true;
       }
       break;
     }
     case ReaderView::kWord:
       if (!reader.previousParagraph()) {
         readerView = ReaderView::kChapterTitle;
+      } else {
+        positionChanged = true;
       }
       showReaderScreen();
       break;
@@ -279,12 +283,15 @@ void navigatePreviousPaused() {
       if (reader.lastParagraph()) {
         readerView = ReaderView::kWord;
         showReaderScreen();
+        positionChanged = true;
       }
       break;
   }
+  if (positionChanged) persistState();
 }
 
 void navigateNextPaused() {
+  bool positionChanged = false;
   switch (readerView) {
     case ReaderView::kChapterTitle:
       reader.restoreAtParagraphStart(state.wordIndex);
@@ -294,6 +301,7 @@ void navigateNextPaused() {
     case ReaderView::kWord: {
       if (reader.nextParagraph()) {
         showReaderScreen();
+        positionChanged = true;
         break;
       }
 
@@ -301,6 +309,7 @@ void navigateNextPaused() {
       if (library.nextChapter(state.bookPath, nextChapterPath) &&
           loadChapter(nextChapterPath)) {
         showReaderScreen();
+        positionChanged = true;
       } else {
         readerView = ReaderView::kTheEnd;
         showReaderScreen();
@@ -310,6 +319,7 @@ void navigateNextPaused() {
     case ReaderView::kTheEnd:
       break;
   }
+  if (positionChanged) persistState();
 }
 
 void onTouchHome(void*) {

tests/test_epub_to_chapters.py

@@ -10,6 +10,7 @@ from epub_to_chapters import (
     SIDECAR_VERSION,
     STYLE_BOLD,
     STYLE_ITALIC,
+    STYLE_MARKER,
     TIMING_DOUBLE,
     TIMING_TRIPLE,
     MAX_TOKEN_BYTES,
@@ -24,8 +25,10 @@ from epub_to_chapters import (
     normalize_text,
     output_section_name,
     parse_document,
+    prompt_for_substitutions,
     source_fingerprint,
     stylesheet_class_styles,
+    substitute_unsupported_characters,
     validate_token_lengths,
     write_style_sidecar,
     write_conversion_report,
@@ -113,12 +116,68 @@ class ConversionReportTests(unittest.TestCase):
                 Path(temporary_directory),
                 ["Chapter 1"],
                 [("Chapter 1.txt", "Plain ASCII text.")],
+                {"\u03a9": "Omega", "\u0416": ""},
             )
 
             report = report_file.read_text(encoding="utf-8")
 
         self.assertFalse(has_unsupported)
         self.assertIn("all generated TOC and chapter text is ASCII-safe", report)
+        self.assertIn("U+03A9 GREEK CAPITAL LETTER OMEGA -> 'Omega'", report)
+        self.assertIn("U+0416 CYRILLIC CAPITAL LETTER ZHE -> ''", report)
+
+
+class UnsupportedCharacterSubstitutionTests(unittest.TestCase):
+    def test_replaces_each_character_everywhere_and_allows_blank_removal(self):
+        chapters = [{
+            "title": "Chapter \u03a9",
+            "text": "\u03a9mega and \u0416",
+        }]
+        replacements = iter(["Omega", ""])
+
+        substitutions = substitute_unsupported_characters(
+            chapters,
+            input_fn=lambda prompt: next(replacements),
+            output_fn=lambda *args: None,
+        )
+
+        self.assertEqual(substitutions, {"\u03a9": "Omega", "\u0416": ""})
+        self.assertEqual(chapters[0]["title"], "Chapter Omega")
+        self.assertEqual(chapters[0]["text"], "Omegamega and")
+
+    def test_reprompts_until_replacement_is_printable_ascii(self):
+        replacements = iter(["\u00e9", "plain text"])
+        messages = []
+
+        substitutions = prompt_for_substitutions(
+            {"\u03a9": 1},
+            input_fn=lambda prompt: next(replacements),
+            output_fn=messages.append,
+        )
+
+        self.assertEqual(substitutions, {"\u03a9": "plain text"})
+        self.assertEqual(messages, [
+            "Replacement must contain printable ASCII characters only.",
+        ])
+
+    def test_replacement_keeps_style_markers_ready_for_retokensizing(self):
+        chapters = [{
+            "title": "",
+            "text": f"{STYLE_MARKER}1alpha\u03a9beta{STYLE_MARKER}0",
+        }]
+
+        substitute_unsupported_characters(
+            chapters,
+            input_fn=lambda prompt: " ",
+            output_fn=lambda *args: None,
+        )
+        text, records = decode_styled_text(chapters[0]["text"])
+
+        self.assertEqual(text, "alpha beta")
+        self.assertEqual(records, [
+            (0, STYLE_ITALIC, 0),
+            (1, STYLE_ITALIC, 0),
+        ])
 
 
 class StyleSidecarTests(unittest.TestCase):