diff --git a/README.md b/README.md index 2c31442..6dbf5b8 100644 --- a/README.md +++ b/README.md @@ -114,13 +114,12 @@ The output directory must be empty unless `--overwrite` is supplied. Copy the resulting `Example Book` directory into `/books` on the card. A section is limited to 150,000 words; its source text and `.rsvp` sidecar may each be up to 1 MiB. The converter writes -`CONVERSION_REPORT.txt` in the book directory. It records any unsupported -non-ASCII characters left after -normalizing common punctuation, ligatures, accents, whitespace, fractions, and -ornamental breaks. Review that report before copying the book: transliterate or -replace reported characters deliberately, or defer the book until a -Unicode-capable font is available. The included display font is best for -printable ASCII. +`CONVERSION_REPORT.txt` in the book directory. If unsupported non-ASCII +characters remain after normalizing common punctuation, ligatures, accents, +whitespace, fractions, and ornamental breaks, the converter asks once for an +ASCII replacement for each character. The replacement is used everywhere that +character occurs; press Enter for a blank replacement. The report records the +chosen substitutions. The included display font is best for printable ASCII. The converter also writes a compact `
.rsvp` file alongside every section. Its versioned, delta-encoded records preserve italic and bold EPUB diff --git a/epub_to_chapters.py b/epub_to_chapters.py index 8ab2411..b1023ab 100644 --- a/epub_to_chapters.py +++ b/epub_to_chapters.py @@ -485,7 +485,70 @@ def remaining_non_ascii(text): return Counter(character for character in text if ord(character) > 127) -def write_conversion_report(output_dir, toc_entries, written_sections): +def apply_substitutions(text, substitutions): + """Replace each unsupported character using its chosen ASCII text.""" + + return "".join(substitutions.get(character, character) for character in text) + + +def prompt_for_substitutions(characters, input_fn=input, output_fn=print): + """Ask once for the printable-ASCII replacement of each character.""" + + substitutions = {} + for character in sorted(characters, key=ord): + codepoint = f"U+{ord(character):04X}" + name = unicodedata.name(character, "UNNAMED") + count = characters[character] + prompt = ( + f"Replace {codepoint} {name} ({count} occurrence" + f"{'s' if count != 1 else ''}; {character!r}) with ASCII " + "text, or press Enter to remove it: " + ) + while True: + try: + replacement = input_fn(prompt) + except EOFError as exc: + raise RuntimeError( + f"input ended before a replacement was supplied for {codepoint}" + ) from exc + if all(32 <= ord(replacement_character) <= 126 + for replacement_character in replacement): + substitutions[character] = replacement + break + output_fn("Replacement must contain printable ASCII characters only.") + return substitutions + + +def substitute_unsupported_characters(chapters, input_fn=input, output_fn=print): + """Interactively make generated section titles and text ASCII-safe. + + Text is changed before style records and timing metadata are generated, so + replacements that add or remove characters cannot invalidate word indexes. + """ + + characters = Counter() + for chapter in chapters: + chapter["title"] = normalize_display_text(chapter["title"] or "") + chapter["text"] = normalize_text(chapter["text"]) + characters.update(remaining_non_ascii(chapter["title"])) + characters.update(remaining_non_ascii(chapter["text"])) + + if not characters: + return {} + + output_fn() + output_fn("Unsupported characters need an ASCII replacement for this book.") + substitutions = prompt_for_substitutions(characters, input_fn, output_fn) + for chapter in chapters: + chapter["title"] = apply_substitutions(chapter["title"], substitutions) + chapter["text"] = normalize_text( + apply_substitutions(chapter["text"], substitutions) + ) + return substitutions + + +def write_conversion_report(output_dir, toc_entries, written_sections, + substitutions=None): """Write a report for unsupported characters left in generated text.""" occurrences = defaultdict(Counter) @@ -506,6 +569,18 @@ def write_conversion_report(output_dir, toc_entries, written_sections): "The reader's current font supports printable ASCII only.", ] + if substitutions: + lines.extend([ + "", + "User substitutions applied to all occurrences:", + ]) + for character in sorted(substitutions, key=ord): + codepoint = f"U+{ord(character):04X}" + name = unicodedata.name(character, "UNNAMED") + lines.append( + f"{codepoint} {name} -> {substitutions[character]!r}" + ) + if not occurrences: lines.extend([ "", @@ -1344,6 +1419,12 @@ def convert_epub(epub_path, output_dir, overwrite=False): class_styles, ) + try: + substitutions = substitute_unsupported_characters(chapters) + except RuntimeError as exc: + print(f"Error: {exc}") + sys.exit(1) + for chapter in chapters: text, style_records = decode_styled_text( normalize_text(chapter["text"]) @@ -1439,6 +1520,7 @@ def convert_epub(epub_path, output_dir, overwrite=False): output_dir, toc_entries, written_sections, + substitutions, ) print() diff --git a/src/main.cpp b/src/main.cpp index edbde9d..42a5585 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -256,6 +256,7 @@ void togglePlayback() { } void navigatePreviousPaused() { + bool positionChanged = false; switch (readerView) { case ReaderView::kChapterTitle: { String previousChapterPath; @@ -264,12 +265,15 @@ void navigatePreviousPaused() { reader.lastParagraph(); readerView = ReaderView::kWord; showReaderScreen(); + positionChanged = true; } break; } case ReaderView::kWord: if (!reader.previousParagraph()) { readerView = ReaderView::kChapterTitle; + } else { + positionChanged = true; } showReaderScreen(); break; @@ -279,12 +283,15 @@ void navigatePreviousPaused() { if (reader.lastParagraph()) { readerView = ReaderView::kWord; showReaderScreen(); + positionChanged = true; } break; } + if (positionChanged) persistState(); } void navigateNextPaused() { + bool positionChanged = false; switch (readerView) { case ReaderView::kChapterTitle: reader.restoreAtParagraphStart(state.wordIndex); @@ -294,6 +301,7 @@ void navigateNextPaused() { case ReaderView::kWord: { if (reader.nextParagraph()) { showReaderScreen(); + positionChanged = true; break; } @@ -301,6 +309,7 @@ void navigateNextPaused() { if (library.nextChapter(state.bookPath, nextChapterPath) && loadChapter(nextChapterPath)) { showReaderScreen(); + positionChanged = true; } else { readerView = ReaderView::kTheEnd; showReaderScreen(); @@ -310,6 +319,7 @@ void navigateNextPaused() { case ReaderView::kTheEnd: break; } + if (positionChanged) persistState(); } void onTouchHome(void*) { diff --git a/tests/test_epub_to_chapters.py b/tests/test_epub_to_chapters.py index 4fa4a42..b8bb0f1 100644 --- a/tests/test_epub_to_chapters.py +++ b/tests/test_epub_to_chapters.py @@ -10,6 +10,7 @@ from epub_to_chapters import ( SIDECAR_VERSION, STYLE_BOLD, STYLE_ITALIC, + STYLE_MARKER, TIMING_DOUBLE, TIMING_TRIPLE, MAX_TOKEN_BYTES, @@ -24,8 +25,10 @@ from epub_to_chapters import ( normalize_text, output_section_name, parse_document, + prompt_for_substitutions, source_fingerprint, stylesheet_class_styles, + substitute_unsupported_characters, validate_token_lengths, write_style_sidecar, write_conversion_report, @@ -113,12 +116,68 @@ class ConversionReportTests(unittest.TestCase): Path(temporary_directory), ["Chapter 1"], [("Chapter 1.txt", "Plain ASCII text.")], + {"\u03a9": "Omega", "\u0416": ""}, ) report = report_file.read_text(encoding="utf-8") self.assertFalse(has_unsupported) self.assertIn("all generated TOC and chapter text is ASCII-safe", report) + self.assertIn("U+03A9 GREEK CAPITAL LETTER OMEGA -> 'Omega'", report) + self.assertIn("U+0416 CYRILLIC CAPITAL LETTER ZHE -> ''", report) + + +class UnsupportedCharacterSubstitutionTests(unittest.TestCase): + def test_replaces_each_character_everywhere_and_allows_blank_removal(self): + chapters = [{ + "title": "Chapter \u03a9", + "text": "\u03a9mega and \u0416", + }] + replacements = iter(["Omega", ""]) + + substitutions = substitute_unsupported_characters( + chapters, + input_fn=lambda prompt: next(replacements), + output_fn=lambda *args: None, + ) + + self.assertEqual(substitutions, {"\u03a9": "Omega", "\u0416": ""}) + self.assertEqual(chapters[0]["title"], "Chapter Omega") + self.assertEqual(chapters[0]["text"], "Omegamega and") + + def test_reprompts_until_replacement_is_printable_ascii(self): + replacements = iter(["\u00e9", "plain text"]) + messages = [] + + substitutions = prompt_for_substitutions( + {"\u03a9": 1}, + input_fn=lambda prompt: next(replacements), + output_fn=messages.append, + ) + + self.assertEqual(substitutions, {"\u03a9": "plain text"}) + self.assertEqual(messages, [ + "Replacement must contain printable ASCII characters only.", + ]) + + def test_replacement_keeps_style_markers_ready_for_retokensizing(self): + chapters = [{ + "title": "", + "text": f"{STYLE_MARKER}1alpha\u03a9beta{STYLE_MARKER}0", + }] + + substitute_unsupported_characters( + chapters, + input_fn=lambda prompt: " ", + output_fn=lambda *args: None, + ) + text, records = decode_styled_text(chapters[0]["text"]) + + self.assertEqual(text, "alpha beta") + self.assertEqual(records, [ + (0, STYLE_ITALIC, 0), + (1, STYLE_ITALIC, 0), + ]) class StyleSidecarTests(unittest.TestCase):