407 lines
13 KiB
Python
407 lines
13 KiB
Python
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from epub_to_chapters import (
|
|
SIDECAR_HEADER,
|
|
SIDECAR_MAGIC,
|
|
SIDECAR_VERSION,
|
|
STYLE_BOLD,
|
|
STYLE_ITALIC,
|
|
STYLE_MARKER,
|
|
TIMING_DOUBLE,
|
|
TIMING_TRIPLE,
|
|
MAX_TOKEN_BYTES,
|
|
MAX_SECTION_WORDS,
|
|
EnglishVocabulary,
|
|
add_unfamiliar_word_timing,
|
|
decode_styled_text,
|
|
document_root,
|
|
extract_chapter,
|
|
html_to_text,
|
|
item_index_map,
|
|
normalize_display_text,
|
|
normalize_text,
|
|
output_section_name,
|
|
parse_document,
|
|
prompt_for_substitutions,
|
|
source_fingerprint,
|
|
stylesheet_class_styles,
|
|
substitute_unsupported_characters,
|
|
validate_token_lengths,
|
|
write_style_sidecar,
|
|
write_conversion_report,
|
|
)
|
|
|
|
|
|
class DisplayTextNormalizationTests(unittest.TestCase):
|
|
def test_converts_ligatures_accents_and_supported_symbols_to_ascii(self):
|
|
source = (
|
|
"\ufb01 \ufb02 \ufb03 caf\u00e9 \u00c5ngstr\u00f6m sm\u00f8rrebr\u00f8d "
|
|
"Stra\u00dfe \u0153uvre \u0141\u00f3d\u017a \u00bd \u2260 \u2022 \u2042"
|
|
)
|
|
|
|
self.assertEqual(
|
|
normalize_display_text(source),
|
|
"fi fl ffi cafe Angstrom smorrebrod Strasse oeuvre Lodz 1/2 != * ***",
|
|
)
|
|
|
|
def test_handles_unicode_whitespace_and_invisible_formatting(self):
|
|
source = "one\u00a0two\u200bthree\u200et\u200f four"
|
|
|
|
self.assertEqual(
|
|
normalize_display_text(source),
|
|
"one two threet four",
|
|
)
|
|
|
|
def test_em_dash_retains_an_rsvp_token_boundary(self):
|
|
self.assertEqual(normalize_text("left\u2014right"), "left- right")
|
|
|
|
def test_section_names_use_the_same_normalization(self):
|
|
used_names = set()
|
|
|
|
self.assertEqual(
|
|
output_section_name("Caf\u00e9 \u2014 Part \u00bd", 1, used_names),
|
|
"Cafe - Part 1-2",
|
|
)
|
|
|
|
def test_section_names_fit_within_the_device_filename_budget(self):
|
|
name = output_section_name("A" * 300, 1, set())
|
|
|
|
self.assertLessEqual(len(name.encode("utf-8")), 200)
|
|
|
|
def test_section_names_do_not_collide_with_generated_files(self):
|
|
used_names = {"toc", "conversion_report"}
|
|
|
|
self.assertEqual(output_section_name("TOC", 1, used_names), "TOC (2)")
|
|
self.assertEqual(
|
|
output_section_name("CONVERSION_REPORT", 2, used_names),
|
|
"CONVERSION_REPORT (2)",
|
|
)
|
|
|
|
|
|
class TokenValidationTests(unittest.TestCase):
|
|
def test_allows_a_token_at_the_configured_byte_limit(self):
|
|
validate_token_lengths("a" * MAX_TOKEN_BYTES)
|
|
|
|
def test_rejects_a_token_larger_than_the_configured_byte_limit(self):
|
|
with self.assertRaises(ValueError):
|
|
validate_token_lengths("a" * (MAX_TOKEN_BYTES + 1))
|
|
|
|
def test_rejects_sections_with_too_many_words(self):
|
|
with self.assertRaises(ValueError):
|
|
validate_token_lengths("a " * (MAX_SECTION_WORDS + 1))
|
|
|
|
|
|
class ConversionReportTests(unittest.TestCase):
|
|
def test_reports_remaining_non_ascii_characters_by_generated_file(self):
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
report_file, has_unsupported = write_conversion_report(
|
|
Path(temporary_directory),
|
|
["Chapter \u03a9"],
|
|
[("Chapter \u03a9.txt", "\u3053\u3093\u306b\u3061\u306f")],
|
|
)
|
|
|
|
report = report_file.read_text(encoding="utf-8")
|
|
|
|
self.assertTrue(has_unsupported)
|
|
self.assertIn("U+03A9 GREEK CAPITAL LETTER OMEGA", report)
|
|
self.assertIn("U+3053 HIRAGANA LETTER KO", report)
|
|
self.assertIn("TOC.txt", report)
|
|
|
|
def test_reports_an_ascii_safe_conversion(self):
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
report_file, has_unsupported = write_conversion_report(
|
|
Path(temporary_directory),
|
|
["Chapter 1"],
|
|
[("Chapter 1.txt", "Plain ASCII text.")],
|
|
{"\u03a9": "Omega", "\u0416": ""},
|
|
)
|
|
|
|
report = report_file.read_text(encoding="utf-8")
|
|
|
|
self.assertFalse(has_unsupported)
|
|
self.assertIn("all generated TOC and chapter text is ASCII-safe", report)
|
|
self.assertIn("U+03A9 GREEK CAPITAL LETTER OMEGA -> 'Omega'", report)
|
|
self.assertIn("U+0416 CYRILLIC CAPITAL LETTER ZHE -> ''", report)
|
|
|
|
|
|
class UnsupportedCharacterSubstitutionTests(unittest.TestCase):
|
|
def test_replaces_each_character_everywhere_and_allows_blank_removal(self):
|
|
chapters = [{
|
|
"title": "Chapter \u03a9",
|
|
"text": "\u03a9mega and \u0416",
|
|
}]
|
|
replacements = iter(["Omega", ""])
|
|
|
|
substitutions = substitute_unsupported_characters(
|
|
chapters,
|
|
input_fn=lambda prompt: next(replacements),
|
|
output_fn=lambda *args: None,
|
|
)
|
|
|
|
self.assertEqual(substitutions, {"\u03a9": "Omega", "\u0416": ""})
|
|
self.assertEqual(chapters[0]["title"], "Chapter Omega")
|
|
self.assertEqual(chapters[0]["text"], "Omegamega and")
|
|
|
|
def test_reprompts_until_replacement_is_printable_ascii(self):
|
|
replacements = iter(["\u00e9", "plain text"])
|
|
messages = []
|
|
|
|
substitutions = prompt_for_substitutions(
|
|
{"\u03a9": 1},
|
|
input_fn=lambda prompt: next(replacements),
|
|
output_fn=messages.append,
|
|
)
|
|
|
|
self.assertEqual(substitutions, {"\u03a9": "plain text"})
|
|
self.assertEqual(messages, [
|
|
"Replacement must contain printable ASCII characters only.",
|
|
])
|
|
|
|
def test_replacement_keeps_style_markers_ready_for_retokensizing(self):
|
|
chapters = [{
|
|
"title": "",
|
|
"text": f"{STYLE_MARKER}1alpha\u03a9beta{STYLE_MARKER}0",
|
|
}]
|
|
|
|
substitute_unsupported_characters(
|
|
chapters,
|
|
input_fn=lambda prompt: " ",
|
|
output_fn=lambda *args: None,
|
|
)
|
|
text, records = decode_styled_text(chapters[0]["text"])
|
|
|
|
self.assertEqual(text, "alpha beta")
|
|
self.assertEqual(records, [
|
|
(0, STYLE_ITALIC, 0),
|
|
(1, STYLE_ITALIC, 0),
|
|
])
|
|
|
|
|
|
class StyleSidecarTests(unittest.TestCase):
|
|
def test_extracts_italic_and_bold_word_styles(self):
|
|
soup = BeautifulSoup(
|
|
"<p>plain <i>italic</i> <strong>bold</strong> "
|
|
"<em><b>both</b></em></p>",
|
|
"html.parser",
|
|
)
|
|
|
|
text, records = decode_styled_text(html_to_text(document_root(soup)))
|
|
|
|
self.assertEqual(text, "plain italic bold both")
|
|
self.assertEqual(
|
|
records,
|
|
[
|
|
(1, STYLE_ITALIC, 0),
|
|
(2, STYLE_BOLD, 0),
|
|
(3, STYLE_ITALIC | STYLE_BOLD, 0),
|
|
],
|
|
)
|
|
|
|
def test_writes_a_versioned_sidecar_with_source_fingerprint(self):
|
|
text = "plain italic bold both"
|
|
records = [
|
|
(1, STYLE_ITALIC, 0),
|
|
(2, STYLE_BOLD, 0),
|
|
(3, STYLE_ITALIC | STYLE_BOLD, 0),
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
sidecar_file = Path(temporary_directory) / "Chapter 1.rsvp"
|
|
write_style_sidecar(sidecar_file, text, records)
|
|
data = sidecar_file.read_bytes()
|
|
|
|
magic, version, reserved, fingerprint, record_count = SIDECAR_HEADER.unpack(
|
|
data[:SIDECAR_HEADER.size]
|
|
)
|
|
self.assertEqual(magic, SIDECAR_MAGIC)
|
|
self.assertEqual(version, SIDECAR_VERSION)
|
|
self.assertEqual(reserved, 0)
|
|
self.assertEqual(fingerprint, source_fingerprint(text))
|
|
self.assertEqual(record_count, len(records))
|
|
self.assertEqual(
|
|
data[SIDECAR_HEADER.size:],
|
|
bytes([
|
|
1, STYLE_ITALIC, 0,
|
|
1, STYLE_BOLD, 0,
|
|
1, STYLE_ITALIC | STYLE_BOLD, 0,
|
|
]),
|
|
)
|
|
|
|
def test_writes_timing_only_records(self):
|
|
text = "Blarenptnip"
|
|
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
sidecar_file = Path(temporary_directory) / "Chapter 1.rsvp"
|
|
write_style_sidecar(sidecar_file, text, [(0, 0, TIMING_TRIPLE)])
|
|
data = sidecar_file.read_bytes()
|
|
|
|
self.assertEqual(
|
|
data[SIDECAR_HEADER.size:],
|
|
bytes([0, 0, TIMING_TRIPLE]),
|
|
)
|
|
|
|
def test_extracts_styles_from_css_classes_and_inline_css(self):
|
|
class FakeItem:
|
|
def __init__(self, name, content):
|
|
self.file_name = name
|
|
self._content = content.encode("utf-8")
|
|
|
|
def get_name(self):
|
|
return self.file_name
|
|
|
|
def get_content(self):
|
|
return self._content
|
|
|
|
class FakeBook:
|
|
def __init__(self, items):
|
|
self._items = items
|
|
|
|
def get_items(self):
|
|
return self._items
|
|
|
|
stylesheet = FakeItem(
|
|
"stylesheet.css",
|
|
".voice { font-style: italic; }\n"
|
|
".both { font: italic 700 1em serif; }",
|
|
)
|
|
document = FakeItem(
|
|
"chapter.xhtml",
|
|
"<p>Plain <span class='voice'>italic words</span> "
|
|
"<span style='font-weight: bold'>bold words</span> "
|
|
"<span class='both'>both</span></p>",
|
|
)
|
|
|
|
class_styles = stylesheet_class_styles(FakeBook([stylesheet, document]))
|
|
soup = parse_document(document, class_styles)
|
|
text, records = decode_styled_text(html_to_text(document_root(soup)))
|
|
|
|
self.assertEqual(text, "Plain italic words bold words both")
|
|
self.assertEqual(
|
|
records,
|
|
[
|
|
(1, STYLE_ITALIC, 0),
|
|
(2, STYLE_ITALIC, 0),
|
|
(3, STYLE_BOLD, 0),
|
|
(4, STYLE_BOLD, 0),
|
|
(5, STYLE_ITALIC | STYLE_BOLD, 0),
|
|
],
|
|
)
|
|
|
|
|
|
class UnfamiliarWordTimingTests(unittest.TestCase):
|
|
def test_marks_first_ten_globally_and_first_word_in_each_chapter(self):
|
|
chapters = [
|
|
{
|
|
"text": " ".join(["Blarenptnip"] * 11),
|
|
"style_records": [(0, STYLE_ITALIC, 0)],
|
|
},
|
|
{
|
|
"text": "Blarenptnip",
|
|
"style_records": [],
|
|
},
|
|
]
|
|
|
|
add_unfamiliar_word_timing(chapters, {"the", "reader"})
|
|
|
|
self.assertEqual(
|
|
chapters[0]["style_records"],
|
|
[(0, STYLE_ITALIC, TIMING_TRIPLE)] +
|
|
[(index, 0, TIMING_TRIPLE) for index in range(1, 10)] +
|
|
[(10, 0, TIMING_DOUBLE)],
|
|
)
|
|
self.assertEqual(chapters[1]["style_records"], [(0, 0, TIMING_TRIPLE)])
|
|
|
|
def test_english_words_and_non_words_are_not_marked(self):
|
|
chapters = [{
|
|
"text": "The reader has 123 pages.",
|
|
"style_records": [],
|
|
}]
|
|
|
|
add_unfamiliar_word_timing(chapters, {"the", "reader", "has", "pages"})
|
|
|
|
self.assertEqual(chapters[0]["style_records"], [])
|
|
|
|
def test_common_variants_of_known_words_are_not_marked(self):
|
|
chapters = [{
|
|
"text": (
|
|
"asked walked eyes using he'd couldn't didn't hasn't that's "
|
|
"father's stories thicker life-threatening well-dressed "
|
|
"Blarenptnip"
|
|
),
|
|
"style_records": [],
|
|
}]
|
|
english_words = {
|
|
"ask", "walk", "eye", "use", "he", "could", "do", "have",
|
|
"that", "father", "story", "thick", "life", "threaten", "well",
|
|
"dress",
|
|
}
|
|
|
|
add_unfamiliar_word_timing(chapters, english_words)
|
|
|
|
self.assertEqual(chapters[0]["style_records"], [(14, 0, TIMING_TRIPLE)])
|
|
|
|
def test_frequency_fallback_accepts_common_word_missing_from_web2(self):
|
|
vocabulary = EnglishVocabulary(
|
|
{"reader"},
|
|
lambda word, language: 2.1 if word == "mudflat" else 0,
|
|
)
|
|
|
|
self.assertIn("reader", vocabulary)
|
|
self.assertIn("mudflat", vocabulary)
|
|
self.assertNotIn("blarenptnip", vocabulary)
|
|
|
|
|
|
class SameDocumentTocTests(unittest.TestCase):
|
|
def test_extracts_each_fragment_when_two_toc_entries_share_a_document(self):
|
|
class FakeItem:
|
|
def __init__(self, name, content):
|
|
self.name = name
|
|
self.content = content
|
|
|
|
def get_content(self):
|
|
return self.content.encode("utf-8")
|
|
|
|
def get_name(self):
|
|
return self.name
|
|
|
|
def get_id(self):
|
|
return self.name
|
|
|
|
def get_type(self):
|
|
from epub_to_chapters import ITEM_DOCUMENT
|
|
return ITEM_DOCUMENT
|
|
|
|
class FakeBook:
|
|
def __init__(self, item):
|
|
self.item = item
|
|
self.spine = [(item.get_id(),)]
|
|
|
|
def get_items(self):
|
|
return [self.item]
|
|
|
|
item = FakeItem(
|
|
"chapter.xhtml",
|
|
"<body><h1 id='one'>One</h1><p>First text.</p>"
|
|
"<h1 id='two'>Two</h1><p>Second text.</p></body>",
|
|
)
|
|
book = FakeBook(item)
|
|
entries = [
|
|
{"title": "One", "item": item, "fragment": "one"},
|
|
{"title": "Two", "item": item, "fragment": "two"},
|
|
]
|
|
positions = item_index_map(book)
|
|
|
|
first, _ = decode_styled_text(extract_chapter(book, entries, 0, positions))
|
|
second, _ = decode_styled_text(extract_chapter(book, entries, 1, positions))
|
|
|
|
self.assertEqual(first, "One\nFirst text.")
|
|
self.assertEqual(second, "Two\nSecond text.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|