239 lines
7.6 KiB
Python
239 lines
7.6 KiB
Python
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from epub_to_chapters import (
|
|
SIDECAR_HEADER,
|
|
SIDECAR_MAGIC,
|
|
SIDECAR_VERSION,
|
|
STYLE_BOLD,
|
|
STYLE_ITALIC,
|
|
TIMING_DOUBLE,
|
|
TIMING_TRIPLE,
|
|
add_unfamiliar_word_timing,
|
|
decode_styled_text,
|
|
document_root,
|
|
html_to_text,
|
|
normalize_display_text,
|
|
normalize_text,
|
|
output_section_name,
|
|
parse_document,
|
|
source_fingerprint,
|
|
stylesheet_class_styles,
|
|
write_style_sidecar,
|
|
write_conversion_report,
|
|
)
|
|
|
|
|
|
class DisplayTextNormalizationTests(unittest.TestCase):
|
|
def test_converts_ligatures_accents_and_supported_symbols_to_ascii(self):
|
|
source = (
|
|
"\ufb01 \ufb02 \ufb03 caf\u00e9 \u00c5ngstr\u00f6m sm\u00f8rrebr\u00f8d "
|
|
"Stra\u00dfe \u0153uvre \u0141\u00f3d\u017a \u00bd \u2260 \u2022 \u2042"
|
|
)
|
|
|
|
self.assertEqual(
|
|
normalize_display_text(source),
|
|
"fi fl ffi cafe Angstrom smorrebrod Strasse oeuvre Lodz 1/2 != * ***",
|
|
)
|
|
|
|
def test_handles_unicode_whitespace_and_invisible_formatting(self):
|
|
source = "one\u00a0two\u200bthree\u200et\u200f four"
|
|
|
|
self.assertEqual(
|
|
normalize_display_text(source),
|
|
"one two threet four",
|
|
)
|
|
|
|
def test_em_dash_retains_an_rsvp_token_boundary(self):
|
|
self.assertEqual(normalize_text("left\u2014right"), "left- right")
|
|
|
|
def test_section_names_use_the_same_normalization(self):
|
|
used_names = set()
|
|
|
|
self.assertEqual(
|
|
output_section_name("Caf\u00e9 \u2014 Part \u00bd", 1, used_names),
|
|
"Cafe - Part 1-2",
|
|
)
|
|
|
|
|
|
class ConversionReportTests(unittest.TestCase):
|
|
def test_reports_remaining_non_ascii_characters_by_generated_file(self):
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
report_file, has_unsupported = write_conversion_report(
|
|
Path(temporary_directory),
|
|
["Chapter \u03a9"],
|
|
[("Chapter \u03a9.txt", "\u3053\u3093\u306b\u3061\u306f")],
|
|
)
|
|
|
|
report = report_file.read_text(encoding="utf-8")
|
|
|
|
self.assertTrue(has_unsupported)
|
|
self.assertIn("U+03A9 GREEK CAPITAL LETTER OMEGA", report)
|
|
self.assertIn("U+3053 HIRAGANA LETTER KO", report)
|
|
self.assertIn("TOC.txt", report)
|
|
|
|
def test_reports_an_ascii_safe_conversion(self):
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
report_file, has_unsupported = write_conversion_report(
|
|
Path(temporary_directory),
|
|
["Chapter 1"],
|
|
[("Chapter 1.txt", "Plain ASCII text.")],
|
|
)
|
|
|
|
report = report_file.read_text(encoding="utf-8")
|
|
|
|
self.assertFalse(has_unsupported)
|
|
self.assertIn("all generated TOC and chapter text is ASCII-safe", report)
|
|
|
|
|
|
class StyleSidecarTests(unittest.TestCase):
|
|
def test_extracts_italic_and_bold_word_styles(self):
|
|
soup = BeautifulSoup(
|
|
"<p>plain <i>italic</i> <strong>bold</strong> "
|
|
"<em><b>both</b></em></p>",
|
|
"html.parser",
|
|
)
|
|
|
|
text, records = decode_styled_text(html_to_text(document_root(soup)))
|
|
|
|
self.assertEqual(text, "plain italic bold both")
|
|
self.assertEqual(
|
|
records,
|
|
[
|
|
(1, STYLE_ITALIC, 0),
|
|
(2, STYLE_BOLD, 0),
|
|
(3, STYLE_ITALIC | STYLE_BOLD, 0),
|
|
],
|
|
)
|
|
|
|
def test_writes_a_versioned_sidecar_with_source_fingerprint(self):
|
|
text = "plain italic bold both"
|
|
records = [
|
|
(1, STYLE_ITALIC, 0),
|
|
(2, STYLE_BOLD, 0),
|
|
(3, STYLE_ITALIC | STYLE_BOLD, 0),
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
sidecar_file = Path(temporary_directory) / "Chapter 1.rsvp"
|
|
write_style_sidecar(sidecar_file, text, records)
|
|
data = sidecar_file.read_bytes()
|
|
|
|
magic, version, reserved, fingerprint, record_count = SIDECAR_HEADER.unpack(
|
|
data[:SIDECAR_HEADER.size]
|
|
)
|
|
self.assertEqual(magic, SIDECAR_MAGIC)
|
|
self.assertEqual(version, SIDECAR_VERSION)
|
|
self.assertEqual(reserved, 0)
|
|
self.assertEqual(fingerprint, source_fingerprint(text))
|
|
self.assertEqual(record_count, len(records))
|
|
self.assertEqual(
|
|
data[SIDECAR_HEADER.size:],
|
|
bytes([
|
|
1, STYLE_ITALIC, 0,
|
|
1, STYLE_BOLD, 0,
|
|
1, STYLE_ITALIC | STYLE_BOLD, 0,
|
|
]),
|
|
)
|
|
|
|
def test_writes_timing_only_records(self):
|
|
text = "Blarenptnip"
|
|
|
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
|
sidecar_file = Path(temporary_directory) / "Chapter 1.rsvp"
|
|
write_style_sidecar(sidecar_file, text, [(0, 0, TIMING_TRIPLE)])
|
|
data = sidecar_file.read_bytes()
|
|
|
|
self.assertEqual(
|
|
data[SIDECAR_HEADER.size:],
|
|
bytes([0, 0, TIMING_TRIPLE]),
|
|
)
|
|
|
|
def test_extracts_styles_from_css_classes_and_inline_css(self):
|
|
class FakeItem:
|
|
def __init__(self, name, content):
|
|
self.file_name = name
|
|
self._content = content.encode("utf-8")
|
|
|
|
def get_name(self):
|
|
return self.file_name
|
|
|
|
def get_content(self):
|
|
return self._content
|
|
|
|
class FakeBook:
|
|
def __init__(self, items):
|
|
self._items = items
|
|
|
|
def get_items(self):
|
|
return self._items
|
|
|
|
stylesheet = FakeItem(
|
|
"stylesheet.css",
|
|
".voice { font-style: italic; }\n"
|
|
".both { font: italic 700 1em serif; }",
|
|
)
|
|
document = FakeItem(
|
|
"chapter.xhtml",
|
|
"<p>Plain <span class='voice'>italic words</span> "
|
|
"<span style='font-weight: bold'>bold words</span> "
|
|
"<span class='both'>both</span></p>",
|
|
)
|
|
|
|
class_styles = stylesheet_class_styles(FakeBook([stylesheet, document]))
|
|
soup = parse_document(document, class_styles)
|
|
text, records = decode_styled_text(html_to_text(document_root(soup)))
|
|
|
|
self.assertEqual(text, "Plain italic words bold words both")
|
|
self.assertEqual(
|
|
records,
|
|
[
|
|
(1, STYLE_ITALIC, 0),
|
|
(2, STYLE_ITALIC, 0),
|
|
(3, STYLE_BOLD, 0),
|
|
(4, STYLE_BOLD, 0),
|
|
(5, STYLE_ITALIC | STYLE_BOLD, 0),
|
|
],
|
|
)
|
|
|
|
|
|
class UnfamiliarWordTimingTests(unittest.TestCase):
|
|
def test_marks_first_ten_globally_and_first_word_in_each_chapter(self):
|
|
chapters = [
|
|
{
|
|
"text": " ".join(["Blarenptnip"] * 11),
|
|
"style_records": [(0, STYLE_ITALIC, 0)],
|
|
},
|
|
{
|
|
"text": "Blarenptnip",
|
|
"style_records": [],
|
|
},
|
|
]
|
|
|
|
add_unfamiliar_word_timing(chapters, {"the", "reader"})
|
|
|
|
self.assertEqual(
|
|
chapters[0]["style_records"],
|
|
[(0, STYLE_ITALIC, TIMING_TRIPLE)] +
|
|
[(index, 0, TIMING_TRIPLE) for index in range(1, 10)] +
|
|
[(10, 0, TIMING_DOUBLE)],
|
|
)
|
|
self.assertEqual(chapters[1]["style_records"], [(0, 0, TIMING_TRIPLE)])
|
|
|
|
def test_english_words_and_non_words_are_not_marked(self):
|
|
chapters = [{
|
|
"text": "The reader has 123 pages.",
|
|
"style_records": [],
|
|
}]
|
|
|
|
add_unfamiliar_word_timing(chapters, {"the", "reader", "has", "pages"})
|
|
|
|
self.assertEqual(chapters[0]["style_records"], [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|