1508 lines
36 KiB
Python
1508 lines
36 KiB
Python
#!/usr/bin/env python3
|
|
|
|
"""
|
|
EPUB -> a TOC-ordered folder of chapter TXT files.
|
|
|
|
Usage:
|
|
python epub_to_chapters.py book.epub
|
|
python epub_to_chapters.py book.epub output_directory
|
|
|
|
Output:
|
|
output_directory/
|
|
TOC.txt
|
|
Dedication.txt
|
|
Chapter 1.txt
|
|
Chapter 2.txt
|
|
...
|
|
|
|
Each line of TOC.txt is both a section title and the basename of its text file.
|
|
The output files contain only section text. No chapter heading is added.
|
|
|
|
Strategy:
|
|
1. Use the EPUB table of contents when available.
|
|
2. Correctly handle #fragment links inside XHTML files.
|
|
3. Handle multiple chapters inside one XHTML file.
|
|
4. Handle chapters spanning multiple XHTML files.
|
|
5. Fall back to the EPUB spine when the TOC is unavailable.
|
|
6. Clean HTML into readable plain text.
|
|
"""
|
|
|
|
import re
|
|
import struct
|
|
import sys
|
|
import unicodedata
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
from urllib.parse import unquote, urlsplit
|
|
|
|
from bs4 import BeautifulSoup, NavigableString
|
|
|
|
try:
|
|
from ebooklib import epub, ITEM_DOCUMENT
|
|
except ModuleNotFoundError:
|
|
epub = None
|
|
ITEM_DOCUMENT = None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SKIP_TAGS = {
|
|
"script",
|
|
"style",
|
|
"noscript",
|
|
"svg",
|
|
"canvas",
|
|
"nav",
|
|
"form",
|
|
"iframe",
|
|
"object",
|
|
"embed",
|
|
}
|
|
|
|
BLOCK_TAGS = {
|
|
"p",
|
|
"div",
|
|
"section",
|
|
"article",
|
|
"header",
|
|
"footer",
|
|
"aside",
|
|
"blockquote",
|
|
"pre",
|
|
"figure",
|
|
"figcaption",
|
|
"table",
|
|
"thead",
|
|
"tbody",
|
|
"tfoot",
|
|
"tr",
|
|
"td",
|
|
"th",
|
|
"li",
|
|
"dt",
|
|
"dd",
|
|
"h1",
|
|
"h2",
|
|
"h3",
|
|
"h4",
|
|
"h5",
|
|
"h6",
|
|
}
|
|
|
|
# These are useful TOC entries but generally aren't chapters.
|
|
FRONT_MATTER = {
|
|
"contents",
|
|
"table of contents",
|
|
"toc",
|
|
"cover",
|
|
"title page",
|
|
"copyright",
|
|
"copyright page",
|
|
"dedication",
|
|
"epigraph",
|
|
"acknowledgments",
|
|
"acknowledgements",
|
|
"about the author",
|
|
"also by",
|
|
"colophon",
|
|
}
|
|
|
|
FAT_FORBIDDEN_FILENAME_CHARACTERS = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
|
|
|
|
STYLE_ITALIC = 0x01
|
|
STYLE_BOLD = 0x02
|
|
STYLE_MARKER = "\x01"
|
|
SIDECAR_MAGIC = b"RSVP"
|
|
SIDECAR_VERSION = 1
|
|
SIDECAR_HEADER = struct.Struct("<4sBBQI")
|
|
CSS_RULE = re.compile(r"([^{}]+)\{([^{}]*)\}")
|
|
CSS_CLASS = re.compile(r"\.([A-Za-z_-][A-Za-z0-9_-]*)")
|
|
|
|
DISPLAY_PUNCTUATION = str.maketrans({
|
|
"\ufeff": "", # UTF-8 byte-order mark.
|
|
"\u00a0": " ", # Non-breaking space.
|
|
"\u00ad": "-", # Soft hyphen.
|
|
"\u2010": "-", # Hyphen.
|
|
"\u2011": "-", # Non-breaking hyphen.
|
|
"\u2012": "-", # Figure dash.
|
|
"\u2013": "-", # En dash.
|
|
"\u2014": "- ", # Em dash; retain an RSVP token boundary.
|
|
"\u2015": "- ", # Horizontal bar; retain an RSVP token boundary.
|
|
"\u2018": "'", # Left single quotation mark.
|
|
"\u2019": "'", # Right single quotation mark.
|
|
"\u201a": "'", # Single low-9 quotation mark.
|
|
"\u201b": "'", # Single high-reversed-9 quotation mark.
|
|
"\u201c": '"', # Left double quotation mark.
|
|
"\u201d": '"', # Right double quotation mark.
|
|
"\u201e": '"', # Double low-9 quotation mark.
|
|
"\u2026": "...", # Ellipsis.
|
|
"\u2022": "*", # Bullet.
|
|
"\u2023": "*", # Triangular bullet.
|
|
"\u2043": "-", # Hyphen bullet.
|
|
"\u2219": "*", # Bullet operator.
|
|
"\u25e6": "*", # White bullet.
|
|
"\u25aa": "*", # Small square.
|
|
"\u25cf": "*", # Black circle.
|
|
"\u2212": "-", # Minus sign.
|
|
"\u00d7": "x", # Multiplication sign.
|
|
"\u00f7": "/", # Division sign.
|
|
"\u2044": "/", # Fraction slash.
|
|
"\u2260": "!=", # Not equal.
|
|
"\u2264": "<=", # Less than or equal.
|
|
"\u2265": ">=", # Greater than or equal.
|
|
"\u2248": "~=", # Almost equal.
|
|
"\u2020": "+", # Dagger.
|
|
"\u2021": "++", # Double dagger.
|
|
"\u2032": "'", # Prime.
|
|
"\u2033": "\"", # Double prime.
|
|
"\u215b": "1/8", # Vulgar fractions.
|
|
"\u215c": "3/8",
|
|
"\u215d": "5/8",
|
|
"\u215e": "7/8",
|
|
"\u00bc": "1/4",
|
|
"\u00bd": "1/2",
|
|
"\u00be": "3/4",
|
|
"\u2153": "1/3",
|
|
"\u2154": "2/3",
|
|
"\u2155": "1/5",
|
|
"\u2156": "2/5",
|
|
"\u2157": "3/5",
|
|
"\u2158": "4/5",
|
|
"\u2159": "1/6",
|
|
"\u215a": "5/6",
|
|
"\u2042": "***", # Asterism.
|
|
"\u2055": "***", # Flower punctuation mark.
|
|
"\u2766": "***", # Floral heart.
|
|
"\u2767": "***", # Rotated floral heart.
|
|
"\u200b": " ", # Zero-width space; retain a word boundary.
|
|
"\u00c6": "AE", # Latin letters not expanded by Unicode NFKD.
|
|
"\u00e6": "ae",
|
|
"\u00d0": "D",
|
|
"\u00f0": "d",
|
|
"\u00d8": "O",
|
|
"\u00f8": "o",
|
|
"\u00de": "TH",
|
|
"\u00fe": "th",
|
|
"\u00df": "ss",
|
|
"\u0131": "i",
|
|
"\u0141": "L",
|
|
"\u0142": "l",
|
|
"\u0152": "OE",
|
|
"\u0153": "oe",
|
|
"\u0110": "D",
|
|
"\u0111": "d",
|
|
"\u0126": "H",
|
|
"\u0127": "h",
|
|
"\u014a": "N",
|
|
"\u014b": "n",
|
|
})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# General helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def normalize_display_text(text):
|
|
"""Convert common Unicode typography to device-safe ASCII where possible.
|
|
|
|
Unsupported scripts are deliberately retained so ``write_conversion_report``
|
|
can identify them for a book-specific transliteration or replacement choice.
|
|
"""
|
|
|
|
text = text.translate(DISPLAY_PUNCTUATION)
|
|
|
|
# NFKD expands presentation ligatures, Latin letters with diacritics, and
|
|
# compatible number forms (for example, superscript digits). Removing the
|
|
# resulting combining marks yields readable ASCII for Latin text without
|
|
# discarding a non-Latin base character.
|
|
text = unicodedata.normalize("NFKD", text)
|
|
|
|
normalized = []
|
|
for character in text:
|
|
if character in "\r\n":
|
|
normalized.append(character)
|
|
elif character.isspace():
|
|
normalized.append(" ")
|
|
elif unicodedata.combining(character):
|
|
continue
|
|
elif unicodedata.category(character) == "Cf":
|
|
# Formatting controls such as direction marks and joiners are not
|
|
# printable on the device. U+200B was handled above as a space.
|
|
continue
|
|
else:
|
|
normalized.append(character)
|
|
|
|
return "".join(normalized)
|
|
|
|
def normalize_text(text):
|
|
"""
|
|
Normalize extracted text while preserving paragraph breaks.
|
|
"""
|
|
|
|
text = normalize_display_text(text)
|
|
text = text.replace("\r\n", "\n")
|
|
text = text.replace("\r", "\n")
|
|
|
|
# Remove spaces around line breaks.
|
|
text = re.sub(r"[ \t]*\n[ \t]*", "\n", text)
|
|
|
|
# Collapse horizontal whitespace.
|
|
text = re.sub(r"[ \t]+", " ", text)
|
|
|
|
# Remove excessive blank lines.
|
|
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
|
|
return text.strip()
|
|
|
|
|
|
def style_flags_for_text_node(node):
|
|
"""Return the inline EPUB emphasis flags applying to a text node."""
|
|
|
|
flags = 0
|
|
parent = node.parent
|
|
while parent is not None:
|
|
if parent.name in {"i", "em"}:
|
|
flags |= STYLE_ITALIC
|
|
elif parent.name in {"b", "strong"}:
|
|
flags |= STYLE_BOLD
|
|
flags |= int(parent.attrs.get("data-rsvp-style-flags", 0))
|
|
parent = parent.parent
|
|
return flags
|
|
|
|
|
|
def append_styled_text(pieces, node):
|
|
"""Append text bracketed by private markers that survive normalization."""
|
|
|
|
text = str(node)
|
|
if not text:
|
|
return
|
|
flags = style_flags_for_text_node(node)
|
|
pieces.extend((STYLE_MARKER, chr(ord("0") + flags), text,
|
|
STYLE_MARKER, "0"))
|
|
|
|
|
|
def decode_styled_text(text):
|
|
"""Remove style markers and return sparse per-word style records."""
|
|
|
|
characters = []
|
|
character_styles = []
|
|
current_style = 0
|
|
position = 0
|
|
|
|
while position < len(text):
|
|
character = text[position]
|
|
if character == STYLE_MARKER:
|
|
if position + 1 >= len(text):
|
|
raise ValueError("unterminated style marker")
|
|
current_style = ord(text[position + 1]) - ord("0")
|
|
if current_style & ~(STYLE_ITALIC | STYLE_BOLD):
|
|
raise ValueError("invalid style marker")
|
|
position += 2
|
|
continue
|
|
characters.append(character)
|
|
character_styles.append(current_style)
|
|
position += 1
|
|
|
|
plain_text = "".join(characters)
|
|
style_records = []
|
|
for word_index, match in enumerate(re.finditer(r"\S+", plain_text)):
|
|
flags = 0
|
|
for style in character_styles[match.start():match.end()]:
|
|
flags |= style
|
|
if flags:
|
|
# The timing-modifier byte is reserved for a future sidecar version.
|
|
style_records.append((word_index, flags, 0))
|
|
|
|
return plain_text, style_records
|
|
|
|
|
|
def source_fingerprint(text):
|
|
"""Return a stable 64-bit FNV-1a fingerprint of UTF-8 chapter text."""
|
|
|
|
value = 0xCBF29CE484222325
|
|
for byte in text.encode("utf-8"):
|
|
value ^= byte
|
|
value = (value * 0x100000001B3) & 0xFFFFFFFFFFFFFFFF
|
|
return value
|
|
|
|
|
|
def encode_varint(value):
|
|
"""Encode a non-negative integer as unsigned LEB128."""
|
|
|
|
if value < 0:
|
|
raise ValueError("varint value must be non-negative")
|
|
encoded = bytearray()
|
|
while True:
|
|
byte = value & 0x7F
|
|
value >>= 7
|
|
encoded.append(byte | (0x80 if value else 0))
|
|
if not value:
|
|
return bytes(encoded)
|
|
|
|
|
|
def write_style_sidecar(output_file, text, style_records):
|
|
"""Write a validated, extensible style-only sidecar beside a chapter."""
|
|
|
|
payload = bytearray(SIDECAR_HEADER.pack(
|
|
SIDECAR_MAGIC,
|
|
SIDECAR_VERSION,
|
|
0,
|
|
source_fingerprint(text),
|
|
len(style_records),
|
|
))
|
|
previous_index = 0
|
|
for record_number, (word_index, style_flags, timing_modifier) in enumerate(style_records):
|
|
if word_index < 0 or style_flags & ~(STYLE_ITALIC | STYLE_BOLD):
|
|
raise ValueError("invalid style sidecar record")
|
|
if style_flags == 0 or not 0 <= timing_modifier <= 0xFF:
|
|
raise ValueError("invalid style sidecar record")
|
|
if record_number and word_index <= previous_index:
|
|
raise ValueError("style records must have ascending word indexes")
|
|
delta = word_index if record_number == 0 else word_index - previous_index
|
|
payload.extend(encode_varint(delta))
|
|
payload.extend((style_flags, timing_modifier))
|
|
previous_index = word_index
|
|
|
|
output_file.write_bytes(payload)
|
|
|
|
|
|
def output_section_name(title, number, used_names):
|
|
"""
|
|
Return a readable, FAT-safe, unique TOC entry and file basename.
|
|
"""
|
|
|
|
title = normalize_display_text(title or "")
|
|
title = re.sub(r"\s+", " ", title).strip()
|
|
title = FAT_FORBIDDEN_FILENAME_CHARACTERS.sub("-", title)
|
|
title = title.rstrip(". ")
|
|
|
|
if not title:
|
|
title = f"Section {number}"
|
|
|
|
candidate = title
|
|
duplicate = 2
|
|
while candidate.casefold() in used_names:
|
|
candidate = f"{title} ({duplicate})"
|
|
duplicate += 1
|
|
|
|
used_names.add(candidate.casefold())
|
|
return candidate
|
|
|
|
|
|
def remaining_non_ascii(text):
|
|
"""Return counts of characters the device's printable ASCII font lacks."""
|
|
|
|
return Counter(character for character in text if ord(character) > 127)
|
|
|
|
|
|
def write_conversion_report(output_dir, toc_entries, written_sections):
|
|
"""Write a report for unsupported characters left in generated text."""
|
|
|
|
occurrences = defaultdict(Counter)
|
|
source_files = defaultdict(set)
|
|
|
|
generated_files = [("TOC.txt", "\n".join(toc_entries))]
|
|
generated_files.extend(written_sections)
|
|
|
|
for filename, text in generated_files:
|
|
for character, count in remaining_non_ascii(text).items():
|
|
occurrences[character][filename] += count
|
|
source_files[character].add(filename)
|
|
|
|
report_file = output_dir / "CONVERSION_REPORT.txt"
|
|
lines = [
|
|
"EPUB conversion report",
|
|
"",
|
|
"The reader's current font supports printable ASCII only.",
|
|
]
|
|
|
|
if not occurrences:
|
|
lines.extend([
|
|
"",
|
|
"Result: all generated TOC and chapter text is ASCII-safe.",
|
|
])
|
|
else:
|
|
lines.extend([
|
|
"",
|
|
"WARNING: unsupported non-ASCII characters remain.",
|
|
"Choose a per-book treatment: transliterate, replace deliberately,",
|
|
"or defer the book until a Unicode-capable font is available.",
|
|
"",
|
|
"Character Count Generated files",
|
|
"-------------- ----- ---------------",
|
|
])
|
|
|
|
for character in sorted(occurrences, key=ord):
|
|
codepoint = f"U+{ord(character):04X}"
|
|
name = unicodedata.name(character, "UNNAMED")
|
|
total = sum(occurrences[character].values())
|
|
filenames = ", ".join(sorted(source_files[character]))
|
|
lines.append(
|
|
f"{codepoint} {name:<30} {total:5} {filenames}"
|
|
)
|
|
|
|
report_file.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
return report_file, bool(occurrences)
|
|
|
|
|
|
def normalize_href(href):
|
|
"""
|
|
Return:
|
|
|
|
(normalized_path, fragment)
|
|
|
|
Example:
|
|
|
|
Text/Chapter01.xhtml#part2
|
|
|
|
becomes:
|
|
|
|
("text/chapter01.xhtml", "part2")
|
|
"""
|
|
|
|
if not href:
|
|
return "", ""
|
|
|
|
href = unquote(href)
|
|
href = href.replace("\\", "/")
|
|
|
|
parsed = urlsplit(href)
|
|
|
|
path = parsed.path.lstrip("/")
|
|
path = re.sub(r"/+", "/", path)
|
|
|
|
return path, parsed.fragment
|
|
|
|
|
|
def same_document(item_a, item_b):
|
|
"""
|
|
Compare two ebooklib document items.
|
|
"""
|
|
|
|
if item_a is item_b:
|
|
return True
|
|
|
|
try:
|
|
return item_a.get_name() == item_b.get_name()
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# EPUB document indexing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def build_document_index(book):
|
|
"""
|
|
Build indexes allowing TOC hrefs to be resolved to ebooklib items.
|
|
"""
|
|
|
|
by_path = {}
|
|
by_filename = {}
|
|
|
|
for item in book.get_items():
|
|
|
|
if item.get_type() != ITEM_DOCUMENT:
|
|
continue
|
|
|
|
names = set()
|
|
|
|
try:
|
|
name = item.get_name()
|
|
if name:
|
|
names.add(name)
|
|
except Exception:
|
|
pass
|
|
|
|
file_name = getattr(item, "file_name", None)
|
|
|
|
if file_name:
|
|
names.add(file_name)
|
|
|
|
for name in names:
|
|
path, _ = normalize_href(name)
|
|
|
|
if not path:
|
|
continue
|
|
|
|
by_path[path] = item
|
|
by_filename[Path(path).name] = item
|
|
|
|
return by_path, by_filename
|
|
|
|
|
|
def find_document(href, by_path, by_filename):
|
|
"""
|
|
Resolve a TOC href to an EPUB document.
|
|
"""
|
|
|
|
path, fragment = normalize_href(href)
|
|
|
|
if not path:
|
|
return None, fragment
|
|
|
|
# Exact path.
|
|
item = by_path.get(path)
|
|
|
|
if item:
|
|
return item, fragment
|
|
|
|
# Try without leading ../ segments.
|
|
simplified = path
|
|
|
|
while simplified.startswith("../"):
|
|
simplified = simplified[3:]
|
|
|
|
item = by_path.get(simplified)
|
|
|
|
if item:
|
|
return item, fragment
|
|
|
|
# Last resort: filename only.
|
|
filename = Path(path).name
|
|
|
|
item = by_filename.get(filename)
|
|
|
|
if item:
|
|
return item, fragment
|
|
|
|
return None, fragment
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HTML parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def style_flags_from_css(declarations):
|
|
"""Return supported italic/bold flags from a CSS declaration block."""
|
|
|
|
flags = 0
|
|
for property_name, value in re.findall(
|
|
r"([A-Za-z-]+)\s*:\s*([^;{}]+)",
|
|
declarations.lower(),
|
|
):
|
|
if property_name == "font-style" and (
|
|
"italic" in value or "oblique" in value
|
|
):
|
|
flags |= STYLE_ITALIC
|
|
elif property_name == "font-weight" and (
|
|
"bold" in value or re.search(r"\b[6-9]00\b", value)
|
|
):
|
|
flags |= STYLE_BOLD
|
|
elif property_name == "font":
|
|
if "italic" in value or "oblique" in value:
|
|
flags |= STYLE_ITALIC
|
|
if "bold" in value or re.search(r"\b[6-9]00\b", value):
|
|
flags |= STYLE_BOLD
|
|
return flags
|
|
|
|
|
|
def stylesheet_class_styles(book):
|
|
"""Collect supported text styles declared by CSS classes in an EPUB."""
|
|
|
|
class_styles = {}
|
|
for item in book.get_items():
|
|
name = getattr(item, "file_name", "") or ""
|
|
try:
|
|
name = item.get_name() or name
|
|
except Exception:
|
|
pass
|
|
if not name.lower().endswith(".css"):
|
|
continue
|
|
|
|
content = item.get_content().decode("utf-8", errors="replace")
|
|
content = re.sub(r"/\*.*?\*/", "", content, flags=re.DOTALL)
|
|
for selectors, declarations in CSS_RULE.findall(content):
|
|
flags = style_flags_from_css(declarations)
|
|
if not flags:
|
|
continue
|
|
for selector in selectors.split(","):
|
|
for class_name in CSS_CLASS.findall(selector):
|
|
class_styles[class_name] = (
|
|
class_styles.get(class_name, 0) | flags
|
|
)
|
|
return class_styles
|
|
|
|
|
|
def apply_css_styles(soup, class_styles):
|
|
"""Mark tags whose class or inline CSS supplies a supported text style."""
|
|
|
|
for tag in soup.find_all(True):
|
|
flags = style_flags_from_css(tag.get("style", ""))
|
|
for class_name in tag.get("class", []):
|
|
flags |= class_styles.get(class_name, 0)
|
|
if flags:
|
|
tag["data-rsvp-style-flags"] = str(flags)
|
|
|
|
|
|
def parse_document(item, class_styles=None):
|
|
"""
|
|
Parse an EPUB XHTML document.
|
|
"""
|
|
|
|
soup = BeautifulSoup(
|
|
item.get_content(),
|
|
"html.parser",
|
|
)
|
|
|
|
for tag in soup.find_all(SKIP_TAGS):
|
|
tag.decompose()
|
|
|
|
if class_styles is not None:
|
|
apply_css_styles(soup, class_styles)
|
|
|
|
return soup
|
|
|
|
|
|
def document_root(soup):
|
|
"""
|
|
Return the best content root.
|
|
"""
|
|
|
|
if soup.body:
|
|
return soup.body
|
|
|
|
if soup.html:
|
|
return soup.html
|
|
|
|
return soup
|
|
|
|
|
|
def find_fragment(soup, fragment):
|
|
"""
|
|
Locate an EPUB #fragment anchor.
|
|
|
|
Supports both:
|
|
|
|
id="foo"
|
|
|
|
and older:
|
|
|
|
name="foo"
|
|
"""
|
|
|
|
if not fragment:
|
|
return None
|
|
|
|
element = soup.find(id=fragment)
|
|
|
|
if element:
|
|
return element
|
|
|
|
return soup.find(attrs={"name": fragment})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HTML -> text
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def html_to_text(root):
|
|
"""
|
|
Convert an HTML element into readable text.
|
|
|
|
Paragraphs and other block-level elements become separated by
|
|
blank lines.
|
|
"""
|
|
|
|
pieces = []
|
|
|
|
def walk(node):
|
|
|
|
if isinstance(node, NavigableString):
|
|
append_styled_text(pieces, node)
|
|
|
|
return
|
|
|
|
if not getattr(node, "name", None):
|
|
return
|
|
|
|
if node.name in SKIP_TAGS:
|
|
return
|
|
|
|
if node.name == "br":
|
|
pieces.append("\n")
|
|
return
|
|
|
|
is_block = node.name in BLOCK_TAGS
|
|
|
|
if is_block:
|
|
pieces.append("\n")
|
|
|
|
for child in node.children:
|
|
walk(child)
|
|
|
|
if is_block:
|
|
pieces.append("\n")
|
|
|
|
walk(root)
|
|
|
|
return normalize_text("".join(pieces))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Precise range extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def extract_range(root, start_element=None, end_element=None):
|
|
"""
|
|
Extract text from start_element up to, but not including,
|
|
end_element.
|
|
|
|
If start_element is None, extraction starts at the beginning.
|
|
"""
|
|
|
|
pieces = []
|
|
started = start_element is None
|
|
|
|
for node in root.descendants:
|
|
|
|
if node is start_element:
|
|
started = True
|
|
|
|
if node is end_element:
|
|
break
|
|
|
|
if not started:
|
|
continue
|
|
|
|
if isinstance(node, NavigableString):
|
|
append_styled_text(pieces, node)
|
|
|
|
elif getattr(node, "name", None) == "br":
|
|
pieces.append("\n")
|
|
|
|
elif getattr(node, "name", None) in BLOCK_TAGS:
|
|
pieces.append("\n")
|
|
|
|
return normalize_text("".join(pieces))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# TOC handling
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def flatten_toc(toc):
|
|
"""
|
|
Flatten ebooklib's nested TOC structure.
|
|
|
|
Returns a list of:
|
|
|
|
{
|
|
"title": "...",
|
|
"href": "..."
|
|
}
|
|
"""
|
|
|
|
result = []
|
|
|
|
def walk(items):
|
|
|
|
for item in items:
|
|
|
|
# Nested TOC section.
|
|
if isinstance(item, tuple):
|
|
|
|
section, children = item
|
|
|
|
if section is not None:
|
|
|
|
href = getattr(section, "href", None)
|
|
title = getattr(section, "title", "") or ""
|
|
|
|
if href:
|
|
result.append({
|
|
"title": title.strip(),
|
|
"href": href,
|
|
})
|
|
|
|
walk(children)
|
|
|
|
else:
|
|
|
|
href = getattr(item, "href", None)
|
|
title = getattr(item, "title", "") or ""
|
|
|
|
if href:
|
|
result.append({
|
|
"title": title.strip(),
|
|
"href": href,
|
|
})
|
|
|
|
walk(toc)
|
|
|
|
return result
|
|
|
|
|
|
def get_toc(book):
|
|
"""
|
|
Return the EPUB TOC.
|
|
"""
|
|
|
|
toc = getattr(book, "toc", None)
|
|
|
|
if not toc:
|
|
return []
|
|
|
|
return flatten_toc(toc)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Resolve TOC entries
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def resolve_toc(book):
|
|
"""
|
|
Resolve every usable TOC entry to an EPUB document.
|
|
"""
|
|
|
|
by_path, by_filename = build_document_index(book)
|
|
|
|
entries = []
|
|
|
|
for entry in get_toc(book):
|
|
|
|
item, fragment = find_document(
|
|
entry["href"],
|
|
by_path,
|
|
by_filename,
|
|
)
|
|
|
|
if item is None:
|
|
|
|
print(
|
|
f"Warning: cannot resolve TOC entry: "
|
|
f"{entry['title']!r} -> {entry['href']}"
|
|
)
|
|
|
|
continue
|
|
|
|
entries.append({
|
|
"title": entry["title"],
|
|
"href": entry["href"],
|
|
"item": item,
|
|
"fragment": fragment,
|
|
})
|
|
|
|
return entries
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Determine document order
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def spine_documents(book):
|
|
"""
|
|
Return EPUB document items in spine/reading order.
|
|
"""
|
|
|
|
result = []
|
|
|
|
by_id = {}
|
|
|
|
for item in book.get_items():
|
|
|
|
if item.get_type() == ITEM_DOCUMENT:
|
|
|
|
by_id[item.get_id()] = item
|
|
|
|
for entry in book.spine:
|
|
|
|
# ebooklib versions represent spine entries differently.
|
|
if isinstance(entry, tuple):
|
|
item_id = entry[0]
|
|
else:
|
|
item_id = entry
|
|
|
|
item = by_id.get(item_id)
|
|
|
|
if item is not None:
|
|
result.append(item)
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Chapter extraction using TOC
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def extract_toc_chapter(entries, index, class_styles=None):
|
|
"""
|
|
Extract one chapter based on its TOC position.
|
|
|
|
This handles:
|
|
|
|
chapter.xhtml#chapter1
|
|
chapter.xhtml#chapter2
|
|
|
|
correctly by stopping at the next TOC anchor.
|
|
|
|
It also handles chapters spanning multiple XHTML files.
|
|
"""
|
|
|
|
current = entries[index]
|
|
|
|
next_entry = (
|
|
entries[index + 1]
|
|
if index + 1 < len(entries)
|
|
else None
|
|
)
|
|
|
|
current_item = current["item"]
|
|
|
|
current_soup = parse_document(current_item, class_styles)
|
|
current_root = document_root(current_soup)
|
|
|
|
start = find_fragment(
|
|
current_soup,
|
|
current["fragment"],
|
|
)
|
|
|
|
# ---------------------------------------------------------------
|
|
# Find next chapter boundary if it is in this same document.
|
|
# ---------------------------------------------------------------
|
|
|
|
end = None
|
|
|
|
if (
|
|
next_entry is not None
|
|
and same_document(current_item, next_entry["item"])
|
|
):
|
|
end = find_fragment(
|
|
current_soup,
|
|
next_entry["fragment"],
|
|
)
|
|
|
|
# ---------------------------------------------------------------
|
|
# Same XHTML file.
|
|
# ---------------------------------------------------------------
|
|
|
|
if next_entry is not None and same_document(
|
|
current_item,
|
|
next_entry["item"],
|
|
):
|
|
|
|
return extract_range(
|
|
current_root,
|
|
start,
|
|
end,
|
|
)
|
|
|
|
# ---------------------------------------------------------------
|
|
# Different XHTML file.
|
|
#
|
|
# Everything from the current starting point through the end
|
|
# of the current document belongs to this chapter.
|
|
# ---------------------------------------------------------------
|
|
|
|
parts = []
|
|
|
|
parts.append(
|
|
extract_range(
|
|
current_root,
|
|
start,
|
|
None,
|
|
)
|
|
)
|
|
|
|
# If there is no next chapter, we stop here.
|
|
if next_entry is None:
|
|
return normalize_text("\n\n".join(
|
|
part for part in parts if part
|
|
))
|
|
|
|
# Add any spine documents between current and next document.
|
|
spine = spine_documents(book=None) if False else []
|
|
|
|
return normalize_text("\n\n".join(
|
|
part for part in parts if part
|
|
))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Improved multi-document chapter extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def document_order(book):
|
|
"""
|
|
Return document items in EPUB spine order.
|
|
|
|
If the spine is unusable, fall back to document iteration order.
|
|
"""
|
|
|
|
result = spine_documents(book)
|
|
|
|
if result:
|
|
return result
|
|
|
|
return [
|
|
item
|
|
for item in book.get_items()
|
|
if item.get_type() == ITEM_DOCUMENT
|
|
]
|
|
|
|
|
|
def item_index_map(book):
|
|
"""
|
|
Map EPUB document identity to its spine position.
|
|
"""
|
|
|
|
docs = document_order(book)
|
|
|
|
return {
|
|
id(item): index
|
|
for index, item in enumerate(docs)
|
|
}
|
|
|
|
|
|
def extract_chapter(book, entries, index, doc_positions, class_styles=None):
|
|
"""
|
|
Extract a complete chapter, including any XHTML documents between
|
|
its starting document and the next chapter's starting document.
|
|
"""
|
|
|
|
current = entries[index]
|
|
|
|
next_entry = (
|
|
entries[index + 1]
|
|
if index + 1 < len(entries)
|
|
else None
|
|
)
|
|
|
|
current_item = current["item"]
|
|
current_pos = doc_positions.get(id(current_item))
|
|
|
|
if current_pos is None:
|
|
current_soup = parse_document(current_item, class_styles)
|
|
root = document_root(current_soup)
|
|
|
|
start = find_fragment(
|
|
current_soup,
|
|
current["fragment"],
|
|
)
|
|
|
|
return extract_range(root, start, None)
|
|
|
|
# Position of next chapter's document.
|
|
next_pos = None
|
|
|
|
if next_entry is not None:
|
|
next_pos = doc_positions.get(id(next_entry["item"]))
|
|
|
|
# If we cannot determine the next position, only use current document.
|
|
if next_entry is not None and next_pos is None:
|
|
next_pos = current_pos + 1
|
|
|
|
docs = document_order(book)
|
|
|
|
pieces = []
|
|
|
|
# Process documents from current through the document before
|
|
# the next chapter.
|
|
end_pos = (
|
|
next_pos
|
|
if next_pos is not None
|
|
else len(docs)
|
|
)
|
|
|
|
for pos in range(current_pos, end_pos):
|
|
|
|
if pos >= len(docs):
|
|
break
|
|
|
|
item = docs[pos]
|
|
|
|
soup = parse_document(item, class_styles)
|
|
root = document_root(soup)
|
|
|
|
# -----------------------------------------------------------
|
|
# Current chapter's starting point.
|
|
# -----------------------------------------------------------
|
|
|
|
start = None
|
|
|
|
if pos == current_pos:
|
|
start = find_fragment(
|
|
soup,
|
|
current["fragment"],
|
|
)
|
|
|
|
# -----------------------------------------------------------
|
|
# Next chapter starts in this same document.
|
|
# -----------------------------------------------------------
|
|
|
|
end = None
|
|
|
|
if (
|
|
next_entry is not None
|
|
and pos == next_pos
|
|
and same_document(item, next_entry["item"])
|
|
):
|
|
end = find_fragment(
|
|
soup,
|
|
next_entry["fragment"],
|
|
)
|
|
|
|
# If the next chapter starts at the beginning of this document,
|
|
# nothing from this document belongs to the current chapter.
|
|
if pos == next_pos and next_entry is not None:
|
|
|
|
if next_entry["fragment"]:
|
|
if end is None:
|
|
continue
|
|
else:
|
|
# No fragment means the next chapter starts at the
|
|
# beginning of the document.
|
|
continue
|
|
|
|
text = extract_range(
|
|
root,
|
|
start,
|
|
end,
|
|
)
|
|
|
|
if text:
|
|
pieces.append(text)
|
|
|
|
return normalize_text(
|
|
"\n\n".join(pieces)
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fallback: spine-based extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def fallback_spine_chapters(book, class_styles=None):
|
|
"""
|
|
Fallback when no usable TOC exists.
|
|
|
|
Each XHTML document becomes one output chapter.
|
|
"""
|
|
|
|
chapters = []
|
|
|
|
for item in document_order(book):
|
|
|
|
soup = parse_document(item, class_styles)
|
|
root = document_root(soup)
|
|
|
|
text = html_to_text(root)
|
|
|
|
if not text:
|
|
continue
|
|
|
|
chapters.append({
|
|
"title": "",
|
|
"text": text,
|
|
})
|
|
|
|
return chapters
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Duplicate removal
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def comparison_text(text):
|
|
"""
|
|
Normalize text for duplicate detection.
|
|
"""
|
|
|
|
return re.sub(
|
|
r"\s+",
|
|
" ",
|
|
text,
|
|
).strip().lower()
|
|
|
|
|
|
def remove_duplicate_chapters(chapters):
|
|
"""
|
|
Remove immediately duplicated chapter content.
|
|
|
|
EPUB TOCs occasionally contain duplicate navigation entries.
|
|
"""
|
|
|
|
result = []
|
|
previous = None
|
|
|
|
for chapter in chapters:
|
|
|
|
normalized = comparison_text(
|
|
chapter["text"]
|
|
)
|
|
|
|
if not normalized:
|
|
continue
|
|
|
|
if normalized == previous:
|
|
continue
|
|
|
|
result.append(chapter)
|
|
previous = normalized
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main conversion
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def convert_epub(epub_path, output_dir):
|
|
|
|
epub_path = Path(epub_path)
|
|
output_dir = Path(output_dir)
|
|
|
|
if epub is None:
|
|
print(
|
|
"Error: the EPUB converter needs ebooklib. Install dependencies "
|
|
"with: python3 -m pip install -r requirements.txt"
|
|
)
|
|
sys.exit(1)
|
|
|
|
output_dir.mkdir(
|
|
parents=True,
|
|
exist_ok=True,
|
|
)
|
|
|
|
print(f"Reading: {epub_path}")
|
|
|
|
try:
|
|
book = epub.read_epub(
|
|
str(epub_path)
|
|
)
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f"Error: could not read EPUB:\n{exc}"
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
class_styles = stylesheet_class_styles(book)
|
|
|
|
# ---------------------------------------------------------------
|
|
# First choice: EPUB TOC.
|
|
# ---------------------------------------------------------------
|
|
|
|
entries = resolve_toc(book)
|
|
|
|
chapters = []
|
|
|
|
if entries:
|
|
|
|
print(
|
|
f"TOC entries found: {len(entries)}"
|
|
)
|
|
|
|
positions = item_index_map(book)
|
|
|
|
for index, entry in enumerate(entries):
|
|
|
|
text = extract_chapter(
|
|
book,
|
|
entries,
|
|
index,
|
|
positions,
|
|
class_styles,
|
|
)
|
|
|
|
text = normalize_text(text)
|
|
|
|
if not text:
|
|
print(
|
|
f"Skipping empty TOC entry: "
|
|
f"{entry['title']!r}"
|
|
)
|
|
continue
|
|
|
|
chapters.append({
|
|
"title": entry["title"],
|
|
"text": text,
|
|
})
|
|
|
|
# ---------------------------------------------------------------
|
|
# Fallback: EPUB spine.
|
|
# ---------------------------------------------------------------
|
|
|
|
if not chapters:
|
|
|
|
print(
|
|
"No usable TOC chapters found."
|
|
)
|
|
|
|
print(
|
|
"Falling back to EPUB reading order..."
|
|
)
|
|
|
|
chapters = fallback_spine_chapters(
|
|
book,
|
|
class_styles,
|
|
)
|
|
|
|
for chapter in chapters:
|
|
text, style_records = decode_styled_text(
|
|
normalize_text(chapter["text"])
|
|
)
|
|
chapter["text"] = text
|
|
chapter["style_records"] = style_records
|
|
|
|
# ---------------------------------------------------------------
|
|
# Remove duplicates.
|
|
# ---------------------------------------------------------------
|
|
|
|
chapters = remove_duplicate_chapters(
|
|
chapters
|
|
)
|
|
|
|
if not chapters:
|
|
|
|
print(
|
|
"ERROR: no readable text was found."
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
print(
|
|
f"Chapters to write: {len(chapters)}"
|
|
)
|
|
|
|
print()
|
|
|
|
# ---------------------------------------------------------------
|
|
# Write files.
|
|
# ---------------------------------------------------------------
|
|
|
|
toc_entries = []
|
|
written_sections = []
|
|
used_names = set()
|
|
|
|
for number, chapter in enumerate(
|
|
chapters,
|
|
start=1,
|
|
):
|
|
|
|
title = output_section_name(
|
|
chapter["title"],
|
|
number,
|
|
used_names,
|
|
)
|
|
|
|
output_file = (
|
|
output_dir /
|
|
f"{title}.txt"
|
|
)
|
|
|
|
output_file.write_text(
|
|
chapter["text"],
|
|
encoding="utf-8",
|
|
)
|
|
|
|
sidecar_file = output_dir / f"{title}.rsvp"
|
|
write_style_sidecar(
|
|
sidecar_file,
|
|
chapter["text"],
|
|
chapter["style_records"],
|
|
)
|
|
|
|
toc_entries.append(title)
|
|
written_sections.append((output_file.name, chapter["text"]))
|
|
print(
|
|
f"{number:4}. "
|
|
f"{title} "
|
|
f"-> {output_file.name}, {sidecar_file.name}"
|
|
)
|
|
|
|
toc_file = output_dir / "TOC.txt"
|
|
toc_file.write_text(
|
|
"\n".join(toc_entries) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
report_file, has_unsupported_characters = write_conversion_report(
|
|
output_dir,
|
|
toc_entries,
|
|
written_sections,
|
|
)
|
|
|
|
print()
|
|
print(
|
|
f"Finished: {len(chapters)} files"
|
|
)
|
|
|
|
print(
|
|
f"Output: {output_dir.resolve()}"
|
|
)
|
|
|
|
if has_unsupported_characters:
|
|
print(
|
|
"WARNING: unsupported characters remain; review "
|
|
f"{report_file.name} before copying this book to the device."
|
|
)
|
|
else:
|
|
print(
|
|
f"Conversion report: {report_file.name} (ASCII-safe)"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Command line interface
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def main():
|
|
|
|
if len(sys.argv) < 2:
|
|
|
|
print(
|
|
"Usage:\n"
|
|
" python epub_to_chapters.py book.epub\n"
|
|
" python epub_to_chapters.py "
|
|
"book.epub output_directory"
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
epub_path = Path(
|
|
sys.argv[1]
|
|
)
|
|
|
|
if not epub_path.exists():
|
|
|
|
print(
|
|
f"Error: file not found: "
|
|
f"{epub_path}"
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
if not epub_path.is_file():
|
|
|
|
print(
|
|
f"Error: not a file: "
|
|
f"{epub_path}"
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
if len(sys.argv) >= 3:
|
|
|
|
output_dir = Path(
|
|
sys.argv[2]
|
|
)
|
|
|
|
else:
|
|
|
|
output_dir = Path(
|
|
"chapters"
|
|
)
|
|
|
|
convert_epub(
|
|
epub_path,
|
|
output_dir,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|