Book-Reader-Assist/tools/split_book.py

"""Export a book PDF to text and split it into per-chapter source-text.md files.

Usage:  python tools/split_book.py "<Book Title>"

Reads library/<Book Title>/source-file.pdf and writes
library/<Book Title>/<Chapter-XX-Title-Slug>/<Chapter-XX>-source-text.md for each section defined in SECTIONS
below (add an entry for a new book). Sections are (folder, title, first_page, last_page,
note) with 1-based PDF pages. Pages not covered by any section are reported.
"""
import json
import re
import sys
from pathlib import Path

import fitz  # PyMuPDF

ROOT = Path(__file__).resolve().parent.parent / "library"

YBYE_NOTE_AFTER = "Includes the Step action plan that closes this step."
YBYE_NOTE_BEFORE = "Includes the Step introduction page that precedes the chapter."


def ch(n):
    return f"Chapter-{n:02d}"


def slug(title, limit=60):
    """Title -> folder-name slug (see AGENTS.md Naming Convention)."""
    t = re.sub(r"^\d+(\.\d+)?[a-z]?\s*-\s*", "", title.strip())  # drop the book's own numbering prefix
    t = re.sub(r"[:,.'\"‘’“”?/\()]", "", t)
    t = re.sub(r"[^\w\s-]", "", t)
    words, out = t.split(), ""
    for w in words:
        if len(out) + len(w) + (1 if out else 0) > limit:
            break
        out = f"{out}-{w}" if out else w
    return re.sub(r"-+", "-", out) or "Untitled"


def folder_name(key, title):
    """Chapter-05 + title -> Chapter-05-Anatomy-of-a-Specification; Front/Back-Matter keep their name."""
    return key if key in ("Front-Matter", "Back-Matter") else f"{key}-{slug(title)}"


def build_cetx():
    d = fitz.open(ROOT / "Context Engineering" / "source-file.pdf")
    top = [(t, p) for lvl, t, p in d.get_toc() if lvl == 1]
    secs = [("Front-Matter", "Front matter", 1, top[0][1] - 1, "")]
    for i, (t, p) in enumerate(top):
        end = (top[i + 1][1] - 1) if i + 1 < len(top) else len(d)
        secs.append((ch(i + 1), t, p, end, ""))
    return secs


def build_12wy():
    d = fitz.open(ROOT / "The 12 Week Year" / "source-file.pdf")
    starts = []
    for lvl, t, p in d.get_toc():
        if t.startswith("Chapter ") and lvl in (1, 2):
            num = int(t.split(":")[0].split()[1])
            starts.append((num, t.split(":", 1)[1].strip(), p))
    secs = [("Front-Matter", "Front matter", 1, starts[0][2] - 1, "")]
    for i, (n, t, p) in enumerate(starts):
        end = starts[i + 1][2] - 1 if i + 1 < len(starts) else 168
        note = ""
        if n == 1:
            note = "Includes the Part I divider page that follows the chapter."
        secs.append((ch(n), t, p, end, note))
    secs.append(("Back-Matter", "References", 169, len(d), ""))
    return secs


def build_ybye():
    chapters = [  # (n, title, first, last, note)
        (0, "Your Best Is Yet to Come (Opening)", 11, 21, ""),
        (1, "Your Beliefs Shape Your Reality", 22, 29, YBYE_NOTE_BEFORE),
        (2, "Some Beliefs Hold You Back", 30, 38, ""),
        (3, "You Can Upgrade Your Beliefs", 39, 53, YBYE_NOTE_AFTER),
        (4, "Thinking Backward Is a Must", 54, 64, YBYE_NOTE_BEFORE),
        (5, "Regret Reveals Opportunity", 65, 73, ""),
        (6, "Gratitude Makes the Difference", 74, 82, YBYE_NOTE_AFTER),
        (7, "Great Goals Check Seven Boxes", 83, 102, YBYE_NOTE_BEFORE),
        (8, "Achievements and Habits Work Together", 103, 109, ""),
        (9, "Seriously, Risk Is Your Friend", 110, 124, YBYE_NOTE_AFTER),
        (10, "Your What Needs a Why", 125, 139, YBYE_NOTE_BEFORE),
        (11, "You Can Master Your Own Motivation", 140, 148, ""),
        (12, "The Journey Is Better with Friends", 149, 160, YBYE_NOTE_AFTER),
        (13, "One Journey Is Many Steps", 161, 170, YBYE_NOTE_BEFORE),
        (14, "You Can Trigger Success", 171, 177, ""),
        (15, "Visibility Is Essential", 178, 190, YBYE_NOTE_AFTER),
    ]
    secs = [("Front-Matter", "Front matter (title, copyright, endorsements, contents)", 1, 10, "")]
    secs += [(ch(n), t, a, b, note) for n, t, a, b, note in chapters]
    secs.append(("Back-Matter", "LEAP Principle, Sample Goal Templates, Notes, Thanks, Index, About the Author", 191, 235, ""))
    return secs  # pages 236-242 (ads, back cover) are intentionally excluded


def parse_table(book, first_col_prefix, skip_rows=0):
    rows = []
    for line in (ROOT / book / "book-structure.md").read_text(encoding="utf-8").splitlines():
        if line.startswith("|") and not line.startswith("| ---") and not line.startswith("| Reading"):
            rows.append([c.strip() for c in line.strip("|").split("|")])
    return rows


def build_sdd():
    d = fitz.open(ROOT / "Spec Driven Development" / "source-file.pdf")
    secs = [("Front-Matter", "Front matter (cover, title, contents)", 1, 10, "")]
    for r in parse_table("Spec Driven Development", ""):
        n = int(r[0].split("-")[1])
        a, b = r[1].replace("–", "-").split("-")
        secs.append((ch(n), r[2], int(a), int(b), ""))
    return secs


def build_focus():
    d = fitz.open(ROOT / "FOCUS Architecture" / "source-file.pdf")
    rows = []
    for r in parse_table("FOCUS Architecture", ""):
        rows.append((r[0], r[1], int(r[2])))
    secs = [("Front-Matter", "Front matter", 1, rows[0][2] - 1, "")]
    for i, (kind, title, start) in enumerate(rows):
        end = rows[i + 1][2] - 1 if i + 1 < len(rows) else len(d)
        folder = "Interlude" if kind == "Interlude" else ch(int(kind.split()[1]))
        secs.append((folder, title, start, end, ""))
    return secs


def build_inference():
    d = fitz.open(ROOT / "Inference Engineering" / "source-file.pdf")
    top = [(t, p) for lvl, t, p in d.get_toc() if lvl <= 2]
    # Skip the Table of Contents bookmark: it is front matter (pages 1-10 with the cover and TOC).
    top = [(t, p) for t, p in top if t != "Table of Contents"]
    last_body = next(p for t, p in top if t == "Acknowledgements")
    secs = [("Front-Matter", "Front matter (cover, title, contents)", 1, top[0][1] - 1, "")]
    body = [(t, p) for t, p in top if p < last_body]
    for i, (t, p) in enumerate(body):
        end = (body[i + 1][1] - 1) if i + 1 < len(body) else last_body - 1
        title = re.sub(r"^Chapter \d+:\s*", "", t)
        secs.append((ch(i + 1), title, p, end, ""))
    secs.append(("Back-Matter", "Acknowledgements", last_body, len(d), ""))
    return secs


def build_os32():
    # Scanned book (no bookmarks): printed page N is PDF page N + 22. Chapters that open a Part
    # take in the Part divider plate and its blank verso, two pages before the chapter plate.
    chapters = [  # (n, title, printed start page, part that opens here)
        (1, "Overview", 3, "I"),
        (2, "General Discussion and Background", 13, ""),
        (3, "The Tasking Model", 25, "II"),
        (4, "Interprocess Communications", 41, ""),
        (5, "Memory Management", 55, ""),
        (6, "The Hardware Interface", 81, ""),
        (7, "OS Initialization", 99, ""),
        (8, "Programming Interfaces", 115, ""),
        (9, "Application Programming", 129, "III"),
        (10, "Systems Programming", 141, ""),
        (11, "The MMURTL Monitor Program", 159, ""),
        (12, "The Debugger", 181, ""),
        (13, "Keyboard Service", 189, ""),
        (14, "File System Service", 201, ""),
        (15, "API Specification", 221, ""),
        (16, "MMURTL Sample Software", 265, ""),
        (17, "Introduction to the Source Code", 327, "IV"),
        (18, "Kernel Code", 335, ""),
        (19, "Memory Management Code", 377, ""),
        (20, "Timer Management Code", 405, ""),
        (21, "Initialization Code", 419, ""),
        (22, "Job Management Code", 439, ""),
        (23, "Debugger Code", 471, ""),
        (24, "Selected Device Driver Code", 497, ""),
        (25, "Keyboard Code", 541, ""),
        (26, "Video Code", 571, ""),
        (27, "File System Code", 591, ""),
        (28, "DASM: A 32-Bit Intel-Based Assembler", 663, "V"),
        (29, "CM32: A 32-Bit C Compiler", 697, ""),
    ]
    index_start = 719 + 22
    starts = [(n, t, s + 22 - (2 if part else 0), part) for n, t, s, part in chapters]
    secs = [("Front-Matter", "Front matter (covers, title, copyright, contents, preface, acknowledgments, about the author)", 1, starts[0][2] - 1, "")]
    for i, (n, t, a, part) in enumerate(starts):
        end = starts[i + 1][2] - 1 if i + 1 < len(starts) else index_start - 1
        note = f"Includes the Part {part} divider plate and its blank verso before the chapter." if part else ""
        secs.append((ch(n), t, a, end, note))
    secs.append(("Back-Matter", "Index, publisher ads, CD-ROM installation and contents, back cover", index_start, 772, ""))
    return secs


def build_three_laws():
    # The PDF has a reliable bookmark outline. Part divider pages are folded into
    # the chapter that follows them so every page belongs to exactly one section.
    d = fitz.open(ROOT / "The Three Laws of Performance" / "source-file.pdf")
    chapters = [  # (number, title, first PDF page)
        (1, "Transforming an Impossible Situation", 40),
        (2, "Where Is the Key to Performance?", 70),
        (3, "Rewriting a Future That's Already Written", 104),
        (4, "With So Many Books on Leadership, Why Are There So Few Leaders?", 130),
        (5, "The Self-Led Organization", 156),
        (6, "Who or What Is Leading Your Life?", 180),
        (7, "The Path to Mastery", 208),
        (8, "Breaking the Performance Barrier", 238),
    ]
    secs = [(
        "Front-Matter",
        "Front matter (cover, title, contents, notes, foreword, and introduction)",
        1,
        chapters[0][2] - 1,
        "",
    )]
    for i, (n, title, start) in enumerate(chapters):
        end = chapters[i + 1][2] - 1 if i + 1 < len(chapters) else 245
        part = {1: "I", 4: "II", 6: "III"}.get(n)
        note = f"Includes the Part {part} divider immediately before the chapter." if part else ""
        secs.append((ch(n), title, start, end, note))
    secs.append((
        "Back-Matter",
        "Appendix, notes, acknowledgments, author biographies, and index",
        246,
        len(d),
        "",
    ))
    return secs


def build_smart_calling():
    # Section divider pages are folded into the chapter that follows them.
    d = fitz.open(ROOT / "Smart Calling" / "source-file.pdf")
    chapters = []
    for level, title, page in d.get_toc():
        if level == 3 and title.startswith("Chapter "):
            match = re.match(r"Chapter (\d+):\s*(.*)", title)
            if match:
                chapters.append((int(match.group(1)), match.group(2).strip(), page))

    section_dividers = {2: 29, 7: 93, 17: 207}
    starts = [
        (number, title, section_dividers.get(number, page))
        for number, title, page in chapters
    ]
    secs = [(
        "Front-Matter",
        "Front matter and introduction",
        1,
        starts[0][2] - 1,
        "",
    )]
    for i, (number, title, start) in enumerate(starts):
        end = starts[i + 1][2] - 1 if i + 1 < len(starts) else 240
        section = {2: "Two", 7: "Three", 17: "Four"}.get(number)
        note = f"Includes the Section {section} divider immediately before the chapter." if section else ""
        secs.append((ch(number), title, start, end, note))
    secs.append(("Back-Matter", "Index and About Art Sobczak", 241, len(d), ""))
    return secs


def build_fanatical_prospecting():
    d = fitz.open(ROOT / "Fanatical Prospecting" / "source-file.pdf")
    chapters = []
    for level, title, page in d.get_toc():
        if level == 1 and title.startswith("Chapter "):
            match = re.match(r"Chapter (\d+)\s+(.*)", title)
            if match:
                chapters.append((int(match.group(1)), match.group(2).strip(), page))

    secs = [(
        "Front-Matter",
        "Front matter, foreword, and free prospecting resources note",
        1,
        chapters[0][2] - 1,
        "",
    )]
    for i, (number, title, start) in enumerate(chapters):
        end = chapters[i + 1][2] - 1 if i + 1 < len(chapters) else 288
        secs.append((ch(number), title, start, end, ""))
    secs.append((
        "Back-Matter",
        "Notes, acknowledgments, about the author, index, and back cover",
        289,
        len(d),
        "",
    ))
    return secs


def build_ogilvy_on_advertising():
    d = fitz.open(ROOT / "Ogilvy on Advertising" / "source-file.pdf")
    chapters = []
    back_start = None
    for level, title, page in d.get_toc():
        match = re.match(r"(\d+)\.\s+(.*)", title)
        if level == 1 and match:
            chapters.append((int(match.group(1)), match.group(2).strip(), page))
        elif level == 1 and title == "Useful reference sources":
            back_start = page

    if not chapters or back_start is None:
        raise SystemExit("Could not locate the Ogilvy chapter or back-matter bookmarks")

    secs = [(
        "Front-Matter",
        "Front matter, publisher's note, and contents",
        1,
        chapters[0][2] - 1,
        "",
    )]
    for i, (number, title, start) in enumerate(chapters):
        end = chapters[i + 1][2] - 1 if i + 1 < len(chapters) else back_start - 1
        secs.append((ch(number), title, start, end, ""))
    secs.append((
        "Back-Matter",
        "Useful reference sources, index, picture credits, and copyright",
        back_start,
        len(d),
        "",
    ))
    return secs


def build_direct_mail_solution():
    d = fitz.open(ROOT / "The Direct Mail Solution" / "source-file.pdf")
    chapters = []
    back_start = None
    for level, title, page in d.get_toc():
        match = re.match(r"Chapter (\d+):\s*(.*)", title)
        if level == 1 and match:
            chapters.append((int(match.group(1)), match.group(2).strip(), page))
        elif level == 1 and title == "About the Authors":
            back_start = page

    if not chapters or back_start is None:
        raise SystemExit("Could not locate The Direct Mail Solution chapter or back-matter bookmarks")

    secs = [(
        "Front-Matter",
        "Front matter, acknowledgements, and preface",
        1,
        chapters[0][2] - 1,
        "",
    )]
    for i, (number, title, start) in enumerate(chapters):
        end = chapters[i + 1][2] - 1 if i + 1 < len(chapters) else back_start - 1
        secs.append((ch(number), title, start, end, ""))
    secs.append((
        "Back-Matter",
        "About the authors, index, and book excerpt",
        back_start,
        len(d),
        "",
    ))
    return secs


def build_hype_handbook():
    d = fitz.open(ROOT / "The Hype Handbook" / "source-file.pdf")
    chapters = []
    back_start = None
    for level, title, page in d.get_toc():
        match = re.match(r"Hype Strategy #(\d+)\s+(.*)", title)
        if level == 1 and match:
            chapters.append((int(match.group(1)), match.group(2).strip(), page))
        elif level == 1 and title == "Epilogue":
            back_start = page

    if not chapters or back_start is None:
        raise SystemExit("Could not locate The Hype Handbook strategy or back-matter bookmarks")

    secs = [(
        "Front-Matter",
        "Front matter, acknowledgments, and introduction",
        1,
        chapters[0][2] - 1,
        "",
    )]
    for i, (number, title, start) in enumerate(chapters):
        end = chapters[i + 1][2] - 1 if i + 1 < len(chapters) else back_start - 1
        secs.append((ch(number), title, start, end, ""))
    secs.append((
        "Back-Matter",
        "Epilogue, bibliography, and index",
        back_start,
        len(d),
        "",
    ))
    return secs


def build_code_of_discipline():
    # Photocopied scan (no bookmarks); the scanner's OCR text layer is used as is. Part divider
    # pages are folded into the chapter that follows them. The book-level Introduction is
    # Chapter 00 and the book-level Conclusion is Chapter 10.
    chapters = [  # (n, title, first PDF page, part that opens here)
        (0, "Introduction: The Discipline Crisis", 5, ""),
        (1, "The War Against Comfort", 16, "1"),
        (2, "Grit Over Feelings", 32, ""),
        (3, "The Focus Formula", 46, ""),
        (4, "The Mental Resistance Trap", 59, "2"),
        (5, "The Science of Self-Discipline", 73, ""),
        (6, "Mastering the Morning Battlefield", 86, ""),
        (7, "The Discipline Code Framework", 99, "3"),
        (8, "The 90-Day Mental Toughness Protocol", 115, ""),
        (9, "The Discipline Code in Work, Fitness, and Life", 128, ""),
        (10, "Conclusion: Becoming a Person of Unbreakable Will", 141, ""),
    ]
    secs = [("Front-Matter", "Front matter (cover, inside cover, contents)", 1, 4, "")]
    for i, (n, title, start, part) in enumerate(chapters):
        end = chapters[i + 1][2] - 1 if i + 1 < len(chapters) else 153
        note = "Text is the scanner's OCR layer; expect character errors (e.g. 'Thaining', 'joumey'), dropped drop-cap letters, and OCR-mangled headings."
        if part:
            note += f" Includes the Part {part} divider immediately before the chapter."
        secs.append((ch(n), title, start, end, note))
    secs.append(("Back-Matter", "Blank pages (About the Author page not scanned) and back cover", 154, 156, ""))
    return secs


WATTLES_WORKS = [  # (title bookmark, short name)
    ("The Science of Getting Rich", "The Science of Getting Rich"),
    ("The Science of Being Well", "The Science of Being Well"),
    ("The Science of Being Great", "The Science of Being Great"),
    ("Hellfire Harrison (A Novel)", "Hellfire Harrison"),
    ("Jesus: The Man and His Work", "Jesus: The Man and His Work"),
    ("A New Christ", "A New Christ"),
    ("How to Get What You Want", "How to Get What You Want"),
    ("Making of the Man Who Can", "Making of the Man Who Can"),
    ("New Science of Living and Healing", "New Science of Living and Healing"),
    ("The Personal Power Course", "The Personal Power Course"),
]
WATTLES_NOVEL_ADS = ["Commencement Days", "The Court of Lucifer", "Ailmeny of the Orchard",
                     "A Cavalier of Virginia", "The Lead of honour", "My heart and Stephanie"]


def wattles_title(work, t):
    """Bookmark title -> chapter title; generic 'Chapter One' titles get the work's name."""
    t = re.sub(r"^Chapter [IVXL]+\.\s*", "", t)  # Science of Being Well
    t = re.sub(r"^Chapter \d+\s*[:-]\s*", "", t)  # Getting Rich, A New Christ, Man Who Can
    t = re.sub(r"^Lesson (\d+)\s+", r"Lesson \1 ", t)  # Personal Power Course
    m = re.match(r"^\d+\.\s*(.*?)\.?$", t)  # New Science of Living: '1. THE SOURCE OF WORK-POWER.'
    if m:
        t = re.sub(r"(?<=\s)(Of|And|In|On|To)\b", lambda w: w.group(1).lower(), m.group(1).title())
    if re.fullmatch(r"Chapter (\w+)", t):
        t = f"{work} {t}"
    return t.replace("Jesus�", "Jesus'")


def build_wattles():
    # One book folder for the whole collection, chapters numbered continuously across the ten
    # works. Each work's title page (and preface, where present) is folded into its first chapter.
    d = fitz.open(ROOT / "Wallace D. Wattles Ultimate Collection" / "source-file.pdf")
    toc = [(t, p) for lvl, t, p in d.get_toc() if lvl == 1 and p >= 5]
    chapters, work, pending_start = [], None, None
    for t, p in toc:
        new_work = next((s for b, s in WATTLES_WORKS if t.startswith(b)), None)
        if new_work:
            work, pending_start = new_work, p
            if work == "Jesus: The Man and His Work":  # a single-section work
                chapters.append((work, work, p))
                pending_start = None
            continue
        if t == "Preface" or t in WATTLES_NOVEL_ADS:
            continue
        chapters.append((work, wattles_title(work, t), pending_start or p))
        pending_start = None
    counts = {w: sum(1 for c in chapters if c[0] == w) for _, w in WATTLES_WORKS}
    secs = [("Front-Matter", "Cover, title page, and collection contents", 1, 4, "")]
    seen = {}
    for i, (w, title, start) in enumerate(chapters):
        end = chapters[i + 1][2] - 1 if i + 1 < len(chapters) else len(d)
        seen[w] = seen.get(w, 0) + 1
        note = f"Work: *{w}*, section {seen[w]} of {counts[w]}."
        if seen[w] == 1 and counts[w] > 1:
            note += " Includes the work's title page" + (" and Preface" if w in ("The Science of Getting Rich", "The Science of Being Well") else "") + "."
        if w == "Hellfire Harrison" and seen[w] == counts[w]:
            note += " Includes the publisher's advertisements for other novels (PDF pages 255-261) that close the novel."
        secs.append((ch(i + 1), title, start, end, note))
    return secs


BUILDERS = {
    "Developing Your Own 32-Bit Operating System": build_os32,
    "Context Engineering": build_cetx,
    "The 12 Week Year": build_12wy,
    "Your Best Year Ever": build_ybye,
    "Spec Driven Development": build_sdd,
    "FOCUS Architecture": build_focus,
    "Inference Engineering": build_inference,
    "The Three Laws of Performance": build_three_laws,
    "Smart Calling": build_smart_calling,
    "Fanatical Prospecting": build_fanatical_prospecting,
    "Ogilvy on Advertising": build_ogilvy_on_advertising,
    "The Direct Mail Solution": build_direct_mail_solution,
    "The Hype Handbook": build_hype_handbook,
    "The Code of Discipline": build_code_of_discipline,
    "Wallace D. Wattles Ultimate Collection": build_wattles,
}


def main(book):
    pdf = ROOT / book / "source-file.pdf"
    doc = fitz.open(pdf)
    secs = BUILDERS[book]()
    ocr_path = ROOT / book / "ocr-image-pages.json"
    ocr_pages = json.loads(ocr_path.read_text(encoding="utf-8")) if ocr_path.exists() else {}
    covered = {}
    for folder, title, a, b, note in secs:
        pages, blank = [], []
        for p in range(a, b + 1):
            if p in covered:
                raise SystemExit(f"page {p} in both {covered[p]} and {folder}")
            covered[p] = folder
            text = doc[p - 1].get_text().strip()
            if not text:
                blank.append(p)
                if str(p) in ocr_pages:
                    text = "[OCR from image-only PDF page; verify layout and illustrations in the PDF.]\n\n" + ocr_pages[str(p)]
            pages.append(f"<!-- PDF page {p} -->\n{text}\n")
        header = [
            f"# {book} — {folder}: {title}",
            f"- **Source**: /library/{book}/source-file.pdf",
            f"- **PDF pages**: {a}–{b}",
            f"- **Pages without text**: {', '.join(map(str, blank)) if blank else 'none'}",
        ]
        if blank and ocr_pages:
            header.append("- **OCR note**: OCR is supplied below for image-only pages; multi-column reading order may be imperfect, so consult the PDF for layout and visuals.")
        if note:
            header.append(f"- **Note**: {note}")
        out = ROOT / book / folder_name(folder, title)
        out.mkdir(exist_ok=True)
        (out / f"{folder}-source-text.md").write_text("\n".join(header) + "\n\n---\n\n" + "\n".join(pages), encoding="utf-8", newline="\n")
        print(f"{folder:14} pp {a}-{b} blank={blank} {title[:50]}")
    missing = [p for p in range(1, len(doc) + 1) if p not in covered]
    print("pages not covered:", missing)


if __name__ == "__main__":
    main(sys.argv[1])