modm_data.pdf2html.convert

  1# Copyright 2022, Niklas Hauser
  2# SPDX-License-Identifier: MPL-2.0
  3
  4import gc
  5import os
  6import hashlib
  7from anytree import RenderTree, PreOrderIter
  8from lxml import etree
  9from typing import Iterable
 10
 11from .html import format_document, write_html
 12from .render import annotate_debug_info
 13from ..utils import pkg_apply_patch, pkg_file_exists, apply_patch
 14from .ast import merge_area
 15from pathlib import Path
 16import pypdfium2 as pp
 17
 18
 19def _write_figures(root, html_file: Path, folder: Path, names: set):
 20    """
 21    Writes the vector graphics of all figures as SVG files and their bitmap
 22    images in their embedded format into the folder, so that the HTML only
 23    contains references to them.
 24
 25    The folder is shared by all HTML files of a document, which may be written
 26    by separate processes that each convert their own pages. The files are
 27    therefore named after the figure number and page, and the images after
 28    their content, which also saves identical images only once, like the
 29    hundreds of tiles of a pattern.
 30    """
 31    html_file, folder = Path(html_file), Path(folder)
 32    for node in PreOrderIter(root, filter_=lambda n: n.name == "figure"):
 33        # The page keeps the name unique, if the figure numbers start over
 34        name = "figure_" + (f"{node.number}_" if node.number >= 0 else "") + f"p{node.obj._page.number}"
 35        # Figures without caption and side-by-side figures share their name
 36        name = next(n for n in [name] + [f"{name}_{ii}" for ii in range(1, 100)] if n not in names)
 37        names.add(name)
 38
 39        files = []
 40        for image in node.obj.images:
 41            suffix, data = image.encode()
 42            files.append(f"image_{hashlib.sha1(data).hexdigest()[:16]}{suffix}")
 43            folder.mkdir(parents=True, exist_ok=True)
 44            (folder / files[-1]).write_bytes(data)
 45        if (svg := node.obj.as_svg(files)) is not None:
 46            folder.mkdir(parents=True, exist_ok=True)
 47            etree.ElementTree(svg).write(folder / f"{name}.svg", pretty_print=True)
 48            # The SVG draws the images itself, so the HTML only references the SVG
 49            files = [f"{name}.svg"]
 50        relative = Path(os.path.relpath(folder, html_file.parent))
 51        node._srcs = [(relative / file).as_posix() for file in dict.fromkeys(files)]
 52
 53
 54def convert(
 55    doc: pp.PdfDocument,
 56    page_range: Iterable[int],
 57    output_path: Path,
 58    figures_path: Path = None,
 59    format_chapters: bool = False,
 60    pretty: bool = True,
 61    render_html: bool = True,
 62    render_pdf: bool = False,
 63    render_all: bool = False,
 64    show_ast: bool = False,
 65    show_tree: bool = False,
 66    show_tags: bool = False,
 67) -> bool:
 68    document = None
 69    debug_doc = None
 70    debug_index = 0
 71    for page in doc.pages(page_range):
 72        # The previous pages create millions of long-lived objects, which the
 73        # garbage collector would otherwise traverse again in every collection
 74        # of the oldest generation, taking a third of the conversion time.
 75        gc.freeze()
 76        if not render_all and not page.is_relevant:
 77            continue
 78        print(f"\n\n=== {page.top} #{page.number} ===\n")
 79
 80        if show_tags:
 81            for struct in page.structures:
 82                print(struct.descr())
 83
 84        if show_tree or render_html or show_ast:
 85            areas = page.content_ast
 86            if show_ast:
 87                print()
 88                for area in areas:
 89                    print(RenderTree(area))
 90            if show_tree or render_html:
 91                for area in areas:
 92                    document = merge_area(document, area)
 93
 94        if render_pdf:
 95            debug_doc = annotate_debug_info(page, debug_doc, debug_index)
 96            debug_index += 1
 97
 98    if render_pdf:
 99        with open(f"debug_{output_path.stem}.pdf", "wb") as file_handle:
100            pp.PdfDocument(debug_doc).save(file_handle)
101
102    if show_tree or render_html:
103        if document is None:
104            print("No pages parsed, empty document!")
105            return True
106
107        document = doc._normalize(document)
108        if show_tree:
109            print(RenderTree(document))
110
111        if render_html:
112            names = set()
113            if format_chapters:
114                for chapter in document.children:
115                    if chapter.name == "chapter":
116                        print(f"\nFormatting HTML for '{chapter.title}'")
117                        output_file = f"{output_path}/chapter_{chapter._filename}.html"
118                        _write_figures(chapter, output_file, figures_path or output_path / "figures", names)
119                        html = format_document(chapter)
120                        print(f"\nWriting HTML '{output_file}'")
121                        write_html(html, output_file, pretty=pretty)
122            else:
123                print("\nFormatting HTML")
124                _write_figures(document, output_path, figures_path or output_path.with_suffix("") / "figures", names)
125                html = format_document(document)
126                print(f"\nWriting HTML '{str(output_path)}'")
127                write_html(html, str(output_path), pretty=pretty)
128
129    return True
130
131
132def patch(doc, data_module, output_path: Path, patch_file: Path = None) -> bool:
133    if patch_file is None:
134        # First try the patch file for the specific version
135        patch_file = f"{doc.name}.patch"
136        if not pkg_file_exists(data_module, patch_file):
137            # Then try the patch file shared between versions
138            patch_file = f"{doc.name.split('-')[0]}.patch"
139            if not pkg_file_exists(data_module, patch_file):
140                return True
141        return pkg_apply_patch(data_module, patch_file, output_path)
142    return apply_patch(patch_file, output_path)