modm_data.pdf2html.page

  1# Copyright 2022, Niklas Hauser
  2# SPDX-License-Identifier: MPL-2.0
  3
  4import logging
  5import statistics
  6from typing import Callable
  7from functools import cached_property
  8from collections import defaultdict
  9from .table import Table
 10from .figure import Figure
 11from .line import CharLine
 12from ..utils import Rectangle, Region
 13from ..pdf import Page as PdfPage, Character
 14from anytree import Node
 15
 16
 17_LOGGER = logging.getLogger(__name__)
 18
 19
 20class Page(PdfPage):
 21    def __init__(self, document, index: int):
 22        super().__init__(document, index)
 23        self._template = "default"
 24        self.is_relevant: bool = True
 25        """Is this page relevant for the conversion?"""
 26
 27    def _unicode_filter(self, code: int) -> int:
 28        return code
 29
 30    @cached_property
 31    def _spacing(self) -> dict[str, float]:
 32        content = 0.1
 33        return {
 34            # Horizontal spacing: left->right
 35            "x_em": 0.01 * self.width,
 36            "x_left": content * self.width,
 37            "x_right": (1 - content) * self.width,
 38            "x_content": 0.2 * self.width,
 39            # Vertical spacing: bottom->top
 40            "y_em": 0.01 * self.height,
 41            # Max table line thickness
 42            "y_tline": 0.005 * self.height,
 43            # Max line height distance to detect paragraphs
 44            "lh": 0.9,
 45            # Max line height distance to detect super-/subscript
 46            "sc": 0.3,
 47            # Table header cell bold text threshold
 48            "th": 0.3,
 49        }
 50
 51    def _line_size(self, line: CharLine) -> str:
 52        rsize = line.height
 53        if rsize >= 17.5:
 54            return "h1"
 55        elif rsize >= 15.5:
 56            return "h2"
 57        elif rsize >= 13.5:
 58            return "h3"
 59        elif rsize >= 11.4:
 60            return "h4"
 61        elif rsize >= 8.5:
 62            return "n"
 63        else:
 64            return "fn"
 65
 66    def _colors(self, color: int) -> str:
 67        if 0xFF <= color <= 0xFF:
 68            return "black"
 69        if 0xFFFFFFFF <= color <= 0xFFFFFFFF:
 70            return "white"
 71        return "unknown"
 72
 73    @cached_property
 74    def _areas(self) -> dict[str, list[Rectangle] | Rectangle]:
 75        content = Rectangle(0.1, 0.1, 0.9, 0.9)
 76        areas = {"content": [content]}
 77        scaled_areas = {}
 78
 79        def _s(r):
 80            return Rectangle(r.left * self.width, r.bottom * self.height, r.right * self.width, r.top * self.height)
 81
 82        for name, area in areas.items():
 83            scaled_areas[name] = [_s(r) for r in area] if isinstance(area, list) else _s(area)
 84        return scaled_areas
 85
 86    def _char_properties(self, line, char):
 87        cp = {
 88            "superscript": False,
 89            "subscript": False,
 90            "bold": any(frag in char.font for frag in {"Bold"}),
 91            "italic": any(frag in char.font for frag in {"Italic", "Oblique"}),
 92            "underline": (char.objlink or char.weblink) is not None,
 93            "size": round(line.height),
 94            "relsize": self._line_size(line),
 95            "char": chr(char.unicode),
 96        }
 97        if line.rotation:
 98            if char.origin.x < (line.origin - 0.25 * line.height):
 99                cp["superscript"] = True
100            elif char.origin.x > (line.origin + 0.15 * line.height):
101                cp["subscript"] = True
102        elif char.origin.y > (line.origin + 0.25 * line.height):
103            cp["superscript"] = True
104        elif char.origin.y < (line.origin - 0.15 * line.height):
105            cp["subscript"] = True
106        return cp
107
108    def text_in_named_area(self, name: str, check_length: bool = True) -> str | None:
109        """
110        Find all text in the named area.
111
112        :param name: the name of the area(s) to query.
113        :param check_length: assert that the text has a length.
114        :return: the concatenated text of the named area(s) or `None` if area not found.
115        """
116        if name not in self._areas:
117            return None
118        text = ""
119        areas = self._areas[name]
120        if not isinstance(areas, list):
121            areas = [areas]
122        for area in areas:
123            text += self.text_in_area(area)
124        if check_length:
125            assert text
126        return text
127
128    def charlines_in_area(
129        self, area: Rectangle, predicate: Callable[[Character], bool] = None, rtol: float = None
130    ) -> list[CharLine]:
131        """
132        Coalesce the characters in the area and predicate into lines.
133
134        1. Every character in the area is filtered by the `predicate`.
135        2. Character orientation is split into horizontal (left->right) and
136           vertical (bottom->top) character lines sorted by x or y position.
137           Lines containing only whitespace are discarded.
138        3. Overlapping character lines are merged into sub- and superscript
139           using `rtol * max(current_line.height, next_line.height)` as the
140           tolerance for checking if the lines overlap.
141        4. The characters in the merged lines are re-sorted by origin.
142
143        :param area: Area to search for characters.
144        :param predicate: Function to discard characters in the area or include all by default.
145        :param rtol: Relative tolerance to separate lines vertically or use `sc` spacing by default.
146        :return: A list of character lines sorted by x or y position.
147        """
148        if rtol is None:
149            rtol = self._spacing["sc"]
150        # Split all chars into lines based on rounded origin
151        origin_lines_y = defaultdict(list)
152        origin_lines_x = defaultdict(list)
153        for char in self.chars_in_area(area):
154            # Ignore all characters we don't want
155            if predicate is not None and not predicate(char):
156                continue
157            cunicode = self._unicode_filter(char.unicode)
158            if cunicode is None:
159                continue
160            char.unicode = cunicode
161            if char.unicode < 32 and char.unicode not in {0xA}:
162                continue
163            # Ignore characters without width that are not spaces
164            if not char.width and char.unicode not in {0xA, 0xD, 0x20}:
165                _LOGGER.error(f"Unknown char width for {char}: {char.bbox}")
166            # Split up the chars depending on the orientation
167            if 45 < char.rotation <= 135 or 225 < char.rotation <= 315:
168                origin_lines_x[round(char.origin.x, 1)].append(char)
169            elif char.rotation <= 45 or 135 < char.rotation <= 225 or 315 < char.rotation:
170                origin_lines_y[round(char.origin.y, 1)].append(char)
171            else:
172                _LOGGER.error("Unknown char rotation:", char, char.rotation)
173
174        # Convert characters into lines
175        bbox_lines_y = []
176        for chars in origin_lines_y.values():
177            # Remove lines with whitespace only
178            if all(c.unicode in {0xA, 0xD, 0x20} for c in chars):
179                continue
180            origin = statistics.fmean(c.origin.y for c in chars)
181            line = CharLine(
182                self,
183                chars,
184                min(c.bbox.bottom for c in chars),
185                origin,
186                max(c.bbox.top for c in chars),
187                max(c.height for c in chars),
188                sort_origin=self.height - origin,
189            )
190            bbox_lines_y.append(line)
191            # print(line, line.top, line.origin, line.bottom, line.height)
192        bbox_lines = sorted(bbox_lines_y, key=lambda line: line._sort_origin)
193
194        bbox_lines_x = []
195        for chars in origin_lines_x.values():
196            # Remove lines with whitespace only
197            if all(c.unicode in {0xA, 0xD, 0x20} for c in chars):
198                continue
199            line = CharLine(
200                self,
201                chars,
202                min(c.bbox.left for c in chars),
203                statistics.fmean(c.origin.x for c in chars),
204                max(c.bbox.right for c in chars),
205                max(c.width for c in chars),
206                270 if sum(c.rotation for c in chars) <= 135 * len(chars) else 90,
207            )
208            bbox_lines_x.append(line)
209        bbox_lines += sorted(bbox_lines_x, key=lambda line: line._sort_origin)
210
211        if not bbox_lines:
212            return []
213
214        # Merge lines that have overlapping bbox_lines
215        # FIXME: This merges lines that "collide" vertically like in formulas
216        merged_lines = []
217        current_line = bbox_lines[0]
218        for next_line in bbox_lines[1:]:
219            height = max(current_line.height, next_line.height)
220            # Calculate overlap via normalize origin (increasing with line index)
221            if (current_line._sort_origin + rtol * height) > (next_line._sort_origin - rtol * height):
222                # if line.rotation or self.rotation:
223                #     # The next line overlaps this one, we merge the shorter line
224                #     # (typically super- and subscript) into taller line
225                #     use_current = len(current_line.chars) >= len(next_line.chars)
226                # else:
227                use_current = current_line.height >= next_line.height
228                line = current_line if use_current else next_line
229                current_line = CharLine(
230                    self,
231                    current_line.chars + next_line.chars,
232                    line.bottom,
233                    line.origin,
234                    line.top,
235                    height,
236                    line.rotation,
237                    sort_origin=line._sort_origin,
238                )
239            else:
240                # The next line does not overlap the current line
241                merged_lines.append(current_line)
242                current_line = next_line
243        # append last line
244        merged_lines.append(current_line)
245
246        # Sort all lines horizontally based on character origin
247        sorted_lines = []
248        for line in merged_lines:
249            if line.rotation == 90:
250
251                def sort_key(char):
252                    if char.unicode in {0xA, 0xD}:
253                        return char.tbbox.midpoint.y - 1e9
254                    return char.tbbox.midpoint.y
255            elif line.rotation == 270:
256
257                def sort_key(char):
258                    if char.unicode in {0xA, 0xD}:
259                        return -char.tbbox.midpoint.y + 1e9
260                    return -char.tbbox.midpoint.y
261            else:
262
263                def sort_key(char):
264                    if char.unicode in {0xA, 0xD}:
265                        return char.origin.x + 1e9
266                    return char.origin.x
267
268            chars = sorted(line.chars, key=sort_key)
269            if not line.rotation:
270                chars = self._chars_with_bullet_space(chars)
271            sorted_lines.append(
272                CharLine(
273                    self,
274                    chars,
275                    line.bottom,
276                    line.origin,
277                    line.top,
278                    line.height,
279                    line.rotation,
280                    area.left,
281                    sort_origin=line._sort_origin,
282                )
283            )
284
285        return sorted_lines
286
287    @staticmethod
288    def _chars_with_bullet_space(chars: list) -> list:
289        # Newer pdfium does not generate a space for every visual gap anymore,
290        # but the list detection relies on the one between bullet and text.
291        for ii, (bullet, char) in enumerate(zip(chars, chars[1:]), 1):
292            if bullet.unicode == 0x20:
293                continue
294            if (
295                bullet.char in {"•", "–"}
296                and char.unicode not in {0x20, 0xA, 0xD}
297                and char.origin.x - bullet.bbox.right >= 0.25 * bullet.height
298            ):
299                return chars[:ii] + [char.generated_space()] + chars[ii:]
300            break
301        return chars
302
303    def graphic_bboxes_in_area(
304        self, area: Rectangle, with_graphics: bool = True
305    ) -> list[tuple[Rectangle, Table | Figure | None]]:
306        """
307        Coalesce the graphics in the area into full width bounding boxes.
308
309        1. Group vertically overlapping graphics.
310        2. Widen the overlapped graphics bounding boxes to the edges of the area.
311
312        :param area: area to search for content.
313        :param with_graphics: search for graphics in the area.
314        :return: list of tuples (bounding box, graphic objects or `None`).
315        """
316        if with_graphics:
317            graphics = self.graphics_in_area(area)
318            regions = []
319            # Check if graphics bounding boxes overlap vertically and group them
320            for graphic in sorted(graphics, key=lambda g: (-g.bbox.top, g.bbox.x)):
321                gbbox = graphic.bbox.joined(graphic.cbbox) if graphic.cbbox else graphic.bbox
322                for reg in regions:
323                    if reg.overlaps(gbbox.bottom, gbbox.top):
324                        # They overlap, so merge them
325                        reg.v0 = min(reg.v0, gbbox.bottom)
326                        reg.v1 = max(reg.v1, gbbox.top)
327                        reg.objs.append(graphic)
328                        break
329                else:
330                    regions.append(Region(gbbox.bottom, gbbox.top, graphic))
331
332            # print(regions)
333            # Coalesce all overlapped graphics objects into full width areas
334            areas = []
335            ypos = area.top
336            for reg in regions:
337                if ypos - reg.v1 > self._spacing["y_em"]:
338                    areas.append((Rectangle(area.left, reg.v1, area.right, ypos), None))
339                for obj in reg.objs:
340                    oarea = obj.bbox.joined(obj.cbbox) if obj.cbbox else obj.bbox
341                    areas.append((oarea, obj))
342                ypos = reg.v0
343            areas.append((Rectangle(area.left, area.bottom, area.right, ypos), None))
344        else:
345            areas = [(area, None)]
346        return areas
347
348    def objects_in_area(self, area: Rectangle, with_graphics: bool = True) -> list[CharLine | Table | Figure]:
349        """
350        Find all content objects in this area.
351
352        :param area: area to search for content.
353        :param with_graphics: search for graphics in the area.
354        :return: list of content objects sorted top to bottom.
355        """
356        self._link_characters()
357        areas = self.graphic_bboxes_in_area(area, with_graphics)
358        objects = []
359        for narea, obj in areas:
360            if obj is None:
361                objects += self.charlines_in_area(narea)
362            else:
363                oarea = obj.bbox.joined(obj.cbbox) if obj.cbbox else obj.bbox
364
365                def predicate(c):
366                    return not obj.bbox.contains(c.origin)
367
368                lines = self.charlines_in_area(oarea, predicate)
369                # print(obj, oarea, lines, [line.content for line in lines])
370                objects += list(sorted(lines + [obj], key=lambda o: (-o.bbox.y, o.bbox.x)))
371        return objects
372
373    def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]:
374        """
375        Find all tables and figures in this area.
376
377        :param area: area to search for graphics.
378        :return: list of tables and figures.
379        """
380        return []
381
382    def ast_in_area(self, area: Rectangle, with_graphics: bool = True) -> Node:
383        """
384        Convert the area content into an abstract syntax tree.
385
386        :param area: area to search for content.
387        :param with_graphics: including graphics in the area.
388        :return: An abstract syntax tree including the content formatting.
389        """
390        return Node("area", obj=area, xpos=int(area.left), page=self)
391
392    @property
393    def content_ast(self) -> list[Node]:
394        """The abstract syntax trees in the content area."""
395        ast = []
396        with_graphics = True
397        for area in self._areas["content"]:
398            ast.append(self.ast_in_area(area, with_graphics=with_graphics))
399        # Add a page node to the first leaf to keep track of where a page starts
400        first_leaf = next((n for n in iter(ast[0].descendants) if n.is_leaf), ast[0])
401        Node("page", parent=first_leaf, xpos=first_leaf.xpos, number=self.number)
402        return ast
403
404    @property
405    def content_objects(self) -> list[CharLine | Table | Figure]:
406        """All objects in the content areas."""
407        objs = []
408        for area in self._areas["content"]:
409            objs.extend(self.objects_in_area(area))
410        return objs
411
412    @property
413    def content_graphics(self) -> list[Table | Figure]:
414        """All graphics in the content areas."""
415        objs = []
416        for area in self._areas["content"]:
417            objs.extend(self.graphics_in_area(area))
418        return objs
419
420    @property
421    def content_lines(self) -> list[CharLine]:
422        """All lines in the content areas."""
423        objs = []
424        for area in self._areas["content"]:
425            objs.extend(self.charlines_in_area(area))
426        return objs
427
428    @property
429    def content_tables(self) -> list[Table]:
430        """All tables in the content areas."""
431        return [o for o in self.content_graphics if isinstance(o, Table)]
432
433    @property
434    def content_figures(self) -> list[Figure]:
435        """All figures in the content areas."""
436        return [o for o in self.content_graphics if isinstance(o, Figure)]
437
438    def __repr__(self) -> str:
439        return f"Page({self.number})"