modm_data.pdf2html.page

  1# Copyright 2022, Niklas Hauser
  2# SPDX-License-Identifier: MPL-2.0
  3
  4import logging
  5import math
  6import statistics
  7from typing import Callable
  8from functools import cached_property
  9from collections import defaultdict
 10from .table import Table
 11from .figure import Figure
 12from .line import CharLine
 13from ..utils import Rectangle, Region, list_strip
 14from ..pdf import Page as PdfPage, Character
 15from anytree import Node
 16
 17
 18_LOGGER = logging.getLogger(__name__)
 19
 20
 21class Page(PdfPage):
 22    def __init__(self, document, index: int):
 23        super().__init__(document, index)
 24        self._template = "default"
 25        self.is_relevant: bool = True
 26        """Is this page relevant for the conversion?"""
 27
 28    def _unicode_filter(self, code: int) -> int:
 29        return code
 30
 31    @cached_property
 32    def _spacing(self) -> dict[str, float]:
 33        content = 0.1
 34        return {
 35            # Horizontal spacing: left->right
 36            "x_em": 0.01 * self.width,
 37            "x_left": content * self.width,
 38            "x_right": (1 - content) * self.width,
 39            "x_content": 0.2 * self.width,
 40            # Vertical spacing: bottom->top
 41            "y_em": 0.01 * self.height,
 42            # Max table line thickness
 43            "y_tline": 0.005 * self.height,
 44            # Max line height distance to detect paragraphs
 45            "lh": 0.9,
 46            # Max line height distance to detect super-/subscript
 47            "sc": 0.3,
 48            # Table header cell bold text threshold
 49            "th": 0.3,
 50        }
 51
 52    def _line_size(self, line: CharLine) -> str:
 53        rsize = line.height
 54        if rsize >= 17.5:
 55            return "h1"
 56        elif rsize >= 15.5:
 57            return "h2"
 58        elif rsize >= 13.5:
 59            return "h3"
 60        elif rsize >= 11.4:
 61            return "h4"
 62        elif rsize >= 8.5:
 63            return "n"
 64        else:
 65            return "fn"
 66
 67    def _colors(self, color: int) -> str:
 68        if 0xFF <= color <= 0xFF:
 69            return "black"
 70        if 0xFFFFFFFF <= color <= 0xFFFFFFFF:
 71            return "white"
 72        return "unknown"
 73
 74    @cached_property
 75    def _areas(self) -> dict[str, list[Rectangle] | Rectangle]:
 76        content = Rectangle(0.1, 0.1, 0.9, 0.9)
 77        areas = {"content": [content]}
 78        scaled_areas = {}
 79
 80        def _s(r):
 81            return Rectangle(r.left * self.width, r.bottom * self.height, r.right * self.width, r.top * self.height)
 82
 83        for name, area in areas.items():
 84            scaled_areas[name] = [_s(r) for r in area] if isinstance(area, list) else _s(area)
 85        return scaled_areas
 86
 87    def _char_properties(self, line, char):
 88        cp = {
 89            "superscript": False,
 90            "subscript": False,
 91            "bold": any(frag in char.font for frag in {"Bold"}),
 92            "italic": any(frag in char.font for frag in {"Italic", "Oblique"}),
 93            "underline": (char.objlink or char.weblink) is not None,
 94            "size": round(line.height),
 95            "relsize": self._line_size(line),
 96            "char": chr(char.unicode),
 97        }
 98        # Sub- and superscript use a smaller font, characters of the full size
 99        # with a shifted baseline are labels next to multi-line text instead
100        if char.height >= 0.95 * line.height:
101            pass
102        elif line.rotation:
103            if char.origin.x < (line.origin - 0.25 * line.height):
104                cp["superscript"] = True
105            elif char.origin.x > (line.origin + 0.15 * line.height):
106                cp["subscript"] = True
107        elif char.origin.y > (line.origin + 0.25 * line.height):
108            cp["superscript"] = True
109        elif char.origin.y < (line.origin - 0.15 * line.height):
110            cp["subscript"] = True
111        return cp
112
113    def text_in_named_area(self, name: str, check_length: bool = True) -> str | None:
114        """
115        Find all text in the named area.
116
117        :param name: the name of the area(s) to query.
118        :param check_length: assert that the text has a length.
119        :return: the concatenated text of the named area(s) or `None` if area not found.
120        """
121        if name not in self._areas:
122            return None
123        text = ""
124        areas = self._areas[name]
125        if not isinstance(areas, list):
126            areas = [areas]
127        for area in areas:
128            text += self.text_in_area(area)
129        if check_length:
130            assert text
131        return text
132
133    def charlines_in_area(
134        self, area: Rectangle, predicate: Callable[[Character], bool] = None, rtol: float = None
135    ) -> list[CharLine]:
136        """
137        Coalesce the characters in the area and predicate into lines.
138
139        1. Every character in the area is filtered by the `predicate`.
140        2. Character orientation is split into horizontal (left->right) and
141           vertical (bottom->top) character lines sorted by x or y position.
142           Lines containing only whitespace are discarded.
143        3. Overlapping character lines are merged into sub- and superscript
144           using `rtol * max(current_line.height, next_line.height)` as the
145           tolerance for checking if the lines overlap.
146        4. The characters in the merged lines are re-sorted by origin.
147
148        :param area: Area to search for characters.
149        :param predicate: Function to discard characters in the area or include all by default.
150        :param rtol: Relative tolerance to separate lines vertically or use `sc` spacing by default.
151        :return: A list of character lines sorted by x or y position.
152        """
153        if rtol is None:
154            rtol = self._spacing["sc"]
155        # Split all chars into lines based on rounded origin
156        origin_lines_y = defaultdict(list)
157        origin_lines_x = defaultdict(list)
158        for char in self.chars_in_area(area):
159            # Ignore all characters we don't want
160            if predicate is not None and not predicate(char):
161                continue
162            cunicode = self._unicode_filter(char.unicode)
163            if cunicode is None:
164                continue
165            char.unicode = cunicode
166            if char.unicode < 32 and char.unicode not in {0xA}:
167                continue
168            # Ignore characters without width that are not spaces
169            if not char.width and char.unicode not in {0xA, 0xD, 0x20}:
170                _LOGGER.error(f"Unknown char width for {char}: {char.bbox}")
171            # Split up the chars depending on the orientation
172            if 45 < char.rotation <= 135 or 225 < char.rotation <= 315:
173                origin_lines_x[round(char.origin.x, 1)].append(char)
174            elif char.rotation <= 45 or 135 < char.rotation <= 225 or 315 < char.rotation:
175                origin_lines_y[round(char.origin.y, 1)].append(char)
176            else:
177                _LOGGER.error("Unknown char rotation:", char, char.rotation)
178
179        # Convert characters into lines
180        bbox_lines_y = []
181        for chars in origin_lines_y.values():
182            # Remove lines with whitespace only
183            if all(c.unicode in {0xA, 0xD, 0x20} for c in chars):
184                continue
185            origin = statistics.fmean(c.origin.y for c in chars)
186            line = CharLine(
187                self,
188                chars,
189                min(c.bbox.bottom for c in chars),
190                origin,
191                max(c.bbox.top for c in chars),
192                max(c.height for c in chars),
193                sort_origin=self.height - origin,
194            )
195            bbox_lines_y.append(line)
196            # print(line, line.top, line.origin, line.bottom, line.height)
197        bbox_lines = sorted(bbox_lines_y, key=lambda line: line._sort_origin)
198
199        bbox_lines_x = []
200        for chars in origin_lines_x.values():
201            # Remove lines with whitespace only
202            if all(c.unicode in {0xA, 0xD, 0x20} for c in chars):
203                continue
204            line = CharLine(
205                self,
206                chars,
207                min(c.bbox.left for c in chars),
208                statistics.fmean(c.origin.x for c in chars),
209                max(c.bbox.right for c in chars),
210                max(c.width for c in chars),
211                270 if sum(c.rotation for c in chars) <= 135 * len(chars) else 90,
212            )
213            bbox_lines_x.append(line)
214        bbox_lines += sorted(bbox_lines_x, key=lambda line: line._sort_origin)
215
216        if not bbox_lines:
217            return []
218
219        def _stacked(line0, line1) -> bool:
220            # Characters of the same font size, that overlap along the line by
221            # more than half their width, are printed on top of each other.
222            horizontal = not line0.rotation
223
224            def _spans(line):
225                chars = (c for c in line.chars if c.unicode not in {0x20, 0xA, 0xD})
226                chars = (c for c in chars if math.isclose(c.height, line.height, rel_tol=0.01))
227                return [(c.bbox.left, c.bbox.right) if horizontal else (c.bbox.bottom, c.bbox.top) for c in chars]
228
229            spans1 = _spans(line1)
230            for start0, stop0 in _spans(line0):
231                for start1, stop1 in spans1:
232                    overlap = min(stop0, stop1) - max(start0, start1)
233                    if overlap > 0.5 * min(stop0 - start0, stop1 - start1):
234                        return True
235            return False
236
237        def _label(line0, line1) -> bool:
238            # A label of the same font size left of a multi-line text is
239            # vertically centered on the lines, so its baseline is shifted by
240            # more than a continuation of the line would be.
241            if line0.rotation or line1.rotation:
242                return False
243            offset = abs(line0._sort_origin - line1._sort_origin)
244            return offset > rtol * line0.height and (
245                line0.bbox.right < line1.bbox.left or line1.bbox.right < line0.bbox.left
246            )
247
248        # Merge lines that have overlapping bbox_lines
249        merged_lines = []
250        current_line = bbox_lines[0]
251        for next_line in bbox_lines[1:]:
252            height = max(current_line.height, next_line.height)
253            same_height = math.isclose(current_line.height, next_line.height, rel_tol=0.01)
254            # Calculate overlap via normalize origin (increasing with line index)
255            if (
256                # Horizontal and vertical lines have different sort origins
257                bool(current_line.rotation) == bool(next_line.rotation)
258                and (current_line._sort_origin + rtol * height) > (next_line._sort_origin - rtol * height)
259                # Lines of the same font size printed on top of each other are
260                # separate lines, like the numerator and denominator of a formula
261                and not (same_height and _stacked(current_line, next_line))
262                and not (same_height and _label(current_line, next_line))
263            ):
264                # The next line overlaps this one, we merge the shorter line
265                # (typically super- and subscript) into the taller line. Lines
266                # of the same font size continue each other with a slightly
267                # shifted baseline, so we merge them into the longer line.
268                if same_height:
269                    use_current = len(current_line.chars) >= len(next_line.chars)
270                else:
271                    use_current = current_line.height > next_line.height
272                line = current_line if use_current else next_line
273                current_line = CharLine(
274                    self,
275                    current_line.chars + next_line.chars,
276                    line.bottom,
277                    line.origin,
278                    line.top,
279                    height,
280                    line.rotation,
281                    sort_origin=line._sort_origin,
282                )
283            else:
284                # The next line does not overlap the current line
285                merged_lines.append(current_line)
286                current_line = next_line
287        # append last line
288        merged_lines.append(current_line)
289
290        # Prepend labels to the first line of their multi-line text
291        labeled_lines = []
292        for line in merged_lines:
293            prev = labeled_lines[-1] if labeled_lines else None
294            if (
295                prev is not None
296                and not line.rotation
297                and math.isclose(prev.height, line.height, rel_tol=0.01)
298                and 0 < line._sort_origin - prev._sort_origin < line.height
299                and line.bbox.right < prev.bbox.left
300            ):
301                # Keep the spaces between the words of the label, but not around it
302                label = sorted((c for c in line.chars if c.unicode not in {0xA, 0xD}), key=lambda c: c.origin.x)
303                label = list_strip(label, lambda c: c.unicode == 0x20)
304                chars = label + [label[-1].generated_space()] + prev.chars
305                labeled_lines[-1] = CharLine(
306                    self, chars, prev.bottom, prev.origin, prev.top, prev.height, sort_origin=prev._sort_origin
307                )
308            else:
309                labeled_lines.append(line)
310        merged_lines = labeled_lines
311
312        # Sort all lines horizontally based on character origin
313        sorted_lines = []
314        for line in merged_lines:
315            if line.rotation == 90:
316
317                def sort_key(char):
318                    if char.unicode in {0xA, 0xD}:
319                        return char.tbbox.midpoint.y - 1e9
320                    return char.tbbox.midpoint.y
321            elif line.rotation == 270:
322
323                def sort_key(char):
324                    if char.unicode in {0xA, 0xD}:
325                        return -char.tbbox.midpoint.y + 1e9
326                    return -char.tbbox.midpoint.y
327            else:
328
329                def sort_key(char):
330                    if char.unicode in {0xA, 0xD}:
331                        return char.origin.x + 1e9
332                    return char.origin.x
333
334            chars = sorted(line.chars, key=sort_key)
335            if not line.rotation:
336                chars = self._chars_with_bullet_space(chars)
337            chars = self._chars_with_gap_spaces(chars, line.rotation)
338            sorted_lines.append(
339                CharLine(
340                    self,
341                    chars,
342                    line.bottom,
343                    line.origin,
344                    line.top,
345                    line.height,
346                    line.rotation,
347                    area.left,
348                    sort_origin=line._sort_origin,
349                )
350            )
351
352        return sorted_lines
353
354    @staticmethod
355    def _chars_with_gap_spaces(chars: list, rotation: int) -> list:
356        # pdfium generates no spaces at the word gaps of fonts without a space
357        # glyph, so we insert them at gaps wider than the typical gap between
358        # the letters of a run of the same font, which may be letter-spaced.
359        def _span(char):
360            if not rotation:
361                return char.bbox.left, char.bbox.right
362            if rotation == 90:
363                return char.bbox.bottom, char.bbox.top
364            return -char.bbox.top, -char.bbox.bottom
365
366        runs, last, key = [], None, None
367        for index, char in enumerate(chars):
368            if char.unicode in {0x20, 0xA, 0xD}:
369                continue
370            start, stop = _span(char)
371            size = round(char.height, 1)
372            # Lines may interleave the characters of several baselines
373            baseline = round(char.origin.x if rotation else char.origin.y, 1)
374            if key != (char.font, size, baseline) or start - last > 0.5 * size:
375                runs.append((char, []))
376            else:
377                runs[-1][1].append((start - last, index))
378            key, last = (char.font, size, baseline), stop
379
380        spaces = []
381        for first, gaps in runs:
382            if not gaps or first.has_space_glyph:
383                continue
384            size = first.height
385            letter_gap = statistics.median(g for g, _ in gaps)
386            spaces += [i for g, i in gaps if g > letter_gap + 0.15 * size and chars[i - 1].unicode != 0x20]
387        for index in sorted(spaces, reverse=True):
388            chars = chars[:index] + [chars[index].generated_space()] + chars[index:]
389        return chars
390
391    @staticmethod
392    def _chars_with_bullet_space(chars: list) -> list:
393        # Newer pdfium does not generate a space for every visual gap anymore,
394        # but the list detection relies on the one between bullet and text.
395        for ii, (bullet, char) in enumerate(zip(chars, chars[1:]), 1):
396            if bullet.unicode == 0x20:
397                continue
398            if (
399                bullet.char in {"•", "–"}
400                and char.unicode not in {0x20, 0xA, 0xD}
401                and char.origin.x - bullet.bbox.right >= 0.25 * bullet.height
402            ):
403                return chars[:ii] + [char.generated_space()] + chars[ii:]
404            break
405        return chars
406
407    def graphic_bboxes_in_area(
408        self, area: Rectangle, with_graphics: bool = True
409    ) -> list[tuple[Rectangle, Table | Figure | None]]:
410        """
411        Coalesce the graphics in the area into full width bounding boxes.
412
413        1. Group vertically overlapping graphics.
414        2. Widen the overlapped graphics bounding boxes to the edges of the area.
415
416        :param area: area to search for content.
417        :param with_graphics: search for graphics in the area.
418        :return: list of tuples (bounding box, graphic objects or `None`).
419        """
420        if with_graphics:
421            graphics = self.graphics_in_area(area)
422            regions = []
423            # Check if graphics bounding boxes overlap vertically and group them
424            for graphic in sorted(graphics, key=lambda g: (-g.bbox.top, g.bbox.x)):
425                gbbox = graphic.bbox.joined(graphic.cbbox) if graphic.cbbox else graphic.bbox
426                for reg in regions:
427                    if reg.overlaps(gbbox.bottom, gbbox.top):
428                        # They overlap, so merge them
429                        reg.v0 = min(reg.v0, gbbox.bottom)
430                        reg.v1 = max(reg.v1, gbbox.top)
431                        reg.objs.append(graphic)
432                        break
433                else:
434                    regions.append(Region(gbbox.bottom, gbbox.top, graphic))
435
436            # print(regions)
437            # Coalesce all overlapped graphics objects into full width areas
438            areas = []
439            ypos = area.top
440            for reg in regions:
441                if ypos - reg.v1 > self._spacing["y_em"]:
442                    areas.append((Rectangle(area.left, reg.v1, area.right, ypos), None))
443                for obj in reg.objs:
444                    oarea = obj.bbox.joined(obj.cbbox) if obj.cbbox else obj.bbox
445                    areas.append((oarea, obj))
446                ypos = reg.v0
447            areas.append((Rectangle(area.left, area.bottom, area.right, ypos), None))
448        else:
449            areas = [(area, None)]
450        return areas
451
452    def objects_in_area(self, area: Rectangle, with_graphics: bool = True) -> list[CharLine | Table | Figure]:
453        """
454        Find all content objects in this area.
455
456        :param area: area to search for content.
457        :param with_graphics: search for graphics in the area.
458        :return: list of content objects sorted top to bottom.
459        """
460        self._link_characters()
461        areas = self.graphic_bboxes_in_area(area, with_graphics)
462        objects = []
463        for narea, obj in areas:
464            if obj is None:
465                objects += self.charlines_in_area(narea)
466            else:
467                oarea = obj.bbox.joined(obj.cbbox) if obj.cbbox else obj.bbox
468
469                def predicate(c):
470                    return not obj.bbox.contains(c.origin)
471
472                lines = self.charlines_in_area(oarea, predicate)
473                # print(obj, oarea, lines, [line.content for line in lines])
474                objects += list(sorted(lines + [obj], key=lambda o: (-o.bbox.y, o.bbox.x)))
475        return objects
476
477    def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]:
478        """
479        Find all tables and figures in this area.
480
481        :param area: area to search for graphics.
482        :return: list of tables and figures.
483        """
484        return []
485
486    def ast_in_area(self, area: Rectangle, with_graphics: bool = True) -> Node:
487        """
488        Convert the area content into an abstract syntax tree.
489
490        :param area: area to search for content.
491        :param with_graphics: including graphics in the area.
492        :return: An abstract syntax tree including the content formatting.
493        """
494        return Node("area", obj=area, xpos=int(area.left), page=self)
495
496    @property
497    def content_ast(self) -> list[Node]:
498        """The abstract syntax trees in the content area."""
499        ast = []
500        with_graphics = True
501        for area in self._areas["content"]:
502            ast.append(self.ast_in_area(area, with_graphics=with_graphics))
503        # Add a page node to the first leaf to keep track of where a page starts
504        first_leaf = next((n for n in iter(ast[0].descendants) if n.is_leaf), ast[0])
505        Node("page", parent=first_leaf, xpos=first_leaf.xpos, number=self.number)
506        return ast
507
508    @property
509    def content_objects(self) -> list[CharLine | Table | Figure]:
510        """All objects in the content areas."""
511        objs = []
512        for area in self._areas["content"]:
513            objs.extend(self.objects_in_area(area))
514        return objs
515
516    @property
517    def content_graphics(self) -> list[Table | Figure]:
518        """All graphics in the content areas."""
519        objs = []
520        for area in self._areas["content"]:
521            objs.extend(self.graphics_in_area(area))
522        return objs
523
524    @property
525    def content_lines(self) -> list[CharLine]:
526        """All lines in the content areas."""
527        objs = []
528        for area in self._areas["content"]:
529            objs.extend(self.charlines_in_area(area))
530        return objs
531
532    @property
533    def content_tables(self) -> list[Table]:
534        """All tables in the content areas."""
535        return [o for o in self.content_graphics if isinstance(o, Table)]
536
537    @property
538    def content_figures(self) -> list[Figure]:
539        """All figures in the content areas."""
540        return [o for o in self.content_graphics if isinstance(o, Figure)]
541
542    def __repr__(self) -> str:
543        return f"Page({self.number})"