modm_data.pdf2html.stmicro

46class Document(PdfDocument):
47    def __init__(self, path: str):
48        super().__init__(path)
49        self._normalize = _normalize_document
50
51    def page(self, index: int) -> StmPage:
52        assert index < self.page_count
53        return StmPage(self, index)
54
55    def __repr__(self) -> str:
56        return f"STMicroDoc({self.name})"

291class Page(BasePage):
292    def __init__(self, document, index: int):
293        super().__init__(document, index)
294        producer = self.pdf.metadata.get("Producer", "").lower()
295        self._template = "black_white"
296        if "acrobat" in producer or "adobe" in producer:
297            pass
298        elif "antenna" in producer:
299            self._template = "blue_gray"
300        else:
301            _LOGGER.error(f"Unknown page template! Defaulting to Black/White template. '{producer}'")
302
303        if "blue_gray" in self._template:
304            self._areas = _areas_blue_gray(self)
305            self._spacing = _spacing_blue_gray(self)
306            self._colors = _colors_blue_gray
307            self._line_size = _linesize_blue_gray
308        elif "black_white" in self._template:
309            self._areas = _areas_black_white(self)
310            self._spacing = _spacing_black_white(self)
311            self._colors = _colors_black_white
312            self._line_size = _linesize_black_white
313
314    def _unicode_filter(self, code: int) -> int:
315        # Ignore Carriage Return characters and ® (superscript issues)
316        if code in {0xD, ord("®")}:
317            return None
318        # Correct some weird unicode stuffing choices
319        if code in {2}:
320            return ord("-")
321        if code in {61623, 61664}:
322            return ord("•")
323        return code
324
325    @cached_property
326    def identifier(self) -> str:
327        return self.text_in_named_area("id", check_length=False)
328
329    @cached_property
330    def top(self) -> str:
331        if self.index == 0:
332            return "Cover"
333        return self.text_in_named_area("top", check_length=False)
334
335    @cached_property
336    def is_relevant(self) -> bool:
337        if any(c in self.top for c in {"Contents", "List of ", "Index"}):
338            return False
339        return True
340
341    @property
342    def content_ast(self) -> list:
343        ast = []
344        with_graphics = True
345        if "DS" in self.pdf.name:
346            # FIXME: Terrible hack to get the ordering information table fixed
347            # Should be done in the AST as a rewrite similar to bit table rewrite with VirtualTable
348            order_page = next(
349                (
350                    item.page_index
351                    for item in self.pdf.toc
352                    if item.level == 0 and re.search("ordering +information|part +numbering", item.title, re.IGNORECASE)
353                ),
354                -1,
355            )
356            with_graphics = order_page != self.index
357        for area in self._areas["content"]:
358            ast.append(self.ast_in_area(area, with_graphics=with_graphics))
359        # Add a page node to the first leaf to keep track of where a page starts
360        first_leaf = next((n for n in iter(ast[0].descendants) if n.is_leaf), ast[0])
361        Node("page", parent=first_leaf, xpos=first_leaf.xpos, number=self.number)
362        return ast
363
364    def _with_labels(self, area: Rectangle, bbox: Rectangle) -> Rectangle:
365        # Labels of bit diagrams end at their leader lines, so they stick out
366        # of the drawn graphics and would otherwise be glued into the text.
367        # Labels that overlap the graphics are part of the figure too.
368        x_em = self._spacing["x_em"]
369        # Captions are never labels, even where their subscripts reach into the graphics
370        captions = {
371            id(c)
372            for line in self.charlines_in_area(area)
373            if re.match(r"\s*(Figure|Table) \d+\.", "".join(c.char for c in line.chars))
374            for c in line.chars
375        }
376        chars = [c for c in self.chars_in_area(area) if c.unicode not in {0x20, 0xA, 0xD} and id(c) not in captions]
377        graphics = bbox
378        while True:
379            grown = bbox
380            for c in chars:
381                # Labels can also stick out at the top or bottom of the graphics:
382                # their baseline or the top of their letters is inside of them.
383                # Captions just above the graphics only touch them with their
384                # descenders. Only the graphics count vertically, otherwise a
385                # label at the top would pull in the caption above it.
386                inside = graphics.bottom + 0.5 < c.origin.y < graphics.top - 0.5 or (
387                    graphics.bottom + 0.3 * (c.bbox.top - c.bbox.bottom) < c.bbox.top < graphics.top
388                )
389                if inside and bbox.left - x_em <= c.bbox.right and c.bbox.left <= bbox.right + x_em:
390                    grown = grown.joined(c.bbox)
391            if grown.isclose(bbox):
392                return bbox
393            bbox = grown
394
395    def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]:
396        # Find all graphic clusters in this area
397        em = self._spacing["y_em"]
398        large_area = area.offset_x(em / 2)
399        graphic_clusters = self.graphic_clusters(lambda p: large_area.contains(p.bbox), em / 2)
400        # for bbox, paths in raw_graphic_clusters:
401        #     # Some docs have large DRAFT chars in the background
402        #     if any(path.fill == 0xe6e6e6ff and path.stroke == 0xff for path in paths):
403        #         continue
404        #     graphic_clusters.append((bbox, paths))
405
406        # Find the captions and group them by y origin to catch side-by-side figures
407        ycaptions = defaultdict(list)
408        for line in self.charlines_in_area(area, lambda c: "Bold" in c.font):
409            for cluster in line.clusters():
410                for phrase in [r"Figure \d+\.", r"Table \d+\."]:
411                    if re.match(phrase, cluster.content):
412                        ycaptions[int(round(cluster.bbox.y / em))].append((phrase, cluster.chars))
413        ycaptions = [ycaptions[k] for k in sorted(ycaptions.keys(), key=lambda y: -y)]
414
415        # Now associate these captions with the graphics bboxes
416        categories = []
417        for captions in ycaptions:
418            width = area.width / len(captions)
419            for ii, (phrase, chars) in enumerate(sorted(captions, key=lambda c: c[1][0].origin.x)):
420                left, right = area.left + ii * width, area.left + (ii + 1) * width
421                bottom, top, height = chars[0].bbox.bottom, chars[0].bbox.top, chars[0].height
422
423                # Find the graphic associated with this caption
424                graphic = next(
425                    ((b, p) for b, p in graphic_clusters if b.bottom <= bottom and left <= b.left and b.right <= right),
426                    None,
427                )
428                if graphic is None:
429                    _LOGGER.error(f"Graphic cluster not found for caption {''.join(c.char for c in chars)}")
430                    continue
431
432                if self._template == "blue_gray":
433                    # Search for all lines of the current caption with the same properties
434                    cbbox = Rectangle(left, bottom, right, top)
435                    cchars = self.chars_in_area(cbbox)
436                    while True:
437                        nbbox = Rectangle(left, max(graphic[0].top, cbbox.bottom - height), right, top)
438                        nchars = self.chars_in_area(nbbox)
439                        if len(cchars) >= len(nchars):
440                            break
441                        cbbox = nbbox
442                        cchars = nchars
443                else:
444                    cbbox = Rectangle(left, min(graphic[0].top, bottom), right, top)
445
446                otype = phrase.split(" ")[0].lower()
447                if "Figure" in phrase:
448                    # Find all other graphics in the bounding box
449                    gbbox = Rectangle(left, graphic[0].bottom, right, cbbox.bottom)
450                    graphics = []
451                    for b, p in graphic_clusters:
452                        if gbbox.overlaps(b):
453                            graphics.append((b, p))
454                    for g in graphics:
455                        graphic_clusters.remove(g)
456                    gbbox = [cluster[0] for cluster in graphics]
457                    gbbox = reduce(lambda r0, r1: r0.joined(r1), gbbox)
458                    paths = [p for cluster in graphics for p in cluster[1]]
459
460                    if self._template == "blue_gray":
461                        # Search for characters below the graphics bbox, max 1 y_em
462                        gbbox = Rectangle(left, gbbox.bottom, right, gbbox.bottom)
463                        while True:
464                            gbbox = Rectangle(left, gbbox.bottom - self._spacing["y_em"], right, gbbox.bottom)
465                            if not self.chars_in_area(gbbox):
466                                break
467                    # Generate the new bounding box which includes the caption
468                    gbbox = self._with_labels(area, Rectangle(left, gbbox.bottom, right, cbbox.bottom))
469                elif "Table" in phrase:
470                    graphic_clusters.remove(graphic)
471                    gbbox, paths = graphic
472                    if (
473                        self._template == "black_white"
474                        and sum(1 for path in paths if path.count == 2) >= len(paths) / 2
475                    ):
476                        otype += "_lines"
477                categories.append((otype, cbbox, gbbox, paths))
478
479        # Deal with the remaining graphic categories
480        for gbbox, paths in graphic_clusters:
481            if gbbox.width < self._spacing["x_em"] or gbbox.height < self._spacing["y_em"]:
482                continue
483            category = ""
484            if any(isinstance(p, Image) for p in paths):
485                category = "figure"
486            elif self._template == "blue_gray":
487                if all(self._colors(path.stroke) == "gray" or self._colors(path.fill) == "darkblue" for path in paths):
488                    category = "table"
489                else:
490                    category = "figure"
491            elif self._template == "black_white":
492                # Some tables are rendered explicitly with filled rectangular
493                # shapes with others are implicitly rendered with stroked lines
494                stroked_table_lines = sum(1 for path in paths if path.count == 2) >= len(paths) / 2
495                is_table = stroked_table_lines or all(
496                    [any(p.isclose(pp) for pp in path.bbox.points) for p in path.points].count(True)
497                    >= len(path.points) * 2 / 3
498                    for path in paths
499                )
500                if len(paths) > 1 and is_table:
501                    category = "table"
502                    if stroked_table_lines:
503                        category += "_lines"
504                else:
505                    category = "figure"
506
507            if "table" in category:
508                # Check if there are only numbers on top of the table
509                cbbox = Rectangle(gbbox.left, gbbox.top, gbbox.right, gbbox.top + self._spacing["y_em"])
510                nchars = [c for c in self.chars_in_area(cbbox) if c.unicode not in {0x20, 0xA, 0xD}]
511
512                if nchars and sum(1 if c.char.isnumeric() else 0 for c in nchars) >= len(nchars) / 3:
513                    # This is a register table with invisible top borders!
514                    cbbox = Rectangle(gbbox.left, gbbox.top, gbbox.right, max(c.bbox.top for c in nchars))
515                    gbbox = Rectangle(gbbox.left, gbbox.bottom, gbbox.right, cbbox.top)
516                    name = "register_" + category
517                else:
518                    cbbox = None
519                    name = category
520                categories.append((name, cbbox, gbbox, paths))
521            else:
522                categories.append(("figure", None, self._with_labels(area, gbbox), paths))
523
524        # Convert the objects into specialized classes
525        categories.sort(key=lambda o: (-o[2].y, o[2].x))
526        objects = []
527        for otype, caption_bbox, graphics_bbox, graphics_paths in categories:
528            if "figure" in otype:
529                figure = Figure(self, graphics_bbox, caption_bbox, graphics_paths)
530                objects.append(figure)
531            elif "table" in otype:
532                xlines, ylines, yhlines = [], [], []
533                for path in graphics_paths:
534                    if self._template == "blue_gray" or "_lines" in otype:
535                        if self._colors(path.stroke) == "gray" or "_lines" in otype:
536                            # Intercell paths in gray
537                            if len(path.lines) == 1:
538                                line = path.lines[0]
539                                if line.direction == line.Direction.VERTICAL:
540                                    xlines.append(line.specialize())
541                                elif line.direction == line.Direction.HORIZONTAL:
542                                    ylines.append(line.specialize())
543                                else:
544                                    _LOGGER.warn(f"Line not vertical or horizontal: {line}")
545                            else:
546                                _LOGGER.warn(f"Path too long: {path}")
547                        elif self._colors(path.fill) == "darkblue":
548                            # Add the bottom line of the dark blue header box as a very thick line
549                            line = HLine(path.bbox.bottom, path.bbox.left, path.bbox.right, 5)
550                            yhlines.append(line)
551
552                    elif self._template == "black_white":
553                        bbox = path.bbox
554                        is_vertical = bbox.width < bbox.height
555                        width = bbox.width if is_vertical else bbox.height
556                        length = bbox.height if is_vertical else bbox.width
557                        if width <= self._spacing["x_em"] / 2:
558                            if length >= self._spacing["y_em"] / 2:
559                                if is_vertical:
560                                    line = VLine(bbox.midpoint.x, bbox.bottom, bbox.top, bbox.width)
561                                    xlines.append(line)
562                                else:
563                                    line = HLine(bbox.midpoint.y, bbox.left, bbox.right, bbox.height)
564                                    ylines.append(line)
565                        else:
566                            # Split the rectangle into it's outline
567                            xlines.append(VLine(bbox.left, bbox.bottom, bbox.top, 0.1))
568                            xlines.append(VLine(bbox.right, bbox.bottom, bbox.top, 0.1))
569                            ylines.append(HLine(bbox.bottom, bbox.left, bbox.right, 0.1))
570                            ylines.append(HLine(bbox.top, bbox.left, bbox.right, 0.1))
571                if yhlines:
572                    yhlines.sort(key=lambda line: line.p0.y)
573                    ylines.append(yhlines[0])
574                if not xlines or not ylines:
575                    continue
576                table = Table(self, graphics_bbox, xlines, ylines, caption_bbox, is_register="register" in otype)
577                objects.append(table)
578
579        return objects
580
581    def ast_in_area(
582        self,
583        area: Rectangle,
584        with_graphics: bool = True,
585        ignore_xpos: bool = False,
586        with_bits: bool = True,
587        with_notes: bool = True,
588    ) -> Node:
589        x_em = self._spacing["x_em"]
590        spacing_content = self._spacing["x_content"]
591        lh_factor = self._spacing["lh"]
592        # spacing_y = self._spacing["y_em"]
593        root = Node("area", obj=area, xpos=int(area.left), page=self)
594
595        def unindent(_xpos, _current, _newlines=1):
596            current = _current
597            # Check if we need to unindent the current node
598            while (_xpos - current.xpos) < -x_em and current.parent is not None and not ignore_xpos:
599                current = current.parent
600            if _newlines >= 2 and current.name == "para":
601                current = current.parent
602            return current
603
604        def parent_name(current):
605            return "" if current.parent is None else current.parent.name
606
607        current = root
608        ypos = area.top
609        for obj in self.objects_in_area(area, with_graphics):
610            xpos = round(obj.bbox.left)
611
612            # Tables should remain in their current hierarchy regardless of indentation
613            if isinstance(obj, (Table, Figure)):
614                current = next((c for c in current.iter_path_reverse() if c.name.startswith("head")), root)
615                name = "figure" if isinstance(obj, Figure) else "table"
616                Node(
617                    name,
618                    parent=current,
619                    obj=obj,
620                    xpos=xpos,
621                    number=-1,
622                    _width=obj.bbox.width / area.width,
623                    _type=obj._type,
624                )
625                ypos = obj.bbox.bottom
626
627            # Lines of text need to be carefully checked for indentation
628            elif isinstance(obj, CharLine):
629                newlines = round((ypos - obj.origin) / (lh_factor * obj.height))
630                content = obj.content
631                lcontent = content.lstrip()
632                content_start = 0
633                linesize = self._line_size(obj)
634
635                # Check when the note has finished (=> paragraphs without italic)
636                if parent_name(current) == "note" and (
637                    (current.parent.type == "note" and not obj.contains_font(current.parent._font))
638                    or (current.parent.type in {"caution", "warning"} and newlines >= 2)
639                ):
640                    current = current.parent.parent
641
642                # Check when the list ends into something indented far too right
643                elif parent_name(current).startswith("list") and (xpos - current.xpos) >= 2 * x_em:
644                    current = current.parent.parent
645
646                # print(obj.fonts, ypos, xpos, current.xpos, f"{obj.height:.2f}", content)
647
648                # Check if line is a heading, which may be multi-line, so we must
649                # be careful not to nest them, but group them properly
650                # Headings are always inserted into the root note!
651                if linesize.startswith("h1") or (
652                    linesize.startswith("h") and xpos < (spacing_content + 2 * x_em) and "Bold" in obj.chars[0].font
653                ):
654                    if (match := re.match(r"^ *(\d+(\.\d+)?(\.\d+)?) *", content)) is not None:
655                        start = min(len(match.group(0)), len(obj.chars) - 1)
656                        marker = match.group(1)
657                        size = marker.count(".") + 2
658                    else:
659                        start = 0
660                        marker = None
661                        size = linesize[1]
662                    name = f"head{size}"
663                    # Check if we're already parsing a heading, do not split into two
664                    if parent_name(current) != name or newlines > 2:
665                        content_start = start
666                        xpos = round(obj.chars[content_start].bbox.left)
667                        current = Node(name, parent=root, obj=obj, xpos=xpos, size=size, marker=marker)
668                        current = Node("para", parent=current, obj=obj, xpos=current.xpos)
669
670                # Check if the line is a note and deal with the indentation correctly
671                elif (
672                    with_notes and (match := re.match(r" *([Nn]ote|[Cc]aution|[Ww]arning):? \d?", content)) is not None
673                ):
674                    content_start = min(len(match.group(0)), len(obj.chars) - 1)
675                    # print(obj.fonts)
676                    # Correct xposition only if the Note: string is very far left
677                    if xpos + 4 * x_em <= current.xpos:
678                        xpos = round(obj.chars[content_start].bbox.left)
679                    # Prevent nesting of notes, they should only be listed
680                    if parent_name(current) == "note":
681                        current = current.parent.parent
682                    current = unindent(xpos, current, 2)
683                    current = Node(
684                        "note",
685                        parent=current,
686                        obj=obj,
687                        xpos=xpos,
688                        type=match.group(1).lower(),
689                        _font=obj.chars[content_start].font,
690                    )
691                    current = Node("para", parent=current, obj=obj, xpos=current.xpos)
692
693                # Check if line is Table or Figure caption
694                elif with_graphics and (
695                    (match := re.match(r" *([Tt]able|[Ff]igure) ?(\d+)\.? ?", content)) is not None
696                    and "Bold" in obj.chars[0].font
697                ):
698                    content_start = min(len(match.group(0)), len(obj.chars) - 1)
699                    current = next((c for c in current.iter_path_reverse() if c.name.startswith("head")), root)
700                    current = Node(
701                        "caption",
702                        parent=current,
703                        obj=obj,
704                        xpos=xpos,
705                        _type=match.group(1).lower(),
706                        number=int(match.group(2)),
707                    )
708                    current = Node("para", parent=current, obj=obj, xpos=current.xpos)
709
710                # Check if line is list and group them according to indentation
711                elif (match := re.match(r"^ *([•–]) ..|^ *(\d+)\. ..|^ *([a-z])\) ?..", content)) is not None:
712                    current = unindent(xpos, current, newlines)
713                    content_start = len(match.group(0)) - 2
714                    xpos = round(obj.chars[content_start].bbox.left)
715                    name = "listb"
716                    value = lcontent[0]
717                    if value in {"–", "-"}:
718                        name = "lists"
719                    elif value.isalpha():
720                        name = "lista"
721                    elif value.isnumeric():
722                        name = "listn"
723                        value = int(match.group(2))
724                    current = Node(name, parent=current, obj=obj, xpos=xpos, value=value)
725                    current = Node("para", parent=current, obj=obj, xpos=current.xpos)
726
727                # Check if line is a register bit definition
728                elif (
729                    with_bits
730                    and re.match(r" *([Bb]ytes? *.+? *)?B[uio]ts? *\d+", content) is not None
731                    # A sentence starting with "Bits" inside the description column is not a new bit
732                    and not any(c.name == "bit" and xpos >= c._middle - x_em for c in current.iter_path_reverse())
733                ):
734                    if obj.contains_font("Bold"):
735                        # Use the bold character as delimiter
736                        content_start = next(xi for xi, c in enumerate(obj.chars) if "Bold" in c.font)
737                    else:
738                        # Default back to the regex
739                        if "Reserved" not in content:
740                            _LOGGER.warning(
741                                f"Fallback to Regex length for Bit pattern '{content}'!\nFonts: {obj.fonts}"
742                            )
743                        content_start = re.match(
744                            r" *([Bb]ytes? *.+? *)?(B[uio]t)( *\d+:?|s *(\d+ *([:-] *\d+ *)? *,? *)+) *", content
745                        )
746                        if content_start is None:
747                            _LOGGER.error(f"Unable to match Bit regex at all! '{content}'!")
748                            content_start = 0
749                        else:
750                            content_start = len(content_start.group(0))
751                        if not content_start:
752                            _LOGGER.error(f"Missing content start (=0)! '{content}'!")
753                        content_start = min(content_start, len(obj.chars) - 1)
754
755                    current = next((c for c in current.iter_path_reverse() if c.name.startswith("head")), root)
756                    middle = obj.chars[content_start].bbox.left
757                    xpos = round(middle)
758                    current = Node(
759                        "bit",
760                        parent=current,
761                        obj=obj,
762                        xpos=xpos,
763                        _page=self,
764                        _middle=middle,
765                        _left=area.left,
766                        _right=area.right,
767                    )
768                    current = Node("para", parent=current, obj=obj, xpos=current.xpos)
769
770                # Check if this is a new paragraph
771                elif newlines >= 2 or current.name not in {"para"}:
772                    # Fix issues where notes are reflowing back left of Note: text
773                    if parent_name(current) in {"note"}:
774                        if xpos < current.parent.xpos:
775                            xpos = current.parent.xpos
776                    # Prevent multiline
777                    current = unindent(xpos, current, newlines)
778                    current = Node("para", parent=current, obj=obj, xpos=xpos if current.is_root else current.xpos)
779
780                elif parent_name(current) not in {"caption", "bit", "area"}:
781                    current = unindent(xpos, current, newlines)
782                    # A line after a list unindents out of the list, but only a paragraph is formatted
783                    if current.name != "para":
784                        current = Node("para", parent=current, obj=obj, xpos=xpos if current.is_root else current.xpos)
785
786                # Add the actual line
787                Node("line", parent=current, obj=obj, xpos=xpos, start=content_start, str=content[content_start:50])
788
789                ypos = obj.origin
790
791        return root
792
793    def __repr__(self) -> str:
794        return f"StmPage({self.number})"