Fix context duplication in pdf output

copienator/commands/reading_grouped_annotations.py

@@ -2,12 +2,15 @@ from __future__ import annotations
 
 import argparse
 import concurrent.futures
+import unicodedata
+from hashlib import sha256
 from collections import defaultdict
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
 from PIL import Image, ImageDraw
+import pymupdf
 
 from copienator import (
     EvaluationWorkspace,
@@ -44,24 +47,51 @@ ScanResult = tuple[dict[str, list[dict[str, Any]]], dict[str, LabelNotes]]
 SCAN_WORKERS = 2
 
 
+def context_content_key(path: str | Path) -> tuple[str, str]:
+    """Identify context by its text, independently of filename and PDF metadata."""
+    with pymupdf.open(path) as document:
+        text = "\n".join(page.get_text(sort=True) for page in document)
+    text = " ".join(unicodedata.normalize("NFC", text).split())
+    if text:
+        return "text", text
+    # Image-only contexts must not all collapse to the same empty-text key.
+    return "file", sha256(Path(path).read_bytes()).hexdigest()
+
+
 def get_extra_pdfs_as_images(
     root_dir: str | Path,
     label: str,
     annotating_module: Any,
     all_labels: list[str],
+    *,
+    seen_contexts: set[tuple[str, str]] | None = None,
 ) -> list[Image.Image]:
-    """Convert the context, question and solution PDFs associated with a label."""
+    """Render supplements with semantic borders and optional per-copy deduplication."""
     paths = [
-        *utils.pdf_images_of_contexts(root_dir, label, all_labels),
-        utils.pdf_image_of_enonce(root_dir, label),
-        utils.pdf_image_of_solution(root_dir, label),
+        *(("context", path) for path in utils.pdf_images_of_contexts(root_dir, label, all_labels)),
+        ("question", utils.pdf_image_of_enonce(root_dir, label)),
+        ("solution", utils.pdf_image_of_solution(root_dir, label)),
     ]
     images = []
-    for path in paths:
+    for role, path in paths:
         if path:
+            key = None
+            if role == "context" and seen_contexts is not None:
+                key = context_content_key(path)
+                if key in seen_contexts:
+                    continue
             image, _, _ = annotating_module.make_base_image(path)
             if image is not None:
+                if role in {"question", "solution"}:
+                    image = image.copy()
+                    ImageDraw.Draw(image).rectangle(
+                        [0, 0, image.width - 1, image.height - 1],
+                        outline="black" if role == "question" else "blue",
+                        width=int((0.2 / 2.54) * 100),
+                    )
                 images.append(image)
+                if key is not None:
+                    seen_contexts.add(key)
     return images
 
 
@@ -74,7 +104,6 @@ def save_paginated_pdf(
         return
     max_width = max(image.width for group in non_empty for image in group)
     max_page_height = int(max_width * 1.414 * 1.25)
-    border = int((0.2 / 2.54) * 100)
     left_margin = int((0.3 / 2.54) * 100)
     vertical_margin = int((0.2 / 2.54) * 100)
     max_content_height = max_page_height - 2 * vertical_margin
@@ -101,21 +130,10 @@ def save_paginated_pdf(
         page_height = 0
 
     for group in non_empty:
-        processed: list[Image.Image] = []
-        for index, image in enumerate(group):
-            if index in (0, 1):
-                image = image.copy()
-                color = "black" if index == 0 else "blue"
-                ImageDraw.Draw(image).rectangle(
-                    [0, 0, image.width - 1, image.height - 1],
-                    outline=color,
-                    width=border,
-                )
-            processed.append(image)
-        group_height = sum(image.height for image in processed)
+        group_height = sum(image.height for image in group)
         if page_images and page_height + group_height > max_content_height:
             finish_page()
-        page_images.extend(processed)
+        page_images.extend(group)
         page_height += group_height
     finish_page()
     pages[0].save(
@@ -259,6 +277,7 @@ def apply_actions_and_regenerate_grouped(
     previous_baseline = baseline(output_dir) if (output_dir / BASELINE).is_file() else None
     concat_images: list[Image.Image] = []
     filtered_groups: list[list[Image.Image]] = []
+    seen_contexts: set[tuple[str, str]] = set()
     incomplete = False
 
     for label, content in sorted(
@@ -306,7 +325,7 @@ def apply_actions_and_regenerate_grouped(
             # Keep previously reviewed content, including handwriting.
             if annotation_dir == "BGnot":
                 extras = get_extra_pdfs_as_images(
-                    workspace.root, label, annotating, all_labels
+                    workspace.root, label, annotating, all_labels, seen_contexts=seen_contexts
                 )
                 filtered_groups.append([*extras, final_image])
                 touched[label] = True
@@ -359,7 +378,8 @@ def apply_actions_and_regenerate_grouped(
         if not perfect or has_notes:
             filtered_labels.append(label)
             extras = (
-                get_extra_pdfs_as_images(workspace.root, label, annotating, all_labels)
+                get_extra_pdfs_as_images(workspace.root, label, annotating, all_labels,
+                                        seen_contexts=seen_contexts)
                 if annotation_dir == "BGnot"
                 else []
             )

copienator/utils.py

@@ -70,7 +70,7 @@ def pdf_images_of_contexts(root_dir, label, all_labels):
 
     pertinent_contexts = []
 
-    for filename in os.listdir(text2_dir):
+    for filename in sorted(os.listdir(text2_dir)):
         if filename.startswith("CTXT ") and filename.endswith(".pdf"):
             # Extract "first_label -> last_label" from "CTXT first_label -> last_label.pdf"
             core = filename[5:-4]