Fix context duplication in pdf output

copienator/commands/reading_grouped_annotations.py

@@ -2,12 +2,15 @@ from __future__ import annotations
 
 import argparse
 import concurrent.futures
+import unicodedata
+from hashlib import sha256
 from collections import defaultdict
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
 from PIL import Image, ImageDraw
+import pymupdf
 
 from copienator import (
     EvaluationWorkspace,
@@ -44,24 +47,51 @@ ScanResult = tuple[dict[str, list[dict[str, Any]]], dict[str, LabelNotes]]
 SCAN_WORKERS = 2
 
 
+def context_content_key(path: str | Path) -> tuple[str, str]:
+    """Identify context by its text, independently of filename and PDF metadata."""
+    with pymupdf.open(path) as document:
+        text = "\n".join(page.get_text(sort=True) for page in document)
+    text = " ".join(unicodedata.normalize("NFC", text).split())
+    if text:
+        return "text", text
+    # Image-only contexts must not all collapse to the same empty-text key.
+    return "file", sha256(Path(path).read_bytes()).hexdigest()
+
+
 def get_extra_pdfs_as_images(
     root_dir: str | Path,
     label: str,
     annotating_module: Any,
     all_labels: list[str],
+    *,
+    seen_contexts: set[tuple[str, str]] | None = None,
 ) -> list[Image.Image]:
-    """Convert the context, question and solution PDFs associated with a label."""
+    """Render supplements with semantic borders and optional per-copy deduplication."""
     paths = [
-        *utils.pdf_images_of_contexts(root_dir, label, all_labels),
+        *(("context", path) for path in utils.pdf_images_of_contexts(root_dir, label, all_labels)),
-        utils.pdf_image_of_enonce(root_dir, label),
+        ("question", utils.pdf_image_of_enonce(root_dir, label)),
-        utils.pdf_image_of_solution(root_dir, label),
+        ("solution", utils.pdf_image_of_solution(root_dir, label)),
     ]
     images = []
-    for path in paths:
+    for role, path in paths:
         if path:
+            key = None
+            if role == "context" and seen_contexts is not None:
+                key = context_content_key(path)
+                if key in seen_contexts:
+                    continue
             image, _, _ = annotating_module.make_base_image(path)
             if image is not None:
+                if role in {"question", "solution"}:
+                    image = image.copy()
+                    ImageDraw.Draw(image).rectangle(
+                        [0, 0, image.width - 1, image.height - 1],
+                        outline="black" if role == "question" else "blue",
+                        width=int((0.2 / 2.54) * 100),
+                    )
                 images.append(image)
+                if key is not None:
+                    seen_contexts.add(key)
     return images
 
 
@@ -74,7 +104,6 @@ def save_paginated_pdf(
         return
     max_width = max(image.width for group in non_empty for image in group)
     max_page_height = int(max_width * 1.414 * 1.25)
-    border = int((0.2 / 2.54) * 100)
     left_margin = int((0.3 / 2.54) * 100)
     vertical_margin = int((0.2 / 2.54) * 100)
     max_content_height = max_page_height - 2 * vertical_margin
@@ -101,21 +130,10 @@ def save_paginated_pdf(
         page_height = 0
 
     for group in non_empty:
-        processed: list[Image.Image] = []
+        group_height = sum(image.height for image in group)
-        for index, image in enumerate(group):
-            if index in (0, 1):
-                image = image.copy()
-                color = "black" if index == 0 else "blue"
-                ImageDraw.Draw(image).rectangle(
-                    [0, 0, image.width - 1, image.height - 1],
-                    outline=color,
-                    width=border,
-                )
-            processed.append(image)
-        group_height = sum(image.height for image in processed)
         if page_images and page_height + group_height > max_content_height:
             finish_page()
-        page_images.extend(processed)
+        page_images.extend(group)
         page_height += group_height
     finish_page()
     pages[0].save(
@@ -259,6 +277,7 @@ def apply_actions_and_regenerate_grouped(
     previous_baseline = baseline(output_dir) if (output_dir / BASELINE).is_file() else None
     concat_images: list[Image.Image] = []
     filtered_groups: list[list[Image.Image]] = []
+    seen_contexts: set[tuple[str, str]] = set()
     incomplete = False
 
     for label, content in sorted(
@@ -306,7 +325,7 @@ def apply_actions_and_regenerate_grouped(
             # Keep previously reviewed content, including handwriting.
             if annotation_dir == "BGnot":
                 extras = get_extra_pdfs_as_images(
-                    workspace.root, label, annotating, all_labels
+                    workspace.root, label, annotating, all_labels, seen_contexts=seen_contexts
                 )
                 filtered_groups.append([*extras, final_image])
                 touched[label] = True
@@ -359,7 +378,8 @@ def apply_actions_and_regenerate_grouped(
         if not perfect or has_notes:
             filtered_labels.append(label)
             extras = (
-                get_extra_pdfs_as_images(workspace.root, label, annotating, all_labels)
+                get_extra_pdfs_as_images(workspace.root, label, annotating, all_labels,
+                                        seen_contexts=seen_contexts)
                 if annotation_dir == "BGnot"
                 else []
             )

copienator/utils.py

@@ -70,7 +70,7 @@ def pdf_images_of_contexts(root_dir, label, all_labels):
 
     pertinent_contexts = []
 
-    for filename in os.listdir(text2_dir):
+    for filename in sorted(os.listdir(text2_dir)):
         if filename.startswith("CTXT ") and filename.endswith(".pdf"):
             # Extract "first_label -> last_label" from "CTXT first_label -> last_label.pdf"
             core = filename[5:-4]

copienator_gui/workflow.py

@@ -795,15 +795,6 @@ def build_workflow(show_personal_steps: bool) -> list[StepDefinition]:
             (manual("deploy"),),
             personal=True,
         ),
-        StepDefinition(
-            "personal_print",
-            "Étapes personnelles",
-            "Imprimer une copie",
-            "Étape manuelle via le lecteur PDF.",
-            (manual("print"),),
-            personal=True,
-            optional=True,
-        ),
         StepDefinition(
             "clean",
             "Archivage",

docs/final_output.md

@@ -177,17 +177,22 @@ For each retained answer, the PDF stacks the following available material:
 3. The model solution from `Sol2/<label>.pdf`.
 4. The same compiled answer block used for the JPEG.
 
-Missing supplementary PDFs are skipped. Contexts can repeat for successive
+Missing supplementary PDFs are skipped. Each distinct context appears only once
-questions. Each question's complete group stays together on one page; groups
+per student PDF, before its first retained question. Duplicate detection uses
+extracted PDF text with Unicode and whitespace normalization, including contexts
+stored under different filenames or validity ranges. Numbers, mathematical
+symbols and case remain significant. Contexts without extractable text use a
+PDF-content hash instead, so unrelated image-only contexts are not collapsed.
+The same rule applies to selective redo and manual annotation modifications.
+Each question's complete group stays together on one page; groups
 are packed until the next would exceed the target height. An oversized group
 gets its own taller page, rather than being split. Pages are raster images saved
 as PDF at 100 dpi, with variable heights, not fixed A4 sheets or searchable text.
 
 The target height is `int(max_image_width * 1.414 * 1.25)` pixels. Small white
-margins are added on the left and above/below the page. The first image of each
+margins are added on the left and above/below the page. The question receives
-group receives a black border and the second a blue border. These borders are
+a black border and the model solution a blue border. Context and student answer
-assigned by position, so they do not consistently identify question and solution
+blocks receive neither border, including when supplementary files are missing.
-when contexts are present or supplementary files are missing.
 
 If nothing survives filtering, `read-grouped` removes old `Concat_F` outputs and
 does not create a new PDF. During a selective `--refaire` merge, saved answer