From 3754c0c11a479141304c44153ee66e6ef3b9d242 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Miquel?= <sebastien.miquel@posteo.eu>
Date: Sun, 11 Oct 2026 20:02:11 +0200
Subject: [PATCH] Fix context duplication in pdf output

---
 .../commands/reading_grouped_annotations.py   | 62 ++++++++++++-------
 copienator/utils.py                           |  2 +-
 copienator_gui/workflow.py                    |  9 ---
 docs/final_output.md                          | 17 +++--
 4 files changed, 53 insertions(+), 37 deletions(-)

diff --git a/copienator/commands/reading_grouped_annotations.py b/copienator/commands/reading_grouped_annotations.py
index c5eb5bf..947f3b0 100644
--- a/copienator/commands/reading_grouped_annotations.py
+++ b/copienator/commands/reading_grouped_annotations.py
@@ -2,12 +2,15 @@ from __future__ import annotations
 
 import argparse
 import concurrent.futures
+import unicodedata
+from hashlib import sha256
 from collections import defaultdict
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
 from PIL import Image, ImageDraw
+import pymupdf
 
 from copienator import (
     EvaluationWorkspace,
@@ -44,24 +47,51 @@ ScanResult = tuple[dict[str, list[dict[str, Any]]], dict[str, LabelNotes]]
 SCAN_WORKERS = 2
 
 
+def context_content_key(path: str | Path) -> tuple[str, str]:
+    """Identify context by its text, independently of filename and PDF metadata."""
+    with pymupdf.open(path) as document:
+        text = "\n".join(page.get_text(sort=True) for page in document)
+    text = " ".join(unicodedata.normalize("NFC", text).split())
+    if text:
+        return "text", text
+    # Image-only contexts must not all collapse to the same empty-text key.
+    return "file", sha256(Path(path).read_bytes()).hexdigest()
+
+
 def get_extra_pdfs_as_images(
     root_dir: str | Path,
     label: str,
     annotating_module: Any,
     all_labels: list[str],
+    *,
+    seen_contexts: set[tuple[str, str]] | None = None,
 ) -> list[Image.Image]:
-    """Convert the context, question and solution PDFs associated with a label."""
+    """Render supplements with semantic borders and optional per-copy deduplication."""
     paths = [
-        *utils.pdf_images_of_contexts(root_dir, label, all_labels),
-        utils.pdf_image_of_enonce(root_dir, label),
-        utils.pdf_image_of_solution(root_dir, label),
+        *(("context", path) for path in utils.pdf_images_of_contexts(root_dir, label, all_labels)),
+        ("question", utils.pdf_image_of_enonce(root_dir, label)),
+        ("solution", utils.pdf_image_of_solution(root_dir, label)),
     ]
     images = []
-    for path in paths:
+    for role, path in paths:
         if path:
+            key = None
+            if role == "context" and seen_contexts is not None:
+                key = context_content_key(path)
+                if key in seen_contexts:
+                    continue
             image, _, _ = annotating_module.make_base_image(path)
             if image is not None:
+                if role in {"question", "solution"}:
+                    image = image.copy()
+                    ImageDraw.Draw(image).rectangle(
+                        [0, 0, image.width - 1, image.height - 1],
+                        outline="black" if role == "question" else "blue",
+                        width=int((0.2 / 2.54) * 100),
+                    )
                 images.append(image)
+                if key is not None:
+                    seen_contexts.add(key)
     return images
 
 
@@ -74,7 +104,6 @@ def save_paginated_pdf(
         return
     max_width = max(image.width for group in non_empty for image in group)
     max_page_height = int(max_width * 1.414 * 1.25)
-    border = int((0.2 / 2.54) * 100)
     left_margin = int((0.3 / 2.54) * 100)
     vertical_margin = int((0.2 / 2.54) * 100)
     max_content_height = max_page_height - 2 * vertical_margin
@@ -101,21 +130,10 @@ def save_paginated_pdf(
         page_height = 0
 
     for group in non_empty:
-        processed: list[Image.Image] = []
-        for index, image in enumerate(group):
-            if index in (0, 1):
-                image = image.copy()
-                color = "black" if index == 0 else "blue"
-                ImageDraw.Draw(image).rectangle(
-                    [0, 0, image.width - 1, image.height - 1],
-                    outline=color,
-                    width=border,
-                )
-            processed.append(image)
-        group_height = sum(image.height for image in processed)
+        group_height = sum(image.height for image in group)
         if page_images and page_height + group_height > max_content_height:
             finish_page()
-        page_images.extend(processed)
+        page_images.extend(group)
         page_height += group_height
     finish_page()
     pages[0].save(
@@ -259,6 +277,7 @@ def apply_actions_and_regenerate_grouped(
     previous_baseline = baseline(output_dir) if (output_dir / BASELINE).is_file() else None
     concat_images: list[Image.Image] = []
     filtered_groups: list[list[Image.Image]] = []
+    seen_contexts: set[tuple[str, str]] = set()
     incomplete = False
 
     for label, content in sorted(
@@ -306,7 +325,7 @@ def apply_actions_and_regenerate_grouped(
             # Keep previously reviewed content, including handwriting.
             if annotation_dir == "BGnot":
                 extras = get_extra_pdfs_as_images(
-                    workspace.root, label, annotating, all_labels
+                    workspace.root, label, annotating, all_labels, seen_contexts=seen_contexts
                 )
                 filtered_groups.append([*extras, final_image])
                 touched[label] = True
@@ -359,7 +378,8 @@ def apply_actions_and_regenerate_grouped(
         if not perfect or has_notes:
             filtered_labels.append(label)
             extras = (
-                get_extra_pdfs_as_images(workspace.root, label, annotating, all_labels)
+                get_extra_pdfs_as_images(workspace.root, label, annotating, all_labels,
+                                        seen_contexts=seen_contexts)
                 if annotation_dir == "BGnot"
                 else []
             )
diff --git a/copienator/utils.py b/copienator/utils.py
index 319cbde..b9398e3 100644
--- a/copienator/utils.py
+++ b/copienator/utils.py
@@ -70,7 +70,7 @@ def pdf_images_of_contexts(root_dir, label, all_labels):
 
     pertinent_contexts = []
 
-    for filename in os.listdir(text2_dir):
+    for filename in sorted(os.listdir(text2_dir)):
         if filename.startswith("CTXT ") and filename.endswith(".pdf"):
             # Extract "first_label -> last_label" from "CTXT first_label -> last_label.pdf"
             core = filename[5:-4]
diff --git a/copienator_gui/workflow.py b/copienator_gui/workflow.py
index 274d4f3..d26507e 100644
--- a/copienator_gui/workflow.py
+++ b/copienator_gui/workflow.py
@@ -795,15 +795,6 @@ def build_workflow(show_personal_steps: bool) -> list[StepDefinition]:
             (manual("deploy"),),
             personal=True,
         ),
-        StepDefinition(
-            "personal_print",
-            "Étapes personnelles",
-            "Imprimer une copie",
-            "Étape manuelle via le lecteur PDF.",
-            (manual("print"),),
-            personal=True,
-            optional=True,
-        ),
         StepDefinition(
             "clean",
             "Archivage",
diff --git a/docs/final_output.md b/docs/final_output.md
index 40861c4..3b400af 100644
--- a/docs/final_output.md
+++ b/docs/final_output.md
@@ -177,17 +177,22 @@ For each retained answer, the PDF stacks the following available material:
 3. The model solution from `Sol2/<label>.pdf`.
 4. The same compiled answer block used for the JPEG.
 
-Missing supplementary PDFs are skipped. Contexts can repeat for successive
-questions. Each question's complete group stays together on one page; groups
+Missing supplementary PDFs are skipped. Each distinct context appears only once
+per student PDF, before its first retained question. Duplicate detection uses
+extracted PDF text with Unicode and whitespace normalization, including contexts
+stored under different filenames or validity ranges. Numbers, mathematical
+symbols and case remain significant. Contexts without extractable text use a
+PDF-content hash instead, so unrelated image-only contexts are not collapsed.
+The same rule applies to selective redo and manual annotation modifications.
+Each question's complete group stays together on one page; groups
 are packed until the next would exceed the target height. An oversized group
 gets its own taller page, rather than being split. Pages are raster images saved
 as PDF at 100 dpi, with variable heights, not fixed A4 sheets or searchable text.
 
 The target height is `int(max_image_width * 1.414 * 1.25)` pixels. Small white
-margins are added on the left and above/below the page. The first image of each
-group receives a black border and the second a blue border. These borders are
-assigned by position, so they do not consistently identify question and solution
-when contexts are present or supplementary files are missing.
+margins are added on the left and above/below the page. The question receives
+a black border and the model solution a blue border. Context and student answer
+blocks receive neither border, including when supplementary files are missing.
 
 If nothing survives filtering, `read-grouped` removes old `Concat_F` outputs and
 does not create a new PDF. During a selective `--refaire` merge, saved answer