Miscs

copienator/commands/cutleft.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 import argparse
 import threading
-import time
 import tkinter as tk
+from tkinter import messagebox
 from collections.abc import Sequence
 from functools import lru_cache
 from pathlib import Path
@@ -23,6 +23,7 @@ from copienator import (
     workspace_from_target,
 )
 from copienator.filesystem import staged_files
+from copienator.copy_errors import clear_copy_error, mark_copy_error, marked_copy_paths
 
 DELIMITER_WIDTH = 5
 DELIMITER_COLOR = (0, 0, 0)
@@ -156,6 +157,11 @@ class ImageReviewer:
     ) -> None:
         self.files = files
         self.output_dir = output_dir
+        self.workspace = EvaluationWorkspace(output_dir.parent)
+        self.completed = False
+        self.had_errors = False
+        self.stop_prefetch = threading.Event()
+        self.current_result = None
         self.index = 0
         self.current_shift = 0
         self.default_max_per_file = default_max_per_file
@@ -174,6 +180,8 @@ class ImageReviewer:
         self.label_info = tk.Label(self.root, text="", font=("Arial", 12, "bold"))
         self.label_info.pack(pady=5)
         self.root.bind("<Return>", self.on_next)
+        self.root.bind("s", self.on_skip)
+        self.root.protocol("WM_DELETE_WINDOW", self.on_close)
         self.root.bind("n", lambda _event: self.on_shift(50))
         self.root.bind("N", lambda _event: self.on_shift(100))
         self.root.bind("t", lambda _event: self.on_shift(-50))
@@ -194,20 +202,25 @@ class ImageReviewer:
 
     def prefetch_worker(self) -> None:
         processed_index = -1
-        while True:
+        while not self.stop_prefetch.is_set():
             target = self.index + 1
             if target < len(self.files) and target != processed_index:
-                get_pdf_pages(self.files[target])
+                try:
+                    get_pdf_pages(self.files[target])
+                except Exception:
+                    pass  # The foreground review reports and flags conversion errors.
                 processed_index = target
-            time.sleep(0.05)
+            self.stop_prefetch.wait(0.05)
 
     def load_current_image(self) -> None:
         if self.index >= len(self.files):
             print("All files processed.")
-            self.root.destroy()
+            self.completed = True
+            self.on_close()
             return
         self.is_processing = False
         self.current_shift = 0
+        self.current_result = None
         self.trigger_processing(self.files[self.index], self.current_shift)
 
     def trigger_processing(self, pdf_path: Path, shift: int) -> None:
@@ -228,13 +241,13 @@ class ImageReviewer:
     def check_manual_queue(self, pdf_path: Path) -> None:
         try:
             result = self.manual_queue.get_nowait()
+            self.is_processing = False
             if result is None:
                 print(f"Failed to process {pdf_path.name}, skipping.")
-                self.index += 1
-                self.load_current_image()
+                self._mark_error(pdf_path, "Échec de la conversion pour la découpe des marges")
+                self._advance()
             else:
                 self.handle_processing_result(result, pdf_path)
-            self.is_processing = False
         except Empty:
             self.root.after(100, lambda: self.check_manual_queue(pdf_path))
 
@@ -244,7 +257,7 @@ class ImageReviewer:
         pdf_path: Path,
     ) -> None:
         self.current_preview = result[0]
-        save_results(result, pdf_path, self.output_dir)
+        self.current_result = result
         self.update_display(pdf_path.name, result[2])
 
     def update_display(self, filename: str, schema: dict[str, object]) -> None:
@@ -258,7 +271,7 @@ class ImageReviewer:
                 f"[{self.index + 1}/{len(self.files)}] {filename} | "
                 f"Shift: {self.current_shift}px\nFiles: {schema['number_of_files']} | "
                 f"Cols: {schema['columns_per_file']}\n"
-                "Enter: Next | n: +50 | N: +100 | t: -50 | "
+                "Enter: Save and next | s: flag error and skip | n: +50 | N: +100 | t: -50 | "
                 "1: use single column"
             ),
             fg="black",
@@ -272,8 +285,34 @@ class ImageReviewer:
         self.trigger_processing(self.files[self.index], self.current_shift)
 
     def on_next(self, _event: object) -> None:
-        if self.is_processing:
+        if self.is_processing or self.current_result is None:
             return
+        pdf_path = self.files[self.index]
+        try:
+            save_results(self.current_result, pdf_path, self.output_dir)
+            clear_copy_error(self.workspace, pdf_path)
+        except Exception as exc:
+            self._mark_error(pdf_path, f"Échec de l’enregistrement : {exc}")
+            messagebox.showerror("Enregistrement impossible", str(exc), parent=self.root)
+            return
+        self._advance()
+
+    def _mark_error(self, pdf_path: Path, reason: str) -> None:
+        mark_copy_error(self.workspace, pdf_path, reason)
+        self.had_errors = True
+        print(f"[Copie signalée] {pdf_path.name}: {reason}")
+
+    def on_skip(self, _event=None) -> None:
+        if self.is_processing or self.index >= len(self.files):
+            return
+        self._mark_error(self.files[self.index], "Problème repéré pendant la découpe des marges")
+        self._advance()
+
+    def on_close(self) -> None:
+        self.stop_prefetch.set()
+        self.root.destroy()
+
+    def _advance(self) -> None:
         self.index += 1
         self.current_shift = 0
         self.current_max_per_file = self.default_max_per_file
@@ -304,23 +343,27 @@ def run(
     target: Path,
     *,
     fullpage: bool = False,
+    marked: bool = False,
 ) -> ExitCode:
-    files = _selected_files(workspace, target)
+    files = marked_copy_paths(workspace) if marked else _selected_files(workspace, target)
     if not files:
         print("No PDF files found.")
         return ExitCode.SUCCESS
     workspace.cutleft_dir.mkdir(parents=True, exist_ok=True)
     _get_pdf_pages_cached.cache_clear()
-    ImageReviewer(
+    reviewer = ImageReviewer(
         files,
         workspace.cutleft_dir,
         default_max_per_file=1 if fullpage else 5,
     )
-    return ExitCode.SUCCESS
+    if not reviewer.completed:
+        return ExitCode.INTERRUPTED
+    return ExitCode.PARTIAL if reviewer.had_errors else ExitCode.SUCCESS
 
 
 def build_parser() -> argparse.ArgumentParser:
     parser = target_parser("Interactively crop the label margin from PDF copies")
+    parser.add_argument("--marked", action="store_true", help="Review flagged copies and clear each flag after saving with Enter")
     parser.add_argument(
         "--fullpage",
         action="store_true",
@@ -334,11 +377,10 @@ def main(argv: Sequence[str] | None = None) -> int:
 
     def handle(args: argparse.Namespace) -> ExitCode:
         workspace, target = workspace_from_target(args)
-        return run(workspace, target, fullpage=args.fullpage)
+        return run(workspace, target, fullpage=args.fullpage, marked=args.marked)
 
     return execute(parser, argv, handle)
 
 
 if __name__ == "__main__":
     raise SystemExit(main())
-

copienator/commands/gemini_for_enonce.py

@@ -190,6 +190,7 @@ def generate_rubrics(client, group_context_text: str) -> dict[str, str]:
             types.Part.from_text(text=f"--- CONTENU DU GROUPE ---\n{group_context_text}"),
         ])],
         config=types.GenerateContentConfig(
+            automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
             system_instruction="Rédige tous les barèmes et consignes de notation en français. Conserve les clés JSON, les labels et les formules mathématiques.",
             temperature=0.2,
             response_mime_type="application/json",
@@ -243,6 +244,7 @@ def refine_existing(workspace: EvaluationWorkspace, mode: str, *, api_client=Non
                 + context + "\n\n" + "\n\n".join(questions.values())
             ))])],
             config=types.GenerateContentConfig(
+                automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
                 temperature=0.1, response_mime_type="application/json",
                 response_json_schema=LabelGroups.model_json_schema(),
             ),
@@ -337,6 +339,7 @@ def process_exam(
     ]
 
     config_1 = types.GenerateContentConfig(
+        automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
         temperature=0.1,
         response_mime_type="application/json",
         response_json_schema=ExamQuestions.model_json_schema(),
@@ -375,6 +378,7 @@ def process_exam(
     ]
 
     config_2 = types.GenerateContentConfig(
+        automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
         temperature=0.1,
         response_mime_type="application/json",
         response_json_schema=ExamSolutions.model_json_schema(),
@@ -411,6 +415,7 @@ def process_exam(
     ]
 
     config_3 = types.GenerateContentConfig(
+        automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
         temperature=0.1,
         response_mime_type="application/json",
         response_json_schema=ExamContext.model_json_schema(),

copienator/commands/gemini_for_labels.py

@@ -76,6 +76,8 @@ be missing.
 
 ##wrong_labels##
 
+##wrong_label_text_context##
+
 Here's a list of the names of the students, pick the one that matches
 the best or `\"Unknown\"` if you cannot read the name
 
@@ -133,6 +135,8 @@ be missing.
 
 ##wrong_labels##
 
+##wrong_label_text_context##
+
 Since this copy isn't the first part of a sequence, simply set the
 name to `\"Continued\"`."""
 
@@ -147,7 +151,66 @@ class AnnotationData(BaseModel):
     )
 
 
-def generate_request(file, labels, names, context_labels, wrong_labels):
+TEXT_CONTEXT_MAX_CHARS = 4000
+
+
+def _label_filename(path: Path) -> str:
+    return path.stem if path.suffix.casefold() in {".tex", ".txt"} else path.name
+
+
+def _common_prefix_length(left: str, right: str) -> int:
+    left_folded = left.casefold()
+    right_folded = right.casefold()
+    limit = min(len(left_folded), len(right_folded))
+    for index in range(limit):
+        if left_folded[index] != right_folded[index]:
+            return index
+    return limit
+
+
+def closest_text_context(
+    workspace: EvaluationWorkspace, wrong_labels: list[str]
+) -> tuple[Path | None, str]:
+    """Return a bounded excerpt from the Text file closest to an invalid label."""
+    text_dir = workspace.root / "Text"
+    if not wrong_labels or not text_dir.is_dir():
+        return None, ""
+
+    ranked: list[tuple[int, str, Path]] = []
+    for path in text_dir.iterdir():
+        if not path.is_file() or path.suffix.casefold() == ".pdf":
+            continue
+        filename = _label_filename(path)
+        prefix_length = max(
+            _common_prefix_length(filename, wrong_label)
+            for wrong_label in wrong_labels
+        )
+        if prefix_length:
+            ranked.append((prefix_length, filename.casefold(), path))
+
+    for _prefix_length, _filename, path in sorted(
+        ranked, key=lambda item: (-item[0], item[1])
+    ):
+        try:
+            content = path.read_text(encoding="utf-8")
+        except (OSError, UnicodeError):
+            continue
+        if len(content) > TEXT_CONTEXT_MAX_CHARS:
+            content = content[:TEXT_CONTEXT_MAX_CHARS] + "\n[excerpt truncated]"
+        return path, content
+    return None, ""
+
+
+def generate_request(
+    file,
+    labels,
+    names,
+    context_labels,
+    wrong_labels,
+    wrong_label_text_context="",
+    wrong_label_text_file: Path | None = None,
+    seed: int = 0,
+):
     """Generates request for Gemini with context."""
 
     image_path = Path(file)
@@ -162,9 +225,36 @@ def generate_request(file, labels, names, context_labels, wrong_labels):
         text = my_prompt2.replace("##labels##", labels)\
                          .replace("##prev_context##", context_str)
     if wrong_labels:
-        text= text.replace("##wrong_labels##\n\n", f"On a previous request, you answered with the following wrong labels : {wrong_labels}. These are wrong, since they do not exactly match any of the labels in the previous list.")
+        formatted_wrong_labels = "\n".join(f'- "{label}"' for label in wrong_labels)
+        text = text.replace(
+            "##wrong_labels##",
+            "On the previous request for this image, you answered with these "
+            "invalid labels:\n"
+            f"{formatted_wrong_labels}\n"
+            "They are wrong because they do not exactly match any label in the "
+            "valid list above.\n\n"
+            "CRITICAL RETRY CONSTRAINT: NEVER return any of the invalid labels "
+            "listed above again. Your answer must use only exact labels copied "
+            "verbatim from the valid list. If the handwriting resembles an "
+            "invalid label, choose the closest exact valid label instead.",
+        )
     else:
-        text = text.replace("##wrong_labels##\n\n", "")
+        text = text.replace("##wrong_labels##", "")
+
+    if wrong_label_text_context and wrong_label_text_file:
+        text = text.replace(
+            "##wrong_label_text_context##",
+            "Here is an excerpt from the exam text file whose name has the "
+            "longest prefix in common with the invalid label(s), "
+            f"`{wrong_label_text_file.name}`:\n\n"
+            "<exam_text_excerpt>\n"
+            f"{wrong_label_text_context}\n"
+            "</exam_text_excerpt>\n\n"
+            "Use this excerpt as extra context for identifying the handwritten "
+            "label, but return only an exact label from the valid list above.",
+        )
+    else:
+        text = text.replace("##wrong_label_text_context##", "")
 
 
     contents = [
@@ -181,9 +271,10 @@ def generate_request(file, labels, names, context_labels, wrong_labels):
     ]
 
     generate_content_config = types.GenerateContentConfig(
+        automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
         temperature=1.0,
         top_p=0.95,
-        seed=0,
+        seed=seed,
         max_output_tokens=65535,
         response_mime_type= "application/json",
         response_json_schema= AnnotationData.model_json_schema(),
@@ -293,17 +384,30 @@ def process_copy_group(
             f"{len(accumulated_labels)} accumulated labels..."
         )
         attempt = 0
+        label_retry_count = 0
         wrong_labels: list[str] = []
         while True:
             if attempt > 0:
                 sleep(10 * attempt)
             try:
+                text_context_file, text_context = closest_text_context(
+                    workspace, wrong_labels
+                )
+                if text_context_file:
+                    print(
+                        f"[{group_key}] Retry context for {image_file.name}: "
+                        f"{text_context_file.relative_to(workspace.root)}"
+                    )
+                request_seed = max(0, label_retry_count - 1)
                 contents, request_config = generate_request(
                     image_file,
                     labels_text,
                     names_text,
                     accumulated_labels,
                     wrong_labels,
+                    text_context,
+                    text_context_file,
+                    seed=request_seed,
                 )
                 response = client.models.generate_content(
                     model=MODEL_ID,
@@ -321,9 +425,23 @@ def process_copy_group(
                         f"Error: {image_file.name} contained unknown labels: "
                         f"{unknown}"
                     )
-                    wrong_labels.extend(unknown)
-                    attempt += 1
-                    continue
+                    unique_unknown = list(dict.fromkeys(unknown))
+                    if (
+                        label_retry_count >= 2
+                        and set(unique_unknown) == set(wrong_labels)
+                    ):
+                        for item in annotation.list:
+                            if item.label in unique_unknown:
+                                item.label = f"??{item.label}"
+                        print(
+                            f"Warning: {image_file.name} repeated the same unknown "
+                            "label(s) on the third try; keeping them with a ?? prefix."
+                        )
+                    else:
+                        wrong_labels = unique_unknown
+                        label_retry_count += 1
+                        attempt += 1
+                        continue
                 if annotation.name not in valid_names:
                     print(
                         f"Error: {image_file.name} returned unknown name: "

copienator/commands/page_splitter.py

@@ -26,6 +26,7 @@ from copienator import (
     workspace_from_target,
 )
 from copienator.platform import launch_pdf_arranger
+from copienator.copy_errors import marked_copy_paths
 
 # Keep the new shortcut available with older personal configuration files.
 PAGE_SPLITTER_KB = {"reverse_pages": "i", **PAGE_SPLITTER_KB}
@@ -647,8 +648,8 @@ def _selected_inputs(
     return list(reversed(candidates))
 
 
-def run(workspace: EvaluationWorkspace, target: Path) -> ExitCode:
-    inputs = _selected_inputs(workspace, target)
+def run(workspace: EvaluationWorkspace, target: Path, *, marked: bool = False) -> ExitCode:
+    inputs = list(reversed(marked_copy_paths(workspace, originals=True))) if marked else _selected_inputs(workspace, target)
     if not inputs:
         print(f"No PDF files found in {target}")
         return ExitCode.SUCCESS
@@ -659,7 +660,9 @@ def run(workspace: EvaluationWorkspace, target: Path) -> ExitCode:
 
 
 def build_parser() -> argparse.ArgumentParser:
-    return target_parser("Interactively split and reorder scanned PDF pages")
+    parser = target_parser("Interactively split and reorder scanned PDF pages")
+    parser.add_argument("--marked", action="store_true", help="Process only copies flagged during margin review")
+    return parser
 
 
 def main(argv: Sequence[str] | None = None) -> int:
@@ -667,7 +670,7 @@ def main(argv: Sequence[str] | None = None) -> int:
 
     def handle(args: argparse.Namespace) -> ExitCode:
         workspace, target = workspace_from_target(args)
-        return run(workspace, target)
+        return run(workspace, target, marked=args.marked)
 
     return execute(parser, argv, handle)
 

copienator/copy_errors.py

@@ -0,0 +1,48 @@
+"""Persistent copy flags shared by page splitting, margin review and the GUI."""
+from pathlib import Path
+
+from copienator import CliError, EvaluationWorkspace, atomic_update_json, read_json
+
+
+def _path(workspace: EvaluationWorkspace) -> Path:
+    return workspace.metadata_dir / "copy_errors.json"
+
+
+def _validate(value) -> dict[str, str]:
+    if not isinstance(value, dict) or any(
+        not isinstance(name, str) or not isinstance(reason, str)
+        or "/" in name or "\\" in name or Path(name).suffix.casefold() != ".pdf"
+        for name, reason in value.items()
+    ):
+        raise CliError("Invalid copy_errors.json: expected PDF filenames and error descriptions")
+    return value
+
+
+def copy_errors(workspace: EvaluationWorkspace) -> dict[str, str]:
+    return _validate(read_json(_path(workspace), default={}))
+
+
+def mark_copy_error(workspace: EvaluationWorkspace, pdf: Path, reason: str) -> None:
+    def update(errors):
+        _validate(errors)[pdf.name] = reason
+    atomic_update_json(_path(workspace), update, default_factory=dict)
+
+
+def clear_copy_error(workspace: EvaluationWorkspace, pdf: Path) -> None:
+    if not _path(workspace).exists():
+        return
+    def update(errors):
+        _validate(errors).pop(pdf.name, None)
+    atomic_update_json(_path(workspace), update, default_factory=dict)
+
+
+def marked_copy_paths(workspace: EvaluationWorkspace, *, originals: bool = False) -> list[Path]:
+    directories = ([workspace.original_copies_dir, workspace.copies_dir, workspace.root]
+                   if originals else [workspace.copies_dir])
+    result = []
+    for name in sorted(copy_errors(workspace), key=str.casefold):
+        path = next((directory / name for directory in directories if (directory / name).is_file()), None)
+        if path is None:
+            raise CliError(f"Marked copy not found: {name}")
+        result.append(path)
+    return result

copienator/prompting.py

@@ -182,6 +182,7 @@ def generate_request(input_dir, file, full_label):
     ]
 
     generate_content_config = types.GenerateContentConfig(
+        automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
         temperature=1.0,
         top_p=0.95,
         seed=0,
@@ -259,6 +260,7 @@ Commentaires d'origine :
     ]
 
     config = types.GenerateContentConfig(
+        automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
         temperature=1.0,
         response_mime_type="application/json",
         response_json_schema=TypeAdapter(List[FeedbackItem]).json_schema()
@@ -288,7 +290,10 @@ Voici les labels possibles. Ta réponse doit être l'un d'entre eux :
     contents = [types.Content(role="user", parts=[
             types.Part.from_bytes(data=get_single_image_bytes(pdf_path), mime_type="image/jpeg"),
             types.Part.from_text(text=prompt)])]
-    config = types.GenerateContentConfig(temperature=1.0)
+    config = types.GenerateContentConfig(
+        automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
+        temperature=1.0,
+    )
     return contents, config
 
 def request_for_additional_answer(pdf_path, label, enonce, labels_txt):
@@ -316,6 +321,9 @@ d'entre eux :
         types.Part.from_bytes(data=get_single_image_bytes(pdf_path), mime_type="image/jpeg"),
         types.Part.from_text(text=prompt)
     ])]
-    config = types.GenerateContentConfig(temperature=1.0, response_mime_type="application/json")
+    config = types.GenerateContentConfig(
+        automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
+        temperature=1.0,
+        response_mime_type="application/json",
+    )
     return contents, config
-