better faulty encoding detection

copienator/commands/correction.py

@@ -21,6 +21,7 @@ from copienator.commands import grouping
 from copienator.answer_origins import save_origins
 from copienator import prompting
 from copienator.feedback_boxes import valid_feedback_box
+from copienator.latex_json import repair_latex_json
 from copienator import (
     CliError,
     EvaluationWorkspace,
@@ -100,6 +101,16 @@ def report_group_progress(completed: int, total: int) -> None:
     )
 
 
+def parse_correction_response(response_text: str):
+    """Decode model JSON without consuming known LaTeX command prefixes."""
+    return json.loads(
+        repair_latex_json(
+            response_text,
+            additional_content_fields={"text"},
+        )
+    )
+
+
 def request_graceful_stop(_signum=None, _frame=None) -> None:
     """Stop scheduling Gemini calls while allowing in-flight calls to finish."""
     if not stop_requested.is_set():
@@ -319,7 +330,7 @@ def correct_boxes_with_gemini(pid, label, pdf_path, original_feedbacks,
 
     contents, config = prompting.request_for_box_correction(pdf_path, original_feedbacks)
     response_text = call_gemini_with_retries(MODEL_ID_flash, contents, config)
-    corrected_feedbacks = json.loads(response_text)
+    corrected_feedbacks = parse_correction_response(response_text)
 
     global_feedbacks = [f for f in original_feedbacks if not f["box_2d"]]
 
@@ -507,7 +518,7 @@ def process_single_task(task_tuple, precomputed_response=None):
                 tprint(f"Asking Gemini {'Flash' if use_flash else 'Pro  '}: {label} {group_name}")
                 full_response_text = call_gemini_with_retries(model_to_use, contents, config)
 
-            json_data = json.loads(full_response_text)
+            json_data = parse_correction_response(full_response_text)
 
             # Ensure consistency of answer placements
             for p in json_data:

copienator/commands/gemini_for_labels.py

@@ -554,11 +554,18 @@ def run(
         client = genai.Client(api_key=api_key)
 
     groups = group_images(images)
+    total_groups = len(groups)
     active: dict[str, tuple[str, float]] = {}
     progress_lock = Lock()
     incomplete: list[Path] = []
     run_started = time.monotonic()
 
+    def print_group_progress(completed: int) -> None:
+        print(
+            f"Label detection progress: {completed}/{total_groups} copy groups completed.",
+            flush=True,
+        )
+
     def report_progress(group: str, description: str, started: float) -> None:
         with progress_lock:
             active[group] = (description, started)
@@ -582,6 +589,7 @@ def run(
             for group_key, files in groups.items()
         }
         pending = set(futures)
+        print_group_progress(0)
         while pending:
             done, pending = wait(pending, timeout=PROGRESS_INTERVAL)
             for future in done:
@@ -591,6 +599,7 @@ def run(
                     future.result()
                 except IncompleteCopyError as exc:
                     incomplete.extend(exc.images)
+            print_group_progress(total_groups - len(pending))
             if pending:
                 now = time.monotonic()
                 with progress_lock:

copienator/commands/plotting.py

@@ -313,6 +313,7 @@ class ImageViewer:
         # Bindings
         self.root.bind(PLOTTING_KB["OK"], self.on_enter)
         self.root.bind(PLOTTING_KB["previous"], self.on_previous)
+        self.root.bind("P", self.on_previous)
         self.root.bind(PLOTTING_KB["edit"], self.on_edit)
         self.root.bind(PLOTTING_KB["open pdf"], self.on_open_pdf)
         self.root.bind(PLOTTING_KB["open original pdf"], self.on_open_ori_pdf)

copienator/commands/post_correction.py

@@ -16,6 +16,7 @@ from copienator import (
     read_json,
     workspace_from_args,
 )
+from copienator.latex_json import repair_decoded_latex_controls
 
 WORD_LIST_FILE = Path(__file__).resolve().parents[1] / "data" / "liste_francais.txt"
 ACCENT_PATTERN = re.compile(r"[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]")
@@ -89,6 +90,7 @@ def fix_hex_corruption_safe(text: str) -> str:
 
 def repair_json_escape_corruption(text: str) -> str:
     r"""Restore observed LaTeX commands consumed as JSON control escapes."""
+    text = repair_decoded_latex_controls(text)
     replacements = (
         ("\x0crac", r"\frac"),
         ("\x0ceuille", r"\equiv"),
@@ -132,6 +134,33 @@ def clean_obj(value: Any, lookup: dict[str, str]) -> Any:
     return value
 
 
+def unresolved_math_control_paths(value: Any, path: str = "$") -> list[str]:
+    """Locate destructive JSON control escapes left in mathematical text."""
+    if isinstance(value, str):
+        return (
+            [path]
+            if any(
+                control in {"\b", "\f"}
+                for match in MATH_PATTERN.finditer(value)
+                for control in match.group(0)
+            )
+            else []
+        )
+    if isinstance(value, list):
+        return [
+            issue
+            for index, item in enumerate(value)
+            for issue in unresolved_math_control_paths(item, f"{path}[{index}]")
+        ]
+    if isinstance(value, dict):
+        return [
+            issue
+            for key, item in value.items()
+            for issue in unresolved_math_control_paths(item, f"{path}[{key!r}]")
+        ]
+    return []
+
+
 def run(
     workspace: EvaluationWorkspace,
     *,
@@ -141,11 +170,18 @@ def run(
     lookup = build_lookup_map(word_list_path)
     data = read_json(workspace.correction_file)
     cleaned = clean_obj(data, lookup)
+    unresolved = unresolved_math_control_paths(cleaned)
     backup = workspace.root / "correction_precleanup.json"
     atomic_write_bytes(backup, workspace.correction_file.read_bytes())
     print(f"Original JSON backed up to {backup}")
     atomic_write_json(workspace.correction_file, cleaned)
     print(f"Fixed JSON saved to {workspace.correction_file}")
+    if unresolved:
+        print(
+            "Warning: unresolved JSON control characters remain in mathematical "
+            "text at: " + ", ".join(unresolved)
+        )
+        return ExitCode.PARTIAL
     return ExitCode.SUCCESS
 
 

copienator/latex_json.py

@@ -13,6 +13,9 @@ _CONTENT_FIELDS = {'question_content', 'solution_content', 'context_content', 'r
 _CONTENT_KEYS = {f'"{field}"' for field in _CONTENT_FIELDS}
 _STRING = re.compile(r'"(?:\\.|[^"\\])*"', re.DOTALL)
 _COMMAND = re.compile(r'[a-zA-Z]+')
+_MATH = re.compile(r"(\$\$.*?\$\$|\$.*?\$|\\\(.*?\\\)|\\\[.*?\\\])", re.DOTALL)
+_DECODED_COMMAND = re.compile(r"[\b\f\n\r\t]([A-Za-z]+)")
+_JSON_CONTROL_PREFIX = {"\b": "b", "\f": "f", "\n": "n", "\r": "r", "\t": "t"}
 
 
 def normalize_literal_newlines(text: str) -> str:
@@ -44,6 +47,29 @@ underbrace underline underset uparrow updownarrow upsilon usepackage
 '''.split())
 
 
+def repair_decoded_latex_controls(text: str) -> str:
+    r"""Restore known TeX commands after JSON decoded their first letter.
+
+    Only math spans and whitelisted command names are changed. This avoids
+    treating ordinary paragraph newlines or tabs as damaged LaTeX.
+    """
+    def repair_math(math_match: re.Match[str]) -> str:
+        def repair_command(command_match: re.Match[str]) -> str:
+            candidate = (
+                _JSON_CONTROL_PREFIX[command_match.group(0)[0]]
+                + command_match.group(1)
+            )
+            return (
+                f"\\{candidate}"
+                if candidate in _AMBIGUOUS_COMMANDS
+                else command_match.group(0)
+            )
+
+        return _DECODED_COMMAND.sub(repair_command, math_match.group(0))
+
+    return _MATH.sub(repair_math, text)
+
+
 def _repair_string(token: str) -> str:
     output = []
     index = 0
@@ -71,7 +97,9 @@ def _repair_string(token: str) -> str:
     return ''.join(output)
 
 
-def repair_latex_json(raw: str) -> str:
+def repair_latex_json(
+    raw: str, *, additional_content_fields: set[str] | None = None
+) -> str:
     """Repair single TeX backslashes without decoding or rewriting other JSON.
 
     Preserve standard escapes, Unicode escapes, and already escaped TeX.
@@ -79,14 +107,22 @@ def repair_latex_json(raw: str) -> str:
     as a broken ``\\not`` command; known TeX command names take precedence.
     Raw responses/caches should be retained to allow reviewing such cases.
     """
+    additional_content_keys = {
+        f'"{field}"' for field in (additional_content_fields or set())
+    }
     output = []
     end = 0
     previous = None
     for match in _STRING.finditer(raw):
         token = match.group()
-        if (previous is not None and raw[previous.end():match.start()].strip() == ':'
-                and previous.group() in _CONTENT_KEYS):
-            token = _repair_string(token)
+        if previous is not None and raw[previous.end():match.start()].strip() == ':':
+            if previous.group() in _CONTENT_KEYS:
+                token = _repair_string(token)
+            elif previous.group() in additional_content_keys:
+                token = _MATH.sub(
+                    lambda math_match: _repair_string(math_match.group(0)),
+                    token,
+                )
         output.extend((raw[end:match.start()], token))
         end = match.end()
         previous = match