Guide d'utilisateur GUI ; Recoverable statement/Persp Gemini generation

copienator/commands/gemini_for_enonce.py

@@ -1,29 +1,36 @@
 from __future__ import annotations
 
 import argparse
+import io
+import json
 import re
 from collections.abc import Sequence
 from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
 from pathlib import Path
+from uuid import uuid4
 
 from google import genai
 from google.genai import types
 from pydantic import BaseModel, Field, field_validator
 
-from copienator import configuration as config
-from copienator import utils
 from copienator import (
     CliError,
     EvaluationWorkspace,
     ExitCode,
+    atomic_write_json,
     atomic_write_text,
     evaluation_parser,
     execute,
+    utils,
     workspace_from_args,
 )
+from copienator import configuration as config
+from copienator.filesystem import staged_files
+from copienator.latex_json import normalize_literal_newlines, repair_latex_json
 from copienator.platform import validate_windows_labels
-from copienator.latex_json import repair_latex_json, normalize_literal_newlines
-from copienator.filesystem import staged_directory
+from copienator.rubrics import present_rubric_labels
+from copienator.statement_checkpoints import extract_stage, report_error
 from copienator.utils import compile_to_pdf
 
 
@@ -130,7 +137,15 @@ def validate_rubric_content(content: str) -> str:
     an intentional newline. Do not guess or rewrite those after decoding.
     """
     if any((ord(char) < 32 and char != "\n") or ord(char) == 127 for char in content):
-        raise ValueError("Rubric contains control characters, likely from unescaped LaTeX")
+        positions = ", ".join(
+            f"U+{ord(char):04X} at offset {index}"
+            for index, char in enumerate(content)
+            if (ord(char) < 32 and char != "\n") or ord(char) == 127
+        )
+        raise ValueError(
+            "Rubric contains control characters, likely from unescaped LaTeX: "
+            f"{positions}; decoded content: {content!r}"
+        )
     return content
 
 # --- Modèle fusionné (pour le reste du script) ---
@@ -224,8 +239,7 @@ def find_file(folder: Path, base_name: str) -> Path | None:
     return None
 
 
-def generate_rubrics(client, group_context_text: str) -> dict[str, str]:
-    """Use the same rubric request for full and selective statement generation."""
+def _request_rubrics(client, group_context_text: str):
     response = client.models.generate_content(
         model=MODEL_ID,
         contents=[types.Content(role="user", parts=[
@@ -240,10 +254,71 @@ def generate_rubrics(client, group_context_text: str) -> dict[str, str]:
             response_json_schema=GroupRubrics.model_json_schema(),
         ),
     )
-    rubrics = GroupRubrics.model_validate_json(repair_latex_json(response.text)).rubrics
-    if len({item.label for item in rubrics}) != len(rubrics):
-        raise ValueError("Gemini returned duplicate rubric labels")
-    return {item.label: validate_rubric_content(item.rubric_content) for item in rubrics}
+    return response
+
+
+def generate_rubrics(
+    client, group_context_text: str, *, workspace: EvaluationWorkspace | None = None,
+    expected_labels: list[str] | None = None,
+) -> dict[str, str]:
+    """Save the raw response before decoding; report failures without hiding text."""
+    raw = None
+    response_path = None
+    try:
+        response = _request_rubrics(client, group_context_text)
+        raw = response.text
+        if workspace is not None:
+            stamp = datetime.now(UTC).strftime('%Y%m%dT%H%M%S%fZ')
+            cache_path = workspace.root / 'Cache' / 'rubric_responses' / f'{stamp}-{uuid4().hex}.json'
+            atomic_write_text(cache_path, raw or '')
+            response_path = cache_path
+        payload = json.loads(repair_latex_json(raw))
+        if not isinstance(payload, dict) or not isinstance(payload.get('rubrics'), list):
+            raise TypeError('Gemini response must contain a rubrics list')
+        result = {}
+        item_errors = []
+        for entry in payload['rubrics']:
+            try:
+                item = RubricItem.model_validate(entry)
+                if item.label in result:
+                    raise ValueError('Gemini returned duplicate rubric labels')
+                if expected_labels is not None and item.label not in expected_labels:
+                    raise ValueError('Unexpected rubric label')
+                result[item.label] = validate_rubric_content(item.rubric_content)
+            except ValueError as error:
+                label = entry.get('label', 'unknown') if isinstance(entry, dict) else 'unknown'
+                item_errors.append(f"Question {label}: {error}; item (escaped): {entry!r}")
+        if workspace is not None and expected_labels is not None:
+            # Retain valid siblings even if another rubric in this response fails.
+            for label, content in result.items():
+                if label in expected_labels and content.strip():
+                    atomic_write_text(workspace.root / 'Persp' / label.replace('/', '_'),
+                                      f'{label}\n{content}')
+        if item_errors:
+            raise ValueError('\n'.join(item_errors))
+        if expected_labels is not None and (
+            set(result) != set(expected_labels) or any(not value.strip() for value in result.values())
+        ):
+            raise CliError("Incomplete or unexpected Gemini rubrics; failed group preserved")
+        return result
+    except Exception as error:
+        diagnostic = (
+            f"Rubric generation failed for {expected_labels or 'unspecified group'}: {error}\n"
+            f"Raw response (escaped): {raw!r}\n"
+        )
+        if response_path is not None:
+            diagnostic += f"Raw response saved to: {response_path}\n"
+        print(diagnostic, end='', flush=True)
+        if workspace is not None:
+            log_path = workspace.root / 'Cache' / 'rubric_errors.log'
+            try:
+                log_path.parent.mkdir(parents=True, exist_ok=True)
+                with log_path.open('a', encoding='utf-8') as log:
+                    log.write(f"[{datetime.now(UTC).isoformat()}]\n{diagnostic}\n")
+                print(f"Error log: {log_path}", flush=True)
+            except OSError as log_error:
+                print(f"Could not save rubric error log: {log_error}", flush=True)
+        raise
 
 
 def validate_groups(groups: list[list[str]], labels: list[str]) -> None:
@@ -253,12 +328,19 @@ def validate_groups(groups: list[list[str]], labels: list[str]) -> None:
         raise CliError("Groups must contain every existing label exactly once")
 
 
-def refine_existing(workspace: EvaluationWorkspace, mode: str, *, api_client=None) -> ExitCode:
-    """Regroup or replace rubrics without regenerating statements or solutions."""
+def refine_existing(workspace: EvaluationWorkspace, mode: str, *, api_client=None,
+                    delete_persp: bool = False) -> ExitCode:
+    """Regroup questions or generate missing rubrics, retaining completed groups."""
     labels = workspace.read_labels()
     if not labels or len(labels) != len(set(labels)):
         raise CliError("Generate unique question labels before refining the statement")
     validate_windows_labels(labels)
+    present = set(present_rubric_labels(workspace.root, labels)) if mode == 'persp' else set()
+    if mode == 'persp':
+        print(f"Persp: {len(present)}/{len(labels)} present, {len(labels) - len(present)} missing.")
+        if len(present) == len(labels) and not delete_persp:
+            print('All Persp rubrics are already present; nothing to generate.')
+            return ExitCode.SUCCESS
     questions = {}
     for label in labels:
         safe_label = label.replace("/", "_")
@@ -304,21 +386,34 @@ def refine_existing(workspace: EvaluationWorkspace, mode: str, *, api_client=Non
                   for line in workspace.label_groups_file.read_text(encoding="utf-8").splitlines()
                   if line.strip()]
         validate_groups(groups, labels)
-        with staged_directory(workspace.root / "Persp") as staging:
-            for group in groups:
-                print(f"Generating rubric (Persp) for group: {', '.join(group)}...")
-                group_content = (
-                    "Contexte général de l’examen, fourni uniquement pour comprendre les questions :\n"
-                    + context + "\n\nProduis des barèmes UNIQUEMENT pour les labels suivants : "
-                    + ", ".join(group) + "\n\n" + "\n\n".join(questions[label] for label in group)
-                )
-                rubrics = generate_rubrics(api_client, group_content)
-                if set(rubrics) != set(group) or any(not value.strip() for value in rubrics.values()):
-                    raise CliError("Incomplete or unexpected Gemini rubrics; previous Persp preserved")
+        if delete_persp:
+            persp_dir = workspace.root / 'Persp'
+            remove = [path.name for path in persp_dir.iterdir() if path.is_file()] if persp_dir.exists() else []
+            with staged_files(persp_dir, remove=remove):
+                pass
+            present.clear()
+            print('Deleted current Persp; generating all rubrics.')
+        generated = 0
+        for group in groups:
+            missing = [label for label in group if label not in present]
+            if not missing:
+                print(f"Skipping existing Persp for group: {', '.join(group)}.")
+                continue
+            print(f"Generating missing rubric (Persp): {', '.join(missing)}...")
+            group_content = (
+                "Contexte général de l’examen, fourni uniquement pour comprendre les questions :\n"
+                + context + "\n\nProduis des barèmes UNIQUEMENT pour les labels suivants : "
+                + ", ".join(missing) + "\n\n" + "\n\n".join(questions[label] for label in group)
+            )
+            rubrics = generate_rubrics(
+                api_client, group_content, workspace=workspace, expected_labels=missing,
+            )
+            with staged_files(workspace.root / "Persp") as staging:
                 for label, rubric in rubrics.items():
                     (staging / label.replace("/", "_")).write_text(
                         f"{label}\n{rubric}", encoding="utf-8")
-        print("Replaced Persp with Gemini rubrics.")
+            generated += len(rubrics)
+        print(f"Generated {generated} missing Persp rubrics; all {len(labels)} are now present.")
     else:
         raise ValueError(f"Unknown statement refinement: {mode}")
     return ExitCode.SUCCESS
@@ -336,9 +431,15 @@ def process_exam(
     cache_dir.mkdir(exist_ok=True)
     tmp_dir.mkdir(exist_ok=True)
 
-    cache_q_file = cache_dir / "gemini_questions.json"
-    cache_s_file = cache_dir / "gemini_solutions.json"
-    cache_c_file = cache_dir / "gemini_context.json"
+    if restart:
+        archive = cache_dir / 'statement_history' / uuid4().hex
+        for path in [*cache_dir.glob('statement_*.json'), *cache_dir.glob('gemini_*.json'),
+                     tmp_dir / 'exam_items.txt', tmp_dir / 'exam_items_full.txt']:
+            if path.is_file():
+                archive.mkdir(parents=True, exist_ok=True)
+                path.replace(archive / path.name)
+        print('Starting a new extraction; previous checkpoints and edits archived.')
+        atomic_write_json(cache_dir / 'statement_reset_outputs.json', {'pending': True})
 
     # 1. Resolve files
     pdf_path = workspace.statement_pdf_file
@@ -392,38 +493,16 @@ def process_exam(
         response_json_schema=ExamQuestions.model_json_schema(),
     )
 
-    if cache_q_file.is_file() and not restart:
-        print("Loading cached questions from Cache/gemini_questions.json...")
-        response_q_text = cache_q_file.read_text(encoding="utf-8")
-    else:
-        print("Sending request 1 (Questions) to Gemini...")
-        response_q = client.models.generate_content(
-            model=MODEL_ID,
-            contents=contents_1,
-            config=config_1
-        )
-        response_q_text = response_q.text
-        print("Saving questions to cache...")
-        atomic_write_text(cache_q_file, response_q_text)
-
-    questions_data = ExamQuestions.model_validate_json(repair_latex_json(response_q_text))
+    questions_data = ExamQuestions(questions=extract_stage(
+        folder, 'questions', QuestionOnlyItem, 'question_content',
+        lambda _pending: client.models.generate_content(model=MODEL_ID, contents=contents_1, config=config_1),
+    ))
 
     # ==========================================
     # REQUÊTE 2 : Extraction des Corrections
     # ==========================================
     extracted_questions_json = questions_data.model_dump_json(indent=2)
 
-    contents_2 = [
-        types.Content(
-            role="user",
-            parts=[
-                types.Part.from_text(text=PROMPT_2),
-                types.Part.from_text(text=f"--- QUESTIONS EXTRAITES ---\n{extracted_questions_json}"),
-                types.Part.from_text(text=f"--- CORRECTION SOURCE ({correction_path.name}) ---\n{correction_text}"),
-            ],
-        )
-    ]
-
     config_2 = types.GenerateContentConfig(
         automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
         temperature=0.1,
@@ -431,21 +510,19 @@ def process_exam(
         response_json_schema=ExamSolutions.model_json_schema(),
     )
 
-    if cache_s_file.is_file() and not restart:
-        print("Loading cached solutions from Cache/gemini_solutions.json...")
-        response_s_text = cache_s_file.read_text(encoding="utf-8")
-    else:
-        print("Sending request 2 (Solutions) to Gemini...")
-        response_s = client.models.generate_content(
-            model=MODEL_ID,
-            contents=contents_2,
-            config=config_2
-        )
-        response_s_text = response_s.text
-        print("Saving solutions to cache...")
-        atomic_write_text(cache_s_file, response_s_text)
-
-    solutions_data = ExamSolutions.model_validate_json(repair_latex_json(response_s_text))
+    question_labels = [q.label for q in questions_data.questions]
+    def request_solutions(pending):
+        requested = pending if pending is not None else question_labels
+        selected = ExamQuestions(questions=[q for q in questions_data.questions if q.label in requested])
+        contents = [types.Content(role='user', parts=[
+            types.Part.from_text(text=PROMPT_2),
+            types.Part.from_text(text='--- QUESTIONS EXTRAITES ---\n' + selected.model_dump_json(indent=2)),
+            types.Part.from_text(text=f'--- CORRECTION SOURCE ({correction_path.name}) ---\n{correction_text}'),
+        ])]
+        return client.models.generate_content(model=MODEL_ID, contents=contents, config=config_2)
+    solutions_data = ExamSolutions(solutions=extract_stage(
+        folder, 'solutions', SolutionOnlyItem, 'solution_content', request_solutions, labels=question_labels,
+    ))
 
     # ==========================================
     # REQUÊTE 3 : Extraction du Contexte (Notations, etc.)
@@ -468,21 +545,22 @@ def process_exam(
         response_json_schema=ExamContext.model_json_schema(),
     )
 
-    if cache_c_file.is_file() and not restart:
-        print("Loading cached context from Cache/gemini_context.json...")
-        response_c_text = cache_c_file.read_text(encoding="utf-8")
-    else:
-        print("Sending request 3 (Context) to Gemini...")
-        response_c = client.models.generate_content(
-            model=MODEL_ID,
-            contents=contents_3,
-            config=config_3
-        )
-        response_c_text = response_c.text
-        print("Saving context to cache...")
-        atomic_write_text(cache_c_file, response_c_text)
+    def request_contexts(pending):
+        contents = contents_3
+        if pending:
+            contents = [types.Content(role='user', parts=[*contents_3[0].parts,
+                types.Part.from_text(text='Produis des contextes UNIQUEMENT pour les labels cibles suivants : '
+                                     + ', '.join(pending))])]
+        return client.models.generate_content(model=MODEL_ID, contents=contents, config=config_3)
+    context_data = ExamContext(contexts=extract_stage(
+        folder, 'contexts', ExtractedContext, 'context_content', request_contexts, labels=question_labels,
+    ))
 
-    context_data = ExamContext.model_validate_json(repair_latex_json(response_c_text))
+    grouping_checkpoint = cache_dir / 'statement_grouping.json'
+    if grouping_checkpoint.is_file():
+        print('Resuming approved question groups from checkpoint.')
+        grouped_extraction = GroupedExamExtraction.model_validate_json(grouping_checkpoint.read_text())
+        return write_exam_outputs(workspace, grouped_extraction, client)
 
     # ==========================================
     # FUSION des trois résultats
@@ -634,8 +712,7 @@ def process_exam(
     # --- WRITE TEXT FILES ---
     print(f"Writing items files to {items_file.name} and {full_items_file.name}...")
 
-    with open(items_file, "w", encoding="utf-8") as f, \
-         open(full_items_file, "w", encoding="utf-8") as f_full:
+    with io.StringIO() as f, io.StringIO() as f_full:
 
         header = "# Edit labels. Modify groups (---). Ensure label uniqueness (XX). Duplicate CONTEXT.\n\n"
         f.write(header)
@@ -664,8 +741,15 @@ def process_exam(
                     f.write(f"CONTEXT ### {safe_content}\n")
                     f_full.write(f"CONTEXT ### {safe_content}\n")
 
+        if not items_file.exists():
+            atomic_write_text(items_file, f.getvalue())
+        if not full_items_file.exists():
+            atomic_write_text(full_items_file, f_full.getvalue())
+
     # --- OPEN EDITOR AND PARSE ---
     while True:
+        print('Check the question list against enonce.pdf before approving: '
+              'verify completeness, labels, order and context coverage.')
         print("Opening items file for editing...")
         utils.edit_file_and_enter(items_file)
 
@@ -704,6 +788,9 @@ def process_exam(
                 all_new_q_labels.append(lbl)
 
         # Mapping from original question index to new label
+        if len(all_new_q_labels) != len(questions_only):
+            raise CliError('The edited groups must retain every extracted question. '
+                           'Grouping edits were preserved; correct the list and rerun.')
         idx_to_new_label = {i: all_new_q_labels[i] for i in range(min(len(questions_only), len(all_new_q_labels)))}
 
         orig_q_idx = 0
@@ -802,37 +889,40 @@ def process_exam(
     # Save labels and proceed
     grouped_extraction = GroupedExamExtraction(groups=grouped_items)
 
-    # 2. Setup output directories
-    text_dir = folder / "Text"
-    sol_dir = folder / "Sol"
-    text2_dir = folder / "Text2"
-    sol2_dir = folder / "Sol2"
-    persp_dir = folder / "Persp"
-    dirs = [text_dir, sol_dir, text2_dir, sol2_dir, persp_dir]
-
-    import shutil
-    # Ask only if any directory already exists
-    if any(d.exists() for d in dirs):
-        answer = input(
-            "Output directories already exist. Delete their contents? [y/N] "
-        ).strip().lower()
-
-        if answer not in ("y", "yes"):
-            raise CliError("Output replacement aborted", ExitCode.INVALID_ARGUMENTS)
-        # Empty each directory
-        for d in dirs:
-            if d.exists():
-                shutil.rmtree(d)
-            d.mkdir(parents=True)
-    else:
-        # Create them if they don't exist
-        for d in dirs:
-            d.mkdir(parents=True)
-
-        text_dir.mkdir(exist_ok=True)
-        sol_dir.mkdir(exist_ok=True)
+    if not labels_list or len(labels_list) != len(set(labels_list)):
+        raise CliError('Approved groups must contain unique question labels')
+    atomic_write_json(grouping_checkpoint, grouped_extraction.model_dump())
+    return write_exam_outputs(workspace, grouped_extraction, client, restart=restart)
 
 
+def write_exam_outputs(workspace: EvaluationWorkspace, grouped_extraction: GroupedExamExtraction,
+                       client, *, restart: bool = False) -> ExitCode:
+    folder = workspace.root
+    labels_list = [item.label for group in grouped_extraction.groups
+                   for item in group if isinstance(item, QuestionItem)]
+    text_dir, sol_dir, text2_dir, sol2_dir, persp_dir = [
+        folder / name for name in ('Text', 'Sol', 'Text2', 'Sol2', 'Persp')]
+    reset_marker = folder / 'Cache' / 'statement_reset_outputs.json'
+    if restart or reset_marker.exists():
+        archive = folder / 'Cache' / 'statement_history' / uuid4().hex
+        for directory in (text_dir, sol_dir, text2_dir, sol2_dir, persp_dir):
+            if directory.exists():
+                archive.mkdir(parents=True, exist_ok=True)
+                directory.replace(archive / directory.name)
+        reset_marker.unlink(missing_ok=True)
+    for directory in (text_dir, sol_dir, text2_dir, sol2_dir, persp_dir):
+        directory.mkdir(parents=True, exist_ok=True)
+    atomic_write_text(workspace.labels_file, ''.join(label + '\n' for label in labels_list))
+    atomic_write_text(workspace.label_groups_file, ''.join(
+        ', '.join(item.label for item in group if isinstance(item, QuestionItem)) + '\n'
+        for group in grouped_extraction.groups if any(isinstance(item, QuestionItem) for item in group)))
+
+    def save_missing(path: Path, text: str):
+        if not path.is_file() or not path.read_text(encoding='utf-8').strip():
+            atomic_write_text(path, text)
+
+    present = set(present_rubric_labels(folder, labels_list))
+    context_sources = {}
     print("Writing grouped question and solution files...")
     processing_errors = []
 
@@ -855,9 +945,15 @@ def process_exam(
 
         group_context_text = "\n\n---\n\n".join(group_text_parts)
 
-        print(f"Generating rubric (Persp) for group: {', '.join(labels)}...")
+        missing_rubrics = [label for label in labels if label not in present]
         try:
-            rubrics_map = generate_rubrics(client, group_context_text)
+            rubrics_map = {}
+            if missing_rubrics:
+                print(f"Generating missing rubric (Persp): {', '.join(missing_rubrics)}...")
+                rubrics_map = generate_rubrics(
+                    client, group_context_text + '\nProduis des barèmes UNIQUEMENT pour : '
+                    + ', '.join(missing_rubrics), workspace=workspace, expected_labels=missing_rubrics,
+                )
         except Exception as e:  # noqa: BLE001 - remote API boundary
             print(f"Error generating rubric for group {labels[0]}: {e}")
             processing_errors.append(str(e))
@@ -887,21 +983,15 @@ def process_exam(
                 text_content_lines.append(f"{item.label} :")
                 text_content_lines.append(tabulated)
 
-                # Write individual Sol file (remains unchanged)
-                safe_label = item.label.replace("/", "_")
-                with open(sol_dir / safe_label, "w", encoding="utf-8") as f_sol:
-                    f_sol.write(f"{item.label}\n{item.solution_content}")
-
-                with open(text2_dir / f"{safe_label}.tex", "w", encoding="utf-8") as f_t2:
-                    f_t2.write(f"\\textbf{{{item.label}}} {item.question_content}")
-
-                with open(sol2_dir / f"{safe_label}.tex", "w", encoding="utf-8") as f_s2:
-                    f_s2.write(f"\\textbf{{{item.label}}} {item.solution_content}")
+                safe_label = item.label.replace('/', '_')
+                save_missing(sol_dir / safe_label, f'{item.label}\n{item.solution_content}')
+                save_missing(text2_dir / f'{safe_label}.tex', f'\\textbf{{{item.label}}} {item.question_content}')
+                save_missing(sol2_dir / f'{safe_label}.tex', f'\\textbf{{{item.label}}} {item.solution_content}')
 
                 # --- Écriture du Barème (Persp) ---
-                rubric_text = rubrics_map.get(item.label, "")
-                with open(persp_dir / safe_label, "w", encoding="utf-8") as f_persp:
-                    f_persp.write(f"{item.label}\n{rubric_text}")
+                if item.label in rubrics_map:
+                    atomic_write_text(persp_dir / safe_label,
+                                      f"{item.label}\n{rubrics_map[item.label]}")
 
             elif isinstance(item, ContextItem):
                 raw_ctx = item.content.strip()
@@ -915,15 +1005,14 @@ def process_exam(
                 ctxt_filename = f"CTXT {safe_first} -> {safe_last}.tex"
                 ctxt_path = text2_dir / ctxt_filename
 
-                # If file exists, prepend some spacing before appending
-                prefix = "\n\n" if ctxt_path.exists() else ""
+                context_sources.setdefault(ctxt_path, [])
+                if item.content not in context_sources[ctxt_path]:
+                    context_sources[ctxt_path].append(item.content)
 
-                with open(ctxt_path, "a", encoding="utf-8") as f_c2:
-                    f_c2.write(prefix + item.content)
+        save_missing(text_dir / safe_group_filename, '\n'.join(text_content_lines))
 
-        # 4. Write the grouped Text file
-        with open(text_dir / safe_group_filename, "w", encoding="utf-8") as f_text:
-            f_text.write("\n".join(text_content_lines))
+    for path, contents in context_sources.items():
+        save_missing(path, '\n\n'.join(contents))
 
     print(f"Wrote {len(grouped_extraction.groups)} groups; checking PDF compilation...")
     # ==========================================
@@ -934,9 +1023,16 @@ def process_exam(
     def compile_worker(tex_file: Path) -> str | None:
         """Helper to read content and call the utility function."""
         try:
-            content = tex_file.read_text(encoding="utf-8")
-            pdf_path = tex_file.with_suffix(".pdf")
+            pdf_path = tex_file.with_suffix('.pdf')
+            status_path = folder / 'Cache' / 'statement_pdf_status' / tex_file.parent.name / (tex_file.stem + '.json')
+            failed_before = status_path.exists() and not json.loads(status_path.read_text()).get('complete')
+            if (pdf_path.is_file() and pdf_path.stat().st_size > 0
+                    and pdf_path.stat().st_mtime_ns >= tex_file.stat().st_mtime_ns and not failed_before):
+                return None
+            atomic_write_json(status_path, {'complete': False})
+            content = tex_file.read_text(encoding='utf-8')
             compile_to_pdf(content, pdf_path, strict=True)
+            atomic_write_json(status_path, {'complete': True})
         except Exception as e:  # noqa: BLE001 - compiler worker boundary
             return f"Error compiling {tex_file.name}: {e}"
         return None
@@ -947,19 +1043,8 @@ def process_exam(
             error for error in executor.map(compile_worker, all_tex_files) if error
         ]
     for error in compile_errors:
-        print(error)
+        report_error(folder, 'PDF compilation', error)
     processing_errors.extend(compile_errors)
-    atomic_write_text(
-        workspace.labels_file,
-        "".join(f"{label}\n" for label in labels_list),
-    )
-    atomic_write_text(
-        workspace.label_groups_file,
-        "".join(", ".join(item.label for item in group if isinstance(item, QuestionItem)) + "\n"
-                for group in grouped_extraction.groups
-                if any(isinstance(item, QuestionItem) for item in group)),
-    )
-
     return ExitCode.PARTIAL if processing_errors else ExitCode.SUCCESS
 
 
@@ -969,26 +1054,31 @@ def build_parser() -> argparse.ArgumentParser:
     actions.add_argument("--groups-only", action="store_true",
                          help="Regroup existing questions with Gemini; update only label_groups")
     actions.add_argument("--persp-only", action="store_true",
-                         help="Replace only Persp with Gemini rubrics for existing groups")
+                         help="Generate only missing Persp rubrics for existing groups")
+    parser.add_argument('--delete-persp', action='store_true',
+                        help='Delete current Persp before generation (requires --persp-only)')
     parser.add_argument(
         "--restart",
         action="store_true",
-        help="Ignore cached Gemini extraction responses",
+        help="Start over; archive previous extraction checkpoints, edits and generated outputs",
     )
     return parser
 
 
 def main(argv: Sequence[str] | None = None) -> int:
     parser = build_parser()
+    def run(args):
+        if args.delete_persp and not args.persp_only:
+            raise CliError('--delete-persp requires --persp-only', ExitCode.INVALID_ARGUMENTS)
+        if args.groups_only or args.persp_only:
+            kwargs = {'delete_persp': True} if args.delete_persp else {}
+            return refine_existing(workspace_from_args(args),
+                                   'groups' if args.groups_only else 'persp', **kwargs)
+        return process_exam(workspace_from_args(args), restart=args.restart)
     return execute(
         parser,
         argv,
-        lambda args: refine_existing(workspace_from_args(args),
-                                     "groups" if args.groups_only else "persp")
-        if args.groups_only or args.persp_only else process_exam(
-            workspace_from_args(args),
-            restart=args.restart,
-        ),
+        run,
     )
 
 

copienator/rubrics.py

@@ -0,0 +1,21 @@
+"""Shared inventory of usable, individual Persp files."""
+from pathlib import Path
+
+
+def present_rubric_labels(root: Path, labels: list[str]) -> list[str]:
+    present = []
+    for label in labels:
+        path = root / 'Persp' / label.replace('/', '_')
+        if not path.is_file():
+            continue
+        try:
+            content = path.read_text(encoding='utf-8').strip()
+        except UnicodeError:
+            continue
+        lines = content.split('\n', 1)
+        body = lines[1].strip() if lines[0].strip() == label and len(lines) > 1 else content
+        if body and body != label and not any(
+            (ord(char) < 32 and char != '\n') or ord(char) == 127 for char in content
+        ):
+            present.append(label)
+    return present

copienator/statement_checkpoints.py

@@ -0,0 +1,154 @@
+"""Extraction checkpoints assume unchanged exam sources until an explicit restart."""
+from __future__ import annotations
+
+import json
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+from uuid import uuid4
+
+from copienator import CliError, atomic_write_json, atomic_write_text
+from copienator.latex_json import repair_latex_json
+
+
+def archive_response(root: Path, stage: str, raw: str | None) -> Path:
+    stamp = datetime.now(UTC).strftime('%Y%m%dT%H%M%S%fZ')
+    path = root / 'Cache' / 'statement_responses' / f'{stamp}-{stage}-{uuid4().hex}.json'
+    atomic_write_text(path, raw or '')
+    return path
+
+
+def report_error(root: Path, stage: str, error, raw=None, response_path=None) -> None:
+    diagnostic = f'{stage}: {error}\n'
+    if raw is not None:
+        diagnostic += f'Raw response (escaped): {raw!r}\n'
+    if response_path:
+        diagnostic += f'Raw response saved to: {response_path}\n'
+    print(diagnostic, end='', flush=True)
+    try:
+        path = root / 'Cache' / 'statement_errors.log'
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open('a', encoding='utf-8') as log:
+            log.write(f'[{datetime.now(UTC).isoformat()}]\n{diagnostic}\n')
+    except OSError as log_error:
+        print(f'Could not save extraction error log: {log_error}', flush=True)
+
+
+def validate_text(text: str) -> None:
+    if not text.strip():
+        raise ValueError('Empty extracted content')
+    bad = [(i, ord(c)) for i, c in enumerate(text)
+           if (ord(c) < 32 and c not in '\n\t') or ord(c) == 127]
+    # Tabs can be legitimate indentation in code, but in math they often
+    # indicate an unknown misescaped TeX command beginning with t.
+    for match in re.finditer(r'(\$\$.*?\$\$|\$.*?\$|\\\(.*?\\\)|\\\[.*?\\\])', text, re.DOTALL):
+        bad.extend((match.start() + i, ord(c)) for i, c in enumerate(match.group()) if c == '\t')
+    if bad:
+        positions = ', '.join(f'U+{code:04X} at offset {i}' for i, code in bad)
+        raise ValueError(f'Control characters: {positions}; decoded content: {text!r}')
+
+
+def extract_stage(root: Path, stage: str, item_model, content_field: str, request,
+                  *, labels: list[str] | None = None) -> list:
+    """Keep valid items; request only pending known labels on the next run.
+
+    Questions are re-requested as a complete ordered inventory because there
+    is no independent list of question labels before extraction. Contexts may
+    legitimately be empty; an empty questions/solutions response may not.
+    """
+    checkpoint = root / 'Cache' / f'statement_{stage}.json'
+    state = json.loads(checkpoint.read_text()) if checkpoint.exists() else {}
+    accepted = state.get(stage, [])
+    inventory = state.get('inventory', [item['label'] for item in accepted] if stage == 'questions' else [])
+    if state.get('complete'):
+        return [item_model.model_validate(item) for item in accepted]
+    pending = state.get('pending')
+    if stage == 'solutions':
+        known = {item['label'] for item in accepted}
+        pending = [label for label in labels if label not in known]
+    legacy = root / 'Cache' / f'gemini_{"context" if stage == "contexts" else stage}.json'
+    raw = None
+    response_path = None
+    try:
+        if not checkpoint.exists() and legacy.exists():
+            raw = legacy.read_text(encoding='utf-8')
+            print(f'Importing legacy {stage} response from {legacy}')
+        else:
+            print(f'Extracting {stage}' + (f' for {pending}' if pending else '') + '...')
+            raw = request(pending).text
+        response_path = archive_response(root, stage, raw)
+        payload = json.loads(repair_latex_json(raw))
+        if not isinstance(payload, dict) or not isinstance(payload.get(stage), list):
+            raise TypeError(f'Response must contain a {stage} list')
+    except Exception as error:
+        # Mark legacy data as attempted too, so an invalid cache is never
+        # replayed indefinitely. Previously accepted items remain untouched.
+        atomic_write_json(checkpoint, {stage: accepted, 'complete': False, 'pending': pending,
+                                       'inventory': inventory})
+        report_error(root, stage, error, raw, response_path)
+        raise CliError(f'{stage} extraction failed; saved progress can be resumed') from error
+
+    errors = []
+    failed_targets = []
+    items = []
+    seen = set()
+    for index, entry in enumerate(payload[stage]):
+        if stage == 'questions' and isinstance(entry, dict):
+            label = entry.get('label')
+            if isinstance(label, str) and label.strip() and label not in inventory:
+                inventory.append(label)
+        try:
+            item = item_model.model_validate(entry)
+            validate_text(getattr(item, content_field))
+            if stage == 'contexts':
+                first, last = item.target_question_label, item.last_question_label
+                if first not in labels or last not in labels or labels.index(first) > labels.index(last):
+                    raise ValueError('Invalid context target or range')
+                if pending and first not in pending:
+                    raise ValueError('Unexpected context target')
+                key = (first, last, item.context_content)
+            else:
+                if not item.label.strip():
+                    raise ValueError('Empty question label')
+                if labels is not None and item.label not in (pending or labels):
+                    raise ValueError('Unexpected solution label')
+                key = item.label
+            if key in seen:
+                raise ValueError('Duplicate extracted item')
+            seen.add(key)
+            items.append(item.model_dump())
+        except (ValueError, TypeError) as error:
+            errors.append(f'Item {index}: {error}; item (escaped): {entry!r}')
+            if isinstance(entry, dict) and entry.get('target_question_label') in (labels or []):
+                failed_targets.append(entry['target_question_label'])
+
+    if stage == 'contexts':
+        # Preserve good contexts even when another context for the same target fails.
+        combined = accepted + [item for item in items if item not in accepted]
+        next_pending = list(dict.fromkeys(failed_targets)) if len(failed_targets) == len(errors) else None
+        absent = [label for label in (pending or []) if not any(
+            item['target_question_label'] == label for item in items)]
+        if absent:
+            errors.append(f'Missing context targets: {absent}')
+            next_pending = list(dict.fromkeys((next_pending or []) + absent))
+    else:
+        previous = {item['label']: item for item in accepted}
+        previous.update({item['label']: item for item in items if item['label'] not in previous})
+        order = [item['label'] for item in items] if stage == 'questions' else labels
+        combined = [previous[label] for label in order if label in previous]
+        combined += [item for label, item in previous.items() if label not in order]
+        missing = [label for label in (labels or []) if label not in previous]
+        if stage == 'questions':
+            missing = [label for label in inventory if label not in seen]
+            if not items:
+                errors.append('No valid questions returned')
+        if missing:
+            errors.append(f'Missing labels: {missing}')
+        next_pending = [label for label in (labels or []) if label not in previous] or None
+    complete = not errors
+    atomic_write_json(checkpoint, {stage: combined, 'complete': complete, 'pending': next_pending,
+                                   'inventory': inventory})
+    if errors:
+        report_error(root, stage, '\n'.join(errors), raw, response_path)
+        raise CliError(f'Incomplete {stage}; {len(combined)} valid items saved. Rerun to resume.')
+    return [item_model.model_validate(item) for item in combined]