Compare commits

...
6 Commits
18 changed files with 652 additions and 109 deletions

No files matched your search

+41 -24
View File
@@ -27,7 +27,7 @@ from copienator.commands import annotating, annotating_with_checks
from copienator.filesystem import staged_directory from copienator.filesystem import staged_directory
from copienator.utils import natural_key from copienator.utils import natural_key
MAX_HEIGHT_PX = 25000 MAX_HEIGHT_PX = 32000
def render_item(item): def render_item(item):
@@ -63,6 +63,11 @@ def save_batch(batch, prefix, group_id, output_root: Path) -> None:
print(f"Generating group PDF: {prefix} G{group_id} ({len(batch)} elements)") print(f"Generating group PDF: {prefix} G{group_id} ({len(batch)} elements)")
max_width = max(item[2].width for item in batch) max_width = max(item[2].width for item in batch)
total_height = sum(item[2].height for item in batch) total_height = sum(item[2].height for item in batch)
if total_height > MAX_HEIGHT_PX:
raise CliError(
f"{prefix} G{group_id} would be {total_height} px high; "
f"the maximum is {MAX_HEIGHT_PX} px."
)
concatenated = Image.new("RGB", (max_width, total_height), "white") concatenated = Image.new("RGB", (max_width, total_height), "white")
draw = ImageDraw.Draw(concatenated) draw = ImageDraw.Draw(concatenated)
checkbox_map: list[dict[str, Any]] = [] checkbox_map: list[dict[str, Any]] = []
@@ -268,32 +273,44 @@ def _existing_group_state(
def split_batches(rendered): def split_batches(rendered):
def split(maximum_height: float): batches = []
batches = [] current = []
current = [] current_height = 0
current_height = 0
previous_student = None def flush() -> None:
for item in rendered: nonlocal current, current_height
student_id = item[0]
image_height = item[2].height
if (
current
and current_height + image_height > maximum_height
and student_id != previous_student
):
batches.append(current)
current = []
current_height = 0
current.append(item)
current_height += image_height
previous_student = student_id
if current: if current:
batches.append(current) batches.append(current)
return batches current = []
current_height = 0
strict = split(MAX_HEIGHT_PX) index = 0
relaxed = split(1.1 * MAX_HEIGHT_PX) while index < len(rendered):
return relaxed if len(relaxed) < len(strict) else strict student_id = rendered[index][0]
end = index + 1
while end < len(rendered) and rendered[end][0] == student_id:
end += 1
student_items = rendered[index:end]
student_height = sum(item[2].height for item in student_items)
if student_height > MAX_HEIGHT_PX:
labels = ", ".join(item[1] for item in student_items)
raise CliError(
f"Copie{student_id} ({labels}) is {student_height} px high; "
f"one student's labels cannot be split across groups and the "
f"maximum group height is {MAX_HEIGHT_PX} px."
)
# Consider the student's complete block before adding its first image.
# Otherwise later images from that same student could overflow the batch.
if current and current_height + student_height > MAX_HEIGHT_PX:
flush()
current.extend(student_items)
current_height += student_height
index = end
flush()
return batches
def _generate_groups( def _generate_groups(
+3 -1
View File
@@ -1026,7 +1026,9 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
manual_path = INPUT_DIR / "manual_resolutions.txt" manual_path = INPUT_DIR / "manual_resolutions.txt"
atomic_write_text( atomic_write_text(
manual_path, manual_path,
"### Use -> x>, -x, ss, sx, xx, xs, c{43}1>, c{43}2x\n" "### Use -> x>, -x, ss, sx, xx, xs, c{43}1>, c{43}2x; "
"delete a line to dismiss its conflict and keep its PDFs unchanged, "
"then run resolve-manual (even if all lines were deleted).\n"
+ "\n".join(unresolved_delayed) + "\n".join(unresolved_delayed)
+ "\n", + "\n",
) )
+3 -2
View File
@@ -195,6 +195,7 @@ class ImageReviewer:
self.root.bind("N", lambda _event: self.on_shift(2 * CROP_SHIFT_STEP)) self.root.bind("N", lambda _event: self.on_shift(2 * CROP_SHIFT_STEP))
self.root.bind("t", lambda _event: self.on_shift(-CROP_SHIFT_STEP)) self.root.bind("t", lambda _event: self.on_shift(-CROP_SHIFT_STEP))
self.root.bind("l", lambda _event: self.on_enlarge(CROP_WIDTH_STEP)) self.root.bind("l", lambda _event: self.on_enlarge(CROP_WIDTH_STEP))
self.root.bind("L", lambda _event: self.on_enlarge(-CROP_WIDTH_STEP))
self.root.bind("1", lambda _event: self.on_set_max_pages(1)) self.root.bind("1", lambda _event: self.on_set_max_pages(1))
Thread(target=self.prefetch_worker, daemon=True).start() Thread(target=self.prefetch_worker, daemon=True).start()
@@ -290,7 +291,7 @@ class ImageReviewer:
f"Files: {schema['number_of_files']} | " f"Files: {schema['number_of_files']} | "
f"Cols: {schema['columns_per_file']}\n" f"Cols: {schema['columns_per_file']}\n"
"Enter: Save and next | s: flag error and skip | n: +50 | N: +100 | t: -50 | " "Enter: Save and next | s: flag error and skip | n: +50 | N: +100 | t: -50 | "
"l: widen by 50 | 1: use full pages" "l: widen by 50 | Shift+l: undo widening by 50 | 1: use full pages"
), ),
fg="black", fg="black",
) )
@@ -305,7 +306,7 @@ class ImageReviewer:
def on_enlarge(self, amount: int) -> None: def on_enlarge(self, amount: int) -> None:
if self.is_processing: if self.is_processing:
return return
self.current_width_offset += amount self.current_width_offset = max(0, self.current_width_offset + amount)
print(f"Applying extra width: {self.current_width_offset}") print(f"Applying extra width: {self.current_width_offset}")
self.trigger_processing(self.files[self.index], self.current_shift) self.trigger_processing(self.files[self.index], self.current_shift)
+34 -6
View File
@@ -22,6 +22,7 @@ from copienator import (
workspace_from_args, workspace_from_args,
) )
from copienator.platform import validate_windows_labels from copienator.platform import validate_windows_labels
from copienator.latex_json import repair_latex_json
from copienator.filesystem import staged_directory from copienator.filesystem import staged_directory
from copienator.utils import compile_to_pdf from copienator.utils import compile_to_pdf
@@ -69,7 +70,7 @@ class ExamContext(BaseModel):
# --- Modèles pour la Requête 4 (Barèmes) --- # --- Modèles pour la Requête 4 (Barèmes) ---
class RubricItem(BaseModel): class RubricItem(BaseModel):
label: str = Field(description="Label exact de la question, à conserver sans traduction.") label: str = Field(description="Label exact de la question, à conserver sans traduction.")
rubric_content: str = Field(description="Barème détaillé sur 4 points : toutes les consignes, explications et justifications doivent être rédigées en français.") rubric_content: str = Field(description="Barème détaillé sur 4 points, en français, avec formules LaTeX. Échapper chaque antislash LaTeX conformément à la syntaxe JSON.")
class GroupRubrics(BaseModel): class GroupRubrics(BaseModel):
rubrics: list[RubricItem] rubrics: list[RubricItem]
@@ -101,6 +102,27 @@ question fournie, cette liste contient un objet avec son `label` exact et
son barème en français dans `rubric_content`. son barème en français dans `rubric_content`.
""" """
LATEX_JSON_INSTRUCTION = r"""
IMPORTANT — Échappement JSON du LaTeX :
Dans les chaînes JSON, chaque antislash LaTeX doit être doublé.
Exemple d’échappement dans un barème (adapte les clés au schéma demandé) :
{"rubrics":[{"label":"1)","rubric_content":"1 point : $\\frac{1}{2}$ et $f \\circ g$.\n1 point : $x \\rightarrow 0$ et $x \\not=0$."}]}
Ne confonds pas les commandes LaTeX avec les échappements JSON :
\\frac, \\rightarrow, \\not, \\theta et \\begin sont des commandes LaTeX ;
\n est réservé à un véritable saut de ligne du texte.
"""
def validate_rubric_content(content: str) -> str:
"""Catch decoded control escapes; ordinary JSON newlines remain valid.
A misescaped LaTeX command starting with n is indistinguishable from
an intentional newline. Do not guess or rewrite those after decoding.
"""
if any((ord(char) < 32 and char != "\n") or ord(char) == 127 for char in content):
raise ValueError("Rubric contains control characters, likely from unescaped LaTeX")
return content
# --- Modèle fusionné (pour le reste du script) --- # --- Modèle fusionné (pour le reste du script) ---
class QuestionItem(BaseModel): class QuestionItem(BaseModel):
label: str label: str
@@ -178,6 +200,12 @@ traduire, et conserve le LaTeX ainsi que les labels exacts.
Renvoie le résultat dans la liste `contexts` de l'objet JSON attendu. Renvoie le résultat dans la liste `contexts` de l'objet JSON attendu.
""" """
PROMPT_1 += LATEX_JSON_INSTRUCTION
PROMPT_2 += LATEX_JSON_INSTRUCTION
PROMPT_3 += LATEX_JSON_INSTRUCTION
PROMPT_4 += LATEX_JSON_INSTRUCTION
def find_file(folder: Path, base_name: str) -> Path | None: def find_file(folder: Path, base_name: str) -> Path | None:
for ext in [".org", ".tex"]: for ext in [".org", ".tex"]:
path = folder / f"{base_name}{ext}" path = folder / f"{base_name}{ext}"
@@ -202,10 +230,10 @@ def generate_rubrics(client, group_context_text: str) -> dict[str, str]:
response_json_schema=GroupRubrics.model_json_schema(), response_json_schema=GroupRubrics.model_json_schema(),
), ),
) )
rubrics = GroupRubrics.model_validate_json(response.text).rubrics rubrics = GroupRubrics.model_validate_json(repair_latex_json(response.text)).rubrics
if len({item.label for item in rubrics}) != len(rubrics): if len({item.label for item in rubrics}) != len(rubrics):
raise ValueError("Gemini returned duplicate rubric labels") raise ValueError("Gemini returned duplicate rubric labels")
return {item.label: item.rubric_content for item in rubrics} return {item.label: validate_rubric_content(item.rubric_content) for item in rubrics}
def validate_groups(groups: list[list[str]], labels: list[str]) -> None: def validate_groups(groups: list[list[str]], labels: list[str]) -> None:
@@ -364,7 +392,7 @@ def process_exam(
print("Saving questions to cache...") print("Saving questions to cache...")
atomic_write_text(cache_q_file, response_q_text) atomic_write_text(cache_q_file, response_q_text)
questions_data = ExamQuestions.model_validate_json(response_q_text) questions_data = ExamQuestions.model_validate_json(repair_latex_json(response_q_text))
# ========================================== # ==========================================
# REQUÊTE 2 : Extraction des Corrections # REQUÊTE 2 : Extraction des Corrections
@@ -403,7 +431,7 @@ def process_exam(
print("Saving solutions to cache...") print("Saving solutions to cache...")
atomic_write_text(cache_s_file, response_s_text) atomic_write_text(cache_s_file, response_s_text)
solutions_data = ExamSolutions.model_validate_json(response_s_text) solutions_data = ExamSolutions.model_validate_json(repair_latex_json(response_s_text))
# ========================================== # ==========================================
# REQUÊTE 3 : Extraction du Contexte (Notations, etc.) # REQUÊTE 3 : Extraction du Contexte (Notations, etc.)
@@ -440,7 +468,7 @@ def process_exam(
print("Saving context to cache...") print("Saving context to cache...")
atomic_write_text(cache_c_file, response_c_text) atomic_write_text(cache_c_file, response_c_text)
context_data = ExamContext.model_validate_json(response_c_text) context_data = ExamContext.model_validate_json(repair_latex_json(response_c_text))
# ========================================== # ==========================================
# FUSION des trois résultats # FUSION des trois résultats
+67 -11
View File
@@ -4,9 +4,10 @@ import argparse
import re import re
import time import time
import typing import typing
from threading import Lock
from collections import defaultdict from collections import defaultdict
from collections.abc import Callable, Sequence from collections.abc import Callable, Sequence
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor, wait
from pathlib import Path from pathlib import Path
from google import genai from google import genai
@@ -271,6 +272,10 @@ def generate_request(
] ]
generate_content_config = types.GenerateContentConfig( generate_content_config = types.GenerateContentConfig(
http_options=types.HttpOptions(
timeout=120_000,
retry_options=types.HttpRetryOptions(attempts=1),
),
automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True), automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
temperature=1.0, temperature=1.0,
top_p=0.95, top_p=0.95,
@@ -282,9 +287,17 @@ def generate_request(
return (contents, generate_content_config) return (contents, generate_content_config)
TARGET_INTERVAL = 3.5 TARGET_INTERVAL = 3.5
MAX_ATTEMPTS = 3
PROGRESS_INTERVAL = 30
Sleep = Callable[[float], None] Sleep = Callable[[float], None]
class IncompleteCopyError(Exception):
def __init__(self, images: list[Path]):
self.images = images
super().__init__(f"Label detection incomplete for {images[0].name}")
def selected_images( def selected_images(
workspace: EvaluationWorkspace, workspace: EvaluationWorkspace,
targets: list[Path], targets: list[Path],
@@ -395,11 +408,12 @@ def process_copy_group(
overwrite: bool, overwrite: bool,
sleep: Sleep = time.sleep, sleep: Sleep = time.sleep,
target_interval: float = TARGET_INTERVAL, target_interval: float = TARGET_INTERVAL,
progress: Callable[[str, str, float], None] | None = None,
) -> int: ) -> int:
"""Process one student's image parts sequentially to preserve context.""" """Process one student's image parts sequentially to preserve context."""
accumulated_labels: list[str] = [] accumulated_labels: list[str] = []
generated = 0 generated = 0
for image_file in files: for image_index, image_file in enumerate(files):
started = time.monotonic() started = time.monotonic()
output_json = workspace.copies_dir / f"{image_file.stem}.json" output_json = workspace.copies_dir / f"{image_file.stem}.json"
if output_json.exists() and not overwrite: if output_json.exists() and not overwrite:
@@ -414,9 +428,13 @@ def process_copy_group(
attempt = 0 attempt = 0
label_retry_count = 0 label_retry_count = 0
wrong_labels: list[str] = [] wrong_labels: list[str] = []
while True: while attempt < MAX_ATTEMPTS:
if progress:
progress(group_key, f"{image_file.name}, attempt {attempt + 1}/{MAX_ATTEMPTS}", started)
if attempt > 0: if attempt > 0:
print(f"[{group_key}] Retrying {image_file.name} in {10 * attempt}s...")
sleep(10 * attempt) sleep(10 * attempt)
print(f"[{group_key}] {image_file.name}: attempt {attempt + 1}/{MAX_ATTEMPTS} (120s timeout).")
try: try:
text_context_file, text_context = closest_text_context( text_context_file, text_context = closest_text_context(
workspace, wrong_labels workspace, wrong_labels
@@ -486,15 +504,23 @@ def process_copy_group(
atomic_write_json(output_json, annotation.model_dump()) atomic_write_json(output_json, annotation.model_dump())
accumulated_labels.extend(box.label for box in annotation.list) accumulated_labels.extend(box.label for box in annotation.list)
generated += 1 generated += 1
print(f"[{group_key}] Saved {image_file.name} after {time.monotonic() - started:.0f}s.")
break break
except KeyboardInterrupt: except KeyboardInterrupt:
raise raise
except Exception as exc: # noqa: BLE001 - remote API retry boundary except Exception as exc: # noqa: BLE001 - remote API retry boundary
print( print(
f"Error processing {image_file.name}: {exc}\n" f"Error processing {image_file.name} "
"\tIt will be retried." f"(attempt {attempt + 1}/{MAX_ATTEMPTS}): {type(exc).__name__}: {exc}"
) )
attempt += 1 attempt += 1
else:
# Later pages need the context of this page; defer them until resume.
incomplete = [path for path in files[image_index:]
if overwrite or not (workspace.copies_dir / f"{path.stem}.json").exists()]
print(f"[{group_key}] Failed after {MAX_ATTEMPTS} attempts; "
"deferring remaining pages of this copy until the next run.")
raise IncompleteCopyError(incomplete)
sleep(max(0.0, target_interval - (time.monotonic() - started))) sleep(max(0.0, target_interval - (time.monotonic() - started)))
return generated return generated
@@ -528,8 +554,17 @@ def run(
client = genai.Client(api_key=api_key) client = genai.Client(api_key=api_key)
groups = group_images(images) groups = group_images(images)
active: dict[str, tuple[str, float]] = {}
progress_lock = Lock()
incomplete: list[Path] = []
run_started = time.monotonic()
def report_progress(group: str, description: str, started: float) -> None:
with progress_lock:
active[group] = (description, started)
with ThreadPoolExecutor(max_workers=max_workers) as executor: with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [ futures = {
executor.submit( executor.submit(
process_copy_group, process_copy_group,
workspace, workspace,
@@ -542,12 +577,33 @@ def run(
valid_names=valid_names, valid_names=valid_names,
overwrite=overwrite, overwrite=overwrite,
sleep=sleep, sleep=sleep,
) progress=report_progress,
): group_key
for group_key, files in groups.items() for group_key, files in groups.items()
] }
for future in futures: pending = set(futures)
future.result() while pending:
return ExitCode.PARTIAL if warnings else ExitCode.SUCCESS done, pending = wait(pending, timeout=PROGRESS_INTERVAL)
for future in done:
with progress_lock:
active.pop(futures[future], None)
try:
future.result()
except IncompleteCopyError as exc:
incomplete.extend(exc.images)
if pending:
now = time.monotonic()
with progress_lock:
snapshot = list(active.items())
print(f"Label detection: {now - run_started:.0f}s elapsed; "
f"{len(pending)} copies outstanding ({len(pending) - len(snapshot)} queued).")
for group, (description, started) in snapshot:
print(f" [{group}] {description}: {now - started:.0f}s elapsed for image.")
if incomplete:
print("Incomplete images (rerun label detection to resume):")
for path in sorted(incomplete):
print(f" {path.name}")
return ExitCode.PARTIAL if warnings or incomplete else ExitCode.SUCCESS
def build_parser() -> argparse.ArgumentParser: def build_parser() -> argparse.ArgumentParser:
+3 -3
View File
@@ -187,7 +187,6 @@ class PDFPreviewer:
action_map = { action_map = {
"move_left": self.move_line_left, "move_left": self.move_line_left,
"move_right": self.move_line_right, "move_right": self.move_line_right,
"confirm_next": self.confirm_and_next_page,
"rotate_page": self.rotate_page, "rotate_page": self.rotate_page,
"rotate_all_pages": self.rotate_all_pages, "rotate_all_pages": self.rotate_all_pages,
"rotate_all_files": self.rotate_all_files, "rotate_all_files": self.rotate_all_files,
@@ -209,7 +208,6 @@ class PDFPreviewer:
# self.master.bind("<Left>", self.move_line_left) # self.master.bind("<Left>", self.move_line_left)
# self.master.bind("<Right>", self.move_line_right) # self.master.bind("<Right>", self.move_line_right)
# self.master.bind("<Return>", self.confirm_and_next_page)
# self.master.bind("c", self.rotate_page) # self.master.bind("c", self.rotate_page)
# self.master.bind("C", self.rotate_all_pages) # self.master.bind("C", self.rotate_all_pages)
# self.master.bind(",", self.rotate_all_files) # self.master.bind(",", self.rotate_all_files)
@@ -260,7 +258,9 @@ class PDFPreviewer:
return return
page = self.doc.load_page(self.current_page_index) page = self.doc.load_page(self.current_page_index)
self.page_label.config(text=f"Page {self.current_page_index + 1} of {len(self.doc)}") self.page_label.config(
text=f"{self.pdf_path.name} — Page {self.current_page_index + 1} of {len(self.doc)}"
)
# --- Calculate Scaling --- # --- Calculate Scaling ---
canvas_width = self.canvas.winfo_width() canvas_width = self.canvas.winfo_width()
+26 -9
View File
@@ -132,6 +132,18 @@ def set_suffix_and_clean_error(
result["error"] = error result["error"] = error
def pending_manual_conflicts(results: dict[str, Any]) -> set[tuple[str, str, str]]:
"""Return the copy/source/target conflicts listed by correction's manual file."""
return {
(item["id"], label, target)
for label, batches in results.items()
for batch in batches
for item in batch
for kind, target in item.get("result", {}).get("delayed", [])
if kind in {"wrong-label", "add-label"}
}
def get_actual_pdf(copies_dir: Path, copy_id: str, label: str) -> Path: def get_actual_pdf(copies_dir: Path, copy_id: str, label: str) -> Path:
base = copies_dir / f"Copie{copy_id}" / f"{label}.pdf" base = copies_dir / f"Copie{copy_id}" / f"{label}.pdf"
for candidate in ( for candidate in (
@@ -157,13 +169,7 @@ def _validate_pdf_inputs(
missing: set[Path] = set() missing: set[Path] = set()
for instruction in instructions: for instruction in instructions:
source = initial_paths[(instruction.copy_id, instruction.old_label)] source = initial_paths[(instruction.copy_id, instruction.old_label)]
destination = initial_paths[(instruction.copy_id, instruction.new_label)] if (instruction.should_merge or instruction.should_copy) and not source.exists():
if instruction.should_merge:
if not source.exists():
missing.add(source)
if not destination.exists():
missing.add(destination)
elif instruction.should_copy and not source.exists():
missing.add(source) missing.add(source)
if missing: if missing:
rendered = ", ".join(str(path) for path in sorted(missing)) rendered = ", ".join(str(path) for path in sorted(missing))
@@ -178,6 +184,10 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
raise CliError("correction.json must contain a JSON object") raise CliError("correction.json must contain a JSON object")
results: dict[str, Any] = loaded results: dict[str, Any] = loaded
instructions = parse_instructions(workspace.manual_resolutions_file) instructions = parse_instructions(workspace.manual_resolutions_file)
retained_conflicts = {
(item.copy_id, item.old_label, item.new_label) for item in instructions
}
dismissed_conflicts = pending_manual_conflicts(results) - retained_conflicts
cut_sources = [(item.copy_id, item.old_label) for item in instructions if item.cut] cut_sources = [(item.copy_id, item.old_label) for item in instructions if item.cut]
if len(set(cut_sources)) != len(cut_sources): if len(set(cut_sources)) != len(cut_sources):
raise CliError("Une seule coupe par label source est autorisée dans une résolution.") raise CliError("Une seule coupe par label source est autorisée dans une résolution.")
@@ -226,9 +236,11 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
try: try:
if instruction.pipe_first: if instruction.pipe_first:
writer.append(source) writer.append(source)
writer.append(destination) if destination.exists():
writer.append(destination)
else: else:
writer.append(destination) if destination.exists():
writer.append(destination)
writer.append(source) writer.append(source)
writer.write(temp_output) writer.write(temp_output)
except Exception: except Exception:
@@ -285,6 +297,9 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
if temporary not in used_temps: if temporary not in used_temps:
temporary.unlink(missing_ok=True) temporary.unlink(missing_ok=True)
for copy_id, label, target in dismissed_conflicts:
# A removed instruction acknowledges the conflict without touching PDFs.
set_suffix_and_clean_error(results, copy_id, label, None, target)
atomic_write_json(workspace.correction_file, results) atomic_write_json(workspace.correction_file, results)
refaire_tasks = [[copy_name, labels] for copy_name, labels in refaire_by_copy.items()] refaire_tasks = [[copy_name, labels] for copy_name, labels in refaire_by_copy.items()]
if refaire_tasks: if refaire_tasks:
@@ -292,6 +307,8 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
workspace.manual_resolutions_file.unlink() workspace.manual_resolutions_file.unlink()
print("Manual resolutions successfully applied.") print("Manual resolutions successfully applied.")
if dismissed_conflicts:
print(f"{len(dismissed_conflicts)} removed conflict(s) dismissed; their PDFs were kept unchanged.")
if refaire_tasks: if refaire_tasks:
print( print(
f"File {workspace.refaire_file.name} generated. Run " f"File {workspace.refaire_file.name} generated. Run "
+29 -10
View File
@@ -19,11 +19,7 @@ from copienator import (
) )
def run(workspace: EvaluationWorkspace, *, client=None) -> ExitCode: def run(workspace: EvaluationWorkspace, *, client=None, overwrite: bool = False) -> ExitCode:
if client is None:
if not config.API_KEY:
raise CliError("GEMINI_API_KEY is not configured")
client = genai.Client(api_key=config.API_KEY)
batches = ( batches = (
( (
"flash", "flash",
@@ -45,16 +41,32 @@ def run(workspace: EvaluationWorkspace, *, client=None) -> ExitCode:
} }
if workspace.batch_jobs_file.is_file(): if workspace.batch_jobs_file.is_file():
previous = read_json(workspace.batch_jobs_file) previous = read_json(workspace.batch_jobs_file)
if isinstance(previous, dict) and isinstance(previous.get("jobs"), dict): if not isinstance(previous, dict) or not isinstance(previous.get("jobs"), dict):
manifest["jobs"] = previous["jobs"] raise CliError(f"Invalid batch manifest: {workspace.batch_jobs_file}")
manifest["jobs"] = previous["jobs"]
started = 0 started = 0
skipped = 0
for tier, file_path, model_id, display_name in batches: for tier, file_path, model_id, display_name in batches:
existing = manifest["jobs"].get(tier)
if tier in manifest["jobs"]:
if (not isinstance(existing, dict)
or not isinstance(existing.get("name"), str)
or not existing["name"].strip()):
raise CliError(f"Invalid batch job in {workspace.batch_jobs_file}: {tier}")
if not overwrite:
print(f"Skipping {tier}: already submitted as {existing['name']}.")
skipped += 1
continue
if not file_path.is_file(): if not file_path.is_file():
print(f"Skipping {model_id}: {file_path.name} does not exist.") print(f"Skipping {model_id}: {file_path.name} does not exist.")
continue continue
if file_path.stat().st_size == 0: if file_path.stat().st_size == 0:
print(f"Skipping {model_id}: {file_path.name} is empty.") print(f"Skipping {model_id}: {file_path.name} is empty.")
continue continue
if client is None:
if not config.API_KEY:
raise CliError("GEMINI_API_KEY is not configured")
client = genai.Client(api_key=config.API_KEY)
print(f"Uploading {file_path.name} for model {model_id}...") print(f"Uploading {file_path.name} for model {model_id}...")
uploaded = client.files.upload( uploaded = client.files.upload(
file=str(file_path), file=str(file_path),
@@ -77,19 +89,26 @@ def run(workspace: EvaluationWorkspace, *, client=None) -> ExitCode:
} }
atomic_write_json(workspace.batch_jobs_file, manifest) atomic_write_json(workspace.batch_jobs_file, manifest)
print(f"Started batch job: {job.name}") print(f"Started batch job: {job.name}")
if not started: if not started and not skipped:
print("No non-empty batch request files were found.") print("No non-empty batch request files were found.")
return ExitCode.PARTIAL return ExitCode.PARTIAL
return ExitCode.SUCCESS return ExitCode.SUCCESS
def build_parser() -> argparse.ArgumentParser: def build_parser() -> argparse.ArgumentParser:
return evaluation_parser("Upload correction JSONL files and start Gemini batches") parser = evaluation_parser("Upload correction JSONL files and start Gemini batches")
parser.add_argument(
"--overwrite", action="store_true",
help="Submit new jobs even for recorded tiers, replacing their saved job IDs",
)
return parser
def main(argv: Sequence[str] | None = None) -> int: def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser() parser = build_parser()
return execute(parser, argv, lambda args: run(workspace_from_args(args))) return execute(parser, argv, lambda args: run(
workspace_from_args(args), overwrite=args.overwrite,
))
if __name__ == "__main__": if __name__ == "__main__":
+9 -7
View File
@@ -177,21 +177,23 @@ def compile_to_pdf(text, output_pdf_path):
text=True, text=True,
check=False) check=False)
generated_pdf = os.path.join(temp_dir, pdf_filename)
pdf_generated = os.path.exists(generated_pdf)
if pdf_generated:
shutil.move(generated_pdf, output_pdf_path)
if result.returncode != 0: if result.returncode != 0:
error_lines = [ error_lines = [
line.strip() for line in result.stdout.splitlines() line.strip() for line in result.stdout.splitlines()
if line.lstrip().startswith("!") if line.lstrip().startswith("!")
] ]
detail = f": {error_lines[0]}" if error_lines else "" detail = f": {error_lines[0]}" if error_lines else ""
pdf_status = "a PDF was generated" if pdf_generated else "no PDF was generated"
print( print(
f"Warning: LaTeX compilation failed for {output_pdf_path} " f"Warning: LaTeX compilation reported errors for {output_pdf_path} "
f"(exit code {result.returncode}){detail}" f"(exit code {result.returncode}; {pdf_status}){detail}"
) )
elif not pdf_generated:
generated_pdf = os.path.join(temp_dir, pdf_filename)
if os.path.exists(generated_pdf):
shutil.move(generated_pdf, output_pdf_path)
else:
print(f"Warning: LaTeX compilation produced no PDF for {output_pdf_path}") print(f"Warning: LaTeX compilation produced no PDF for {output_pdf_path}")
except Exception as e: except Exception as e:
print(f"Compilation error for {output_pdf_path}: {e}") print(f"Compilation error for {output_pdf_path}: {e}")
+76 -21
View File
@@ -10,7 +10,8 @@ from pathlib import Path
from tkinter import filedialog, messagebox, ttk from tkinter import filedialog, messagebox, ttk
from typing import Any from typing import Any
from copienator import CliError, EvaluationWorkspace, ExitCode, atomic_write_json from copienator import CliError, EvaluationWorkspace, ExitCode, atomic_write_json, read_json
from copienator.commands.resolve_manual import pending_manual_conflicts
from copienator.copy_errors import copy_errors from copienator.copy_errors import copy_errors
from copienator.filesystem import staged_files from copienator.filesystem import staged_files
from copienator.platform import ( from copienator.platform import (
@@ -244,7 +245,14 @@ def has_manual_conflicts(path: Path) -> bool:
# If the file cannot be inspected, keep the step visible rather than # If the file cannot be inspected, keep the step visible rather than
# silently claiming that there is nothing to resolve. # silently claiming that there is nothing to resolve.
return True return True
return any(line.strip() and not line.lstrip().startswith("###") for line in lines) if any(line.strip() and not line.lstrip().startswith("###") for line in lines):
return True
# An emptied manual file still needs execution to acknowledge deletions.
try:
results = read_json(path.parent / "correction.json", default={})
return bool(pending_manual_conflicts(results))
except (OSError, ValueError, TypeError, KeyError, AttributeError):
return True
class CopienatorApp(tk.Tk): class CopienatorApp(tk.Tk):
@@ -591,7 +599,10 @@ class CopienatorApp(tk.Tk):
opened = {self.tree.item(item, "text"): self.tree.item(item, "open") for item in self.tree.get_children()} opened = {self.tree.item(item, "text"): self.tree.item(item, "open") for item in self.tree.get_children()}
self.tree.delete(*self.tree.get_children()) self.tree.delete(*self.tree.get_children())
section_items: dict[str, str] = {} section_items: dict[str, str] = {}
show_batches = self._has_batch_correction()
for step in self.steps: for step in self.steps:
if step.skip_for_live_correction and not show_batches:
continue
if step.section not in section_items: if step.section not in section_items:
section_id = f"section:{len(section_items)}" section_id = f"section:{len(section_items)}"
section_items[step.section] = section_id section_items[step.section] = section_id
@@ -607,6 +618,25 @@ class CopienatorApp(tk.Tk):
) )
if selected and self.tree.exists(selected): if selected and self.tree.exists(selected):
self.tree.selection_set(selected) self.tree.selection_set(selected)
elif selected and self.tree.exists("correction"):
self.tree.selection_set("correction")
def _has_batch_correction(self) -> bool:
evaluation = self.state_store.evaluation
if evaluation is None:
return False
correction = self.state_store.step("correction")
if correction.get("batch_started") or correction.get("last_run_variant") in {"batch", "hybrid", "integrate"}:
return True
for name in ("batch_requests_flash.jsonl", "batch_requests_pro.jsonl",
"batch_jobs.json", "batched_correction_result.jsonl"):
path = evaluation / name
try:
if path.is_file() and path.stat().st_size > 0:
return True
except OSError:
continue
return False
def _step_status(self, step: StepDefinition) -> str: def _step_status(self, step: StepDefinition) -> str:
if self.state_store.evaluation: if self.state_store.evaluation:
@@ -721,16 +751,6 @@ class CopienatorApp(tk.Tk):
# trigger the same automatic behavior twice. # trigger the same automatic behavior twice.
self.state_store.update_step(step.id, visited=True) self.state_store.update_step(step.id, visited=True)
if step.skip_for_live_correction:
correction = self.state_store.step("correction")
if correction.get("variant", "live") == "live":
self.after_idle(
lambda step_id=step.id: self._automatic_skip(
step_id, "correction immédiate sélectionnée"
)
)
return
if step.skip_without_manual_conflicts: if step.skip_without_manual_conflicts:
evaluation = self.evaluation evaluation = self.evaluation
conflicts = evaluation / "manual_resolutions.txt" if evaluation else None conflicts = evaluation / "manual_resolutions.txt" if evaluation else None
@@ -1217,7 +1237,9 @@ class CopienatorApp(tk.Tk):
def _progression_ids(self, step_id: str) -> list[str]: def _progression_ids(self, step_id: str) -> list[str]:
redo = step_id.startswith("refaire_") redo = step_id.startswith("refaire_")
return [step.id for step in self.steps if step.id.startswith("refaire_") == redo] show_batches = self._has_batch_correction()
return [step.id for step in self.steps if step.id.startswith("refaire_") == redo
and (not step.skip_for_live_correction or show_batches)]
def _refaire_selection(self) -> list[list]: def _refaire_selection(self) -> list[list]:
evaluation = self.evaluation evaluation = self.evaluation
@@ -1445,6 +1467,10 @@ class CopienatorApp(tk.Tk):
self._save_current_form() self._save_current_form()
run_values = self._values() run_values = self._values()
ordered_ids = self._progression_ids(step.id) ordered_ids = self._progression_ids(step.id)
if step.id == "correction" and variant.id == "integrate":
# Integration consumes the retrieved batches; it does not invalidate them.
ordered_ids = [step_id for step_id in ordered_ids
if not self.step_by_id[step_id].skip_for_live_correction]
self.state_store.invalidate_after(ordered_ids, step.id) self.state_store.invalidate_after(ordered_ids, step.id)
self.state_store.update_step( self.state_store.update_step(
step.id, step.id,
@@ -1473,6 +1499,8 @@ class CopienatorApp(tk.Tk):
self._append_console(f"\n$ {command_display(command)}\n") self._append_console(f"\n$ {command_display(command)}\n")
try: try:
self.runner.start(command, self.repository, environment, log_path) self.runner.start(command, self.repository, environment, log_path)
if step.id == "correction" and variant.id in {"batch", "hybrid"}:
self.state_store.update_step("correction", batch_started=True)
except (OSError, RuntimeError) as exc: except (OSError, RuntimeError) as exc:
self.state_store.update_step(step.id, status="failed") self.state_store.update_step(step.id, status="failed")
self.state_store.add_history( self.state_store.add_history(
@@ -1551,8 +1579,7 @@ class CopienatorApp(tk.Tk):
if self.current_step and self.current_step.optional: if self.current_step and self.current_step.optional:
step_id = self.current_step.id step_id = self.current_step.id
self._mark_step("skipped") self._mark_step("skipped")
if step_id.startswith("refaire_"): self._move_selection_from(step_id, 1)
self._move_selection_from(step_id, 1)
def _poll_runner(self) -> None: def _poll_runner(self) -> None:
while True: while True:
@@ -1615,15 +1642,18 @@ class CopienatorApp(tk.Tk):
self._populate_tree() self._populate_tree()
message = f"{evaluation.name} : tous les résultats batch sont prêts à récupérer." message = f"{evaluation.name} : tous les résultats batch sont prêts à récupérer."
self.info_var.set(message) self.info_var.set(message)
try: self._notify_completion("Copienator — batchs terminés", message)
notify_desktop("Copienator — batchs terminés", message)
except (OSError, RuntimeError) as exc:
self.bell()
self._append_console(f"Notification de bureau indisponible : {exc}\n{message}\n")
if (self.current_step and self.current_step.id == "batch_status" if (self.current_step and self.current_step.id == "batch_status"
and not self.active_step_id and not self.runner.running): and not self.active_step_id and not self.runner.running):
self._move_selection_from("batch_status", 1) self._move_selection_from("batch_status", 1)
def _notify_completion(self, title: str, message: str) -> None:
try:
notify_desktop(title, message)
except (OSError, RuntimeError) as exc:
self.bell()
self._append_console(f"Notification de bureau indisponible : {exc}\n{message}\n")
def _finish_process(self, return_code: int, interrupted: bool) -> None: def _finish_process(self, return_code: int, interrupted: bool) -> None:
step_id = self.active_step_id step_id = self.active_step_id
if not step_id: if not step_id:
@@ -1710,12 +1740,28 @@ class CopienatorApp(tk.Tk):
"sur « Marquer terminée »." "sur « Marquer terminée »."
) )
return return
notified_completions = {
"labels": "détection des labels terminée",
"correction": "étape de correction terminée",
"annotation": "génération des copies annotées terminée",
"read_annotations": "lecture des annotations manuscrites terminée",
}
if step_id in notified_completions and self.state_store.evaluation:
completion = notified_completions[step_id]
outcome = STATUS_LABELS[status].lower()
self._notify_completion(
f"Copienator — {completion} ({outcome})",
f"{self.state_store.evaluation.name} : {completion} — {outcome} "
f"(code {return_code}).",
)
if status == "success": if status == "success":
if step_id == "batch_status" and self.batch_monitor.active: if step_id == "batch_status" and self.batch_monitor.active:
self.batch_monitor.stop() self.batch_monitor.stop()
self._batch_results_ready() self._batch_results_ready()
return return
if step_id == "manual_resolution": if step_id == "fetch_batches":
self.after_idle(self._select_batch_integration)
elif step_id == "manual_resolution":
self.after_idle(self._select_refaire_correction) self.after_idle(self._select_refaire_correction)
else: else:
self.after_idle( self.after_idle(
@@ -1813,6 +1859,15 @@ class CopienatorApp(tk.Tk):
self.tree.selection_set(ids[target]) self.tree.selection_set(ids[target])
self.tree.see(ids[target]) self.tree.see(ids[target])
def _select_batch_integration(self) -> None:
"""Prepare integration after batch results have been retrieved."""
if self.runner.running or self.active_step_id or not self.state_store.evaluation:
return
self._save_current_form()
self.state_store.update_step("correction", variant="integrate")
self.tree.selection_set("correction")
self.tree.see("correction")
def _select_refaire_correction(self) -> None: def _select_refaire_correction(self) -> None:
"""Return from manual conflict resolution to the main redo correction.""" """Return from manual conflict resolution to the main redo correction."""
if self.runner.running or self.active_step_id or not self.state_store.evaluation: if self.runner.running or self.active_step_id or not self.state_store.evaluation:
+9 -5
View File
@@ -29,7 +29,8 @@ c{43}2x : couper la source à 43 %, conserver la fin et remplacer la cible par l
Cut permet de placer la coupe, avec accrochage entre les pages. Entrée ferme l’aperçu et affiche la commande à recopier ci-dessus : aucun fichier n’est modifié. Une séparation entre pages conserve les pages entières. La source complète est archivée en _old ; les deux labels modifiés sont générés en _new et ajoutés à refaire.json. Cut permet de placer la coupe, avec accrochage entre les pages. Entrée ferme l’aperçu et affiche la commande à recopier ci-dessus : aucun fichier n’est modifié. Une séparation entre pages conserve les pages entières. La source complète est archivée en _old ; les deux labels modifiés sont générés en _new et ajoutés à refaire.json.
Pour les fusions : « Source x> Cible| » place la cible avant la source ; « Source x> |Cible » place la source avant la cible. Même règle avec -> et c{…}1>/c{…}2> (pour la partie transférée) ; sans |, la cible vient en premier. Pour les fusions : « Source x> Cible| » place la cible avant la source ; « Source x> |Cible » place la source avant la cible. Même règle avec -> et c{…}1>/c{…}2> (pour la partie transférée) ; sans |, la cible vient en premier.
Vous pouvez changer l’opérateur, déplacer |, corriger les labels ou retirer une instruction. Les lignes vides et celles commençant par ### sont ignorées. Retirer/commenter une ligne ne résout pas son conflit. Si le PDF cible n’existe pas, il est considéré comme vide : la source (ou la partie transférée après une coupe) devient le nouveau PDF cible.
Vous pouvez changer l’opérateur, déplacer |, corriger les labels ou retirer une instruction. Les lignes vides et celles commençant par ### sont ignorées. Retirer/commenter une ligne signifie conserver les PDF sans modification et considérer ce conflit comme résolu après Exécuter. Exécutez cette étape même si vous avez retiré toutes les instructions : ces conflits ne seront plus proposés lors de la reprise avec refaire.json.
Enregistrez dans l’éditeur, puis cliquez sur Recharger et enfin sur Exécuter. Seul le fichier enregistré est appliqué. L’archivage utilise le suffixe _old ; les PDF créés utilisent _new. Après succès, manual_resolutions.txt est supprimé et correction.json est mis à jour. Si des PDF sont créés, refaire.json est généré : relancez la correction avec --refaire.""" Enregistrez dans l’éditeur, puis cliquez sur Recharger et enfin sur Exécuter. Seul le fichier enregistré est appliqué. L’archivage utilise le suffixe _old ; les PDF créés utilisent _new. Après succès, manual_resolutions.txt est supprimé et correction.json est mis à jour. Si des PDF sont créés, refaire.json est généré : relancez la correction avec --refaire."""
@@ -46,7 +47,10 @@ class ManualResolutionPanel(ttk.Frame):
actions, text="Ouvrir dans un éditeur de texte", command=self.open_editor actions, text="Ouvrir dans un éditeur de texte", command=self.open_editor
) )
self.editor_button.pack(side="left") self.editor_button.pack(side="left")
ttk.Button(actions, text="Recharger", command=self.reload).pack(side="left", padx=6) self.reload_button = ttk.Button(
actions, text="Recharger", command=self.reload
)
self.reload_button.pack(side="left", padx=6)
self.cut_result = tk.StringVar() self.cut_result = tk.StringVar()
result = ttk.Frame(self) result = ttk.Frame(self)
result.pack(fill="x", pady=4) result.pack(fill="x", pady=4)
@@ -116,10 +120,10 @@ class ManualResolutionPanel(ttk.Frame):
else: else:
if instructions: if instructions:
instruction = instructions[0] instruction = instructions[0]
for title, label in (("PDF source", instruction.old_label), for title, label in (("Source", instruction.old_label),
("PDF cible", instruction.new_label)): ("Cible", instruction.new_label)):
button = ttk.Button( button = ttk.Button(
self.text, text=title, self.text, text=f"{title} : {label}",
command=lambda label=label, copy_id=instruction.copy_id, root=path.parent: self.open_pdf(root, copy_id, label), command=lambda label=label, copy_id=instruction.copy_id, root=path.parent: self.open_pdf(root, copy_id, label),
) )
self.pdf_buttons.append(button) self.pdf_buttons.append(button)
-1
View File
@@ -44,7 +44,6 @@ MODEL_PRO_ID = "gemini-3.8-flash"
PAGE_SPLITTER_KB = { PAGE_SPLITTER_KB = {
"move_left": "<Left>", "move_left": "<Left>",
"move_right": "<Right>", "move_right": "<Right>",
"confirm_next": "<Return>",
"rotate_page": "r", "rotate_page": "r",
"rotate_all_pages": "R", "rotate_all_pages": "R",
"rotate_all_files": ",", "rotate_all_files": ",",
+149 -2
View File
@@ -133,7 +133,8 @@ class GuiConvenienceTests(unittest.TestCase):
self.assertNotEqual(self.app.current_step.id, "giving_names") self.assertNotEqual(self.app.current_step.id, "giving_names")
def test_batch_status_waits_until_all_jobs_are_ready(self): def test_batch_status_waits_until_all_jobs_are_ready(self):
self.app.state_store.update_step("correction", variant="batch") self.app.state_store.update_step("correction", variant="batch", last_run_variant="batch")
self.app._populate_tree()
self.app.state_store.update_step("batch_status", visited=True) self.app.state_store.update_step("batch_status", visited=True)
self.app.tree.selection_set("batch_status") self.app.tree.selection_set("batch_status")
self.app.update() self.app.update()
@@ -149,8 +150,59 @@ class GuiConvenienceTests(unittest.TestCase):
self.app.update() self.app.update()
self.assertEqual(self.app.current_step.id, "fetch_batches") self.assertEqual(self.app.current_step.id, "fetch_batches")
def test_batch_retrieval_success_selects_integration_without_starting_it(self):
self.app.state_store.update_step("correction", variant="batch", last_run_variant="batch")
self.app._populate_tree()
self.app.tree.selection_set("fetch_batches")
self.app.update()
self.app.active_step_id = "fetch_batches"
with patch.object(self.app.runner, "start") as start:
self.app._finish_process(0, False)
self.app.update()
start.assert_not_called()
self.assertEqual(self.app.current_step.id, "correction")
self.assertEqual(self.app.variant_var.get(), "integrate")
self.assertEqual(self.app.state_store.step("correction")["variant"], "integrate")
self.assertIn("--deal-with-batched", self.app._make_command())
def test_integration_preserves_batch_statuses_but_invalidates_downstream_steps(self):
for variant in ("integrate", "batch"):
with self.subTest(variant=variant):
self.app.active_step_id = None
self.app.current_step = None
self.app.state_store.update_step("correction", variant=variant, batch_started=True)
statuses = {"submit_batches": "success", "batch_status": "detected",
"fetch_batches": "success", "annotation": "success"}
for step_id, status in statuses.items():
self.app.state_store.update_step(step_id, status=status)
self.app._populate_tree()
self.app.tree.selection_set("correction")
self.app.update()
with patch.object(self.app.runner, "start"), \
patch.object(self.app, "_missing_requirements", return_value=[]):
self.app._run_current_step()
self.assertEqual(self.app.active_step_id, "correction")
for step_id, status in statuses.items():
expected = status if variant == "integrate" and step_id != "annotation" else "stale"
self.assertEqual(self.app.state_store.step(step_id)["status"], expected)
self.app.active_step_id = None
def test_unsuccessful_batch_retrieval_stays_on_retrieval(self):
self.app.state_store.update_step("correction", variant="batch", last_run_variant="batch")
self.app._populate_tree()
self.app.tree.selection_set("fetch_batches")
self.app.update()
for code, interrupted in ((4, False), (1, False), (0, True)):
with self.subTest(code=code, interrupted=interrupted):
self.app.active_step_id = "fetch_batches"
self.app._finish_process(code, interrupted)
self.app.update()
self.assertEqual(self.app.current_step.id, "fetch_batches")
self.assertEqual(self.app.state_store.step("correction")["variant"], "batch")
def test_batch_watch_buttons_use_loaded_evaluation_and_stop_on_reload(self): def test_batch_watch_buttons_use_loaded_evaluation_and_stop_on_reload(self):
self.app.state_store.update_step("correction", variant="batch") self.app.state_store.update_step("correction", variant="batch", last_run_variant="batch")
self.app._populate_tree()
self.app.tree.selection_set("batch_status") self.app.tree.selection_set("batch_status")
self.app.update() self.app.update()
with patch.object(self.app.batch_monitor, "start") as start: with patch.object(self.app.batch_monitor, "start") as start:
@@ -175,6 +227,97 @@ class GuiConvenienceTests(unittest.TestCase):
self.assertEqual(self.app.current_step.id, "inputs") self.assertEqual(self.app.current_step.id, "inputs")
self.assertEqual(self.app.state_store.step("batch_status")["status"], "success") self.assertEqual(self.app.state_store.step("batch_status")["status"], "success")
def test_batch_steps_hidden_until_a_correction_is_started(self):
batch_ids = ("submit_batches", "batch_status", "fetch_batches")
self.app.state_store.update_step("correction", variant="batch")
self.app._populate_tree()
for step_id in batch_ids:
self.assertFalse(self.app.tree.exists(step_id))
self.assertNotIn(step_id, self.app._progression_ids("correction"))
self.app.state_store.update_step("correction", last_run_variant="hybrid", status="running")
self.app._populate_tree()
for step_id in batch_ids:
self.assertTrue(self.app.tree.exists(step_id))
self.assertIn(step_id, self.app._progression_ids("correction"))
def test_existing_batch_artifacts_reveal_steps_even_in_live_mode(self):
for filename in ("batch_requests_flash.jsonl", "batch_requests_pro.jsonl",
"batch_jobs.json", "batched_correction_result.jsonl"):
with self.subTest(filename=filename):
path = self.evaluation / filename
path.write_text("")
self.app._populate_tree()
self.assertFalse(self.app.tree.exists("batch_status"))
path.write_text('{"jobs":{"flash":{"name":"batches/1"}}}\n')
self.app._populate_tree()
self.assertTrue(self.app.tree.exists("batch_status"))
self.app._handle_first_visit(self.app.step_by_id["batch_status"])
self.app.update()
self.assertNotEqual(self.app.state_store.step("batch_status").get("status"), "skipped")
path.unlink()
def test_started_batches_remain_visible_after_mode_change_and_reload(self):
self.app.state_store.update_step("correction", batch_started=True, last_run_variant="live")
self.app.state_store.load(self.evaluation)
self.app._populate_tree()
self.assertTrue(self.app.tree.exists("batch_status"))
fresh = self.repository / "Fresh"
fresh.mkdir()
self.app.state_store.load(fresh)
self.app._populate_tree()
self.assertFalse(self.app.tree.exists("batch_status"))
def test_navigation_skips_hidden_batch_steps(self):
with patch.object(self.app.tree, "selection_set") as select:
self.app._move_selection_from("correction", 1)
select.assert_called_once_with("manual_resolution")
def test_long_running_steps_notify_once_on_success(self):
expected_text = {
"labels": "labels",
"correction": "correction",
"annotation": "copies annotées",
"read_annotations": "annotations manuscrites",
}
for step_id, completion_text in expected_text.items():
with self.subTest(step=step_id), patch("copienator_gui.app.notify_desktop") as notify:
self.app.active_step_id = step_id
self.app._finish_process(0, False)
self.app._finish_process(0, False)
notify.assert_called_once()
self.assertIn(self.evaluation.name, notify.call_args.args[1])
self.assertIn(completion_text, notify.call_args.args[1])
def test_incomplete_failed_and_interrupted_steps_notify_the_outcome_once(self):
for step_id in ("labels", "correction", "annotation", "read_annotations"):
for code, interrupted, outcome in (
(4, False, "partielle"), (1, False, "échouée"),
(0, True, "interrompue"), (130, False, "interrompue"),
):
with self.subTest(step=step_id, code=code, interrupted=interrupted), \
patch("copienator_gui.app.notify_desktop") as notify:
self.app.active_step_id = step_id
self.app._finish_process(code, interrupted)
self.app._finish_process(code, interrupted)
notify.assert_called_once()
title, message = notify.call_args.args
self.assertIn(outcome, title)
self.assertIn(outcome, message)
self.assertIn(self.evaluation.name, message)
self.assertIn(f"code {code}", message)
self.assertNotIn("réussie", message)
def test_notification_failure_keeps_success_and_uses_existing_fallback(self):
self.app.active_step_id = "labels"
with patch("copienator_gui.app.notify_desktop", side_effect=RuntimeError("unavailable")), \
patch.object(self.app, "bell") as bell, \
patch.object(self.app, "_append_console") as console:
self.app._finish_process(0, False)
bell.assert_called_once()
self.assertTrue(any("Notification de bureau indisponible" in call.args[0]
for call in console.call_args_list))
self.assertEqual(self.app.state_store.step("labels")["status"], "success")
def test_optional_blank_crop_can_target_a_copy_or_be_skipped(self): def test_optional_blank_crop_can_target_a_copy_or_be_skipped(self):
copies = self.evaluation/"Copies" copies = self.evaluation/"Copies"
copies.mkdir() copies.mkdir()
@@ -190,7 +333,9 @@ class GuiConvenienceTests(unittest.TestCase):
command = self.app._make_command() command = self.app._make_command()
self.assertEqual(command[-4:], ["crop-margins", str(source), "--workers", "5"]) self.assertEqual(command[-4:], ["crop-margins", str(source), "--workers", "5"])
self.app._skip_step() self.app._skip_step()
self.app.update()
self.assertEqual(self.app.state_store.step("crop_blank_margins")["status"], "skipped") self.assertEqual(self.app.state_store.step("crop_blank_margins")["status"], "skipped")
self.assertNotEqual(self.app.current_step.id, "crop_blank_margins")
def test_optional_answer_bottom_crop_uses_parallel_workers_and_can_be_skipped(self): def test_optional_answer_bottom_crop_uses_parallel_workers_and_can_be_skipped(self):
answers = self.evaluation / "Copies" / "Copie01" answers = self.evaluation / "Copies" / "Copie01"
@@ -207,9 +352,11 @@ class GuiConvenienceTests(unittest.TestCase):
["crop-answer-bottoms", self.app._evaluation_arg(), "--workers", "5"], ["crop-answer-bottoms", self.app._evaluation_arg(), "--workers", "5"],
) )
self.app._skip_step() self.app._skip_step()
self.app.update()
self.assertEqual( self.assertEqual(
self.app.state_store.step("crop_exercise_bottoms")["status"], "skipped" self.app.state_store.step("crop_exercise_bottoms")["status"], "skipped"
) )
self.assertNotEqual(self.app.current_step.id, "crop_exercise_bottoms")
def test_redo_targets_selected_copy_and_copies_runnable_command(self): def test_redo_targets_selected_copy_and_copies_runnable_command(self):
for folder in ("Copies", "Copies Originales"): for folder in ("Copies", "Copies Originales"):
+28 -4
View File
@@ -19,6 +19,7 @@ from PIL import Image
from pypdf import PdfReader, PdfWriter from pypdf import PdfReader, PdfWriter
from copienator import ( from copienator import (
CliError,
EvaluationWorkspace, EvaluationWorkspace,
WorkspaceNotFoundError, WorkspaceNotFoundError,
WorkspaceValidationError, WorkspaceValidationError,
@@ -2058,7 +2059,23 @@ class StandardCliTests(unittest.TestCase):
self.assertEqual(groups, [["Ex 1"], ["Ex 2"]]) self.assertEqual(groups, [["Ex 1"], ["Ex 2"]])
editor.assert_not_called() editor.assert_not_called()
def test_grouped_batching_does_not_split_one_student(self) -> None: def test_grouped_batching_looks_ahead_to_keep_below_the_hard_limit(self) -> None:
module = self.modules["annotating_by_label"]
large = Image.new("RGB", (10, 80), "white")
small = Image.new("RGB", (10, 15), "white")
rendered = [
("01", "Ex 1", large, 0, []),
("02", "Ex 1", small, 0, []),
("02", "Ex 2", small, 0, []),
]
with patch.object(module, "MAX_HEIGHT_PX", 100):
batches = module.split_batches(rendered)
self.assertEqual([len(batch) for batch in batches], [1, 2])
self.assertTrue(
all(sum(item[2].height for item in batch) <= 100 for batch in batches)
)
def test_grouped_batching_never_splits_one_student(self) -> None:
module = self.modules["annotating_by_label"] module = self.modules["annotating_by_label"]
image = Image.new("RGB", (10, 60), "white") image = Image.new("RGB", (10, 60), "white")
rendered = [ rendered = [
@@ -2066,9 +2083,16 @@ class StandardCliTests(unittest.TestCase):
("01", "Ex 2", image, 0, []), ("01", "Ex 2", image, 0, []),
("02", "Ex 1", image, 0, []), ("02", "Ex 1", image, 0, []),
] ]
with patch.object(module, "MAX_HEIGHT_PX", 100): with patch.object(module, "MAX_HEIGHT_PX", 100), self.assertRaises(
batches = module.split_batches(rendered) CliError
self.assertEqual([len(batch) for batch in batches], [2, 1]) ) as raised:
module.split_batches(rendered)
self.assertIn("Copie01", str(raised.exception))
self.assertIn("cannot be split", str(raised.exception))
def test_grouped_annotation_height_limit_is_32000_pixels(self) -> None:
module = self.modules["annotating_by_label"]
self.assertEqual(module.MAX_HEIGHT_PX, 32000)
class WorkflowTests(unittest.TestCase): class WorkflowTests(unittest.TestCase):
+19 -1
View File
@@ -38,13 +38,31 @@ class ManualResolutionPanelTests(unittest.TestCase):
self.assertEqual(panel.text.get("1.0", "end-1c"), content) self.assertEqual(panel.text.get("1.0", "end-1c"), content)
self.assertEqual(len(panel.pdf_buttons), 4) self.assertEqual(len(panel.pdf_buttons), 4)
self.assertEqual([button.cget("text") for button in panel.pdf_buttons], self.assertEqual([button.cget("text") for button in panel.pdf_buttons],
["PDF source", "PDF cible"] * 2) ["Source : Ex 1", "Cible : Ex 2",
"Source : Ex 3", "Cible : Ex 4"])
with patch("copienator_gui.manual_resolution.open_path") as opened: with patch("copienator_gui.manual_resolution.open_path") as opened:
panel.editor_button.invoke() panel.editor_button.invoke()
for button in panel.pdf_buttons: for button in panel.pdf_buttons:
button.invoke() button.invoke()
self.assertEqual(opened.call_args_list, [call(self.path), *map(call, paths)]) self.assertEqual(opened.call_args_list, [call(self.path), *map(call, paths)])
def test_reload_updates_the_displayed_target_and_its_pdf_action(self):
self.path.write_text("Copie01 A -> B|\n", encoding="utf-8")
copies = self.evaluation / "Copies" / "Copie01"
copies.mkdir(parents=True)
(copies / "A.pdf").touch()
(copies / "C.pdf").touch()
panel = self.panel()
self.path.write_text("Copie01 A -> C|\n", encoding="utf-8")
panel.reload_button.invoke()
self.assertEqual(panel.text.get("1.0", "end-1c"), "Copie01 A -> C|\n")
self.assertEqual(panel.pdf_buttons[1].cget("text"), "Cible : C")
with patch("copienator_gui.manual_resolution.open_path") as opened:
panel.pdf_buttons[1].invoke()
opened.assert_called_once_with(copies / "C.pdf")
def test_reload_keeps_invalid_lines_visible_and_removes_stale_actions(self): def test_reload_keeps_invalid_lines_visible_and_removes_stale_actions(self):
self.path.write_text("Copie01 A -> B|\n", encoding="utf-8") self.path.write_text("Copie01 A -> B|\n", encoding="utf-8")
panel = self.panel() panel = self.panel()
+102 -1
View File
@@ -6,11 +6,40 @@ from unittest.mock import Mock, patch
import pymupdf import pymupdf
from copienator import EvaluationWorkspace, atomic_write_json, read_json from copienator import CliError, EvaluationWorkspace, atomic_write_json, read_json
from copienator.commands import correction, resolve_manual from copienator.commands import correction, resolve_manual
from copienator_gui.app import has_manual_conflicts
class ManualResolutionStateTests(unittest.TestCase): class ManualResolutionStateTests(unittest.TestCase):
def test_merge_into_missing_target_uses_source_and_schedules_redo(self):
for operator in ("->", "x>"):
for target in ("B|", "|B"):
with self.subTest(operator=operator, target=target), tempfile.TemporaryDirectory() as temporary:
workspace = self.fixture(Path(temporary), operator)
copies = workspace.copies_dir / "Copie01"
(copies / "B.pdf").unlink()
workspace.manual_resolutions_file.write_text(f"Copie01 A {operator} {target}\n")
resolve_manual.resolve_manual(workspace)
with pymupdf.open(copies / "B_new.pdf") as document:
self.assertEqual(len(document), 1)
self.assertIn("A", document[0].get_text())
self.assertEqual((copies / "A_old.pdf").exists(), operator == "x>")
self.assertEqual((copies / "A.pdf").exists(), operator == "->")
self.assertFalse((copies / "B_old.pdf").exists())
self.assertEqual(read_json(workspace.refaire_file), [["Copie01", ["B"]]])
self.assertNotIn("delayed", read_json(workspace.correction_file)["A"][0][0]["result"])
def test_merge_still_requires_source_without_changing_state(self):
with tempfile.TemporaryDirectory() as temporary:
workspace = self.fixture(Path(temporary), "x>")
(workspace.copies_dir / "Copie01" / "A.pdf").unlink()
before = workspace.correction_file.read_bytes()
with self.assertRaises(CliError):
resolve_manual.resolve_manual(workspace)
self.assertEqual(workspace.correction_file.read_bytes(), before)
self.assertTrue(workspace.manual_resolutions_file.exists())
def test_acknowledgement_clears_only_this_target_and_copy(self): def test_acknowledgement_clears_only_this_target_and_copy(self):
for error in ("wrg-lbl:B?", "wrg-lbl:B?delayed", "wrg-lbl:B?exists", "al:(->)B?(->)C?", "al:(delayed)B"): for error in ("wrg-lbl:B?", "wrg-lbl:B?delayed", "wrg-lbl:B?exists", "al:(->)B?(->)C?", "al:(delayed)B"):
with self.subTest(error=error): with self.subTest(error=error):
@@ -63,3 +92,75 @@ class ManualResolutionStateTests(unittest.TestCase):
self.assertEqual(correction.run_configured(args), 0) self.assertEqual(correction.run_configured(args), 0)
self.assertFalse(workspace.manual_resolutions_file.exists()) self.assertFalse(workspace.manual_resolutions_file.exists())
self.assertNotIn("delayed", correction.results["A"][0][0]["result"]) self.assertNotIn("delayed", correction.results["A"][0][0]["result"])
def test_deleted_conflict_stays_resolved_when_other_resolution_generates_refaire(self):
for kind in ("wrong-label", "add-label"):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temporary:
workspace = self.fixture(Path(temporary), "x>")
other = workspace.copies_dir / "Copie02"
other.mkdir()
for label in ("A", "B"):
(other / f"{label}.pdf").write_bytes(
(workspace.copies_dir / "Copie01" / f"{label}.pdf").read_bytes())
before = {path.name: path.read_bytes() for path in other.iterdir()}
results = read_json(workspace.correction_file)
results["A"][0].append({"id": "02", "result": {
"error": "wrg-lbl:B?" if kind == "wrong-label" else "al:(->)B?",
"delayed": [[kind, "B"]], "score": 3,
}})
atomic_write_json(workspace.correction_file, results)
# Only Copie01 remains in the edited file; Copie02 was deleted.
resolve_manual.resolve_manual(workspace)
self.assertEqual(read_json(workspace.refaire_file), [["Copie01", ["B"]]])
self.assertEqual(before, {path.name: path.read_bytes() for path in other.iterdir()})
saved = read_json(workspace.correction_file)["A"][0][1]["result"]
self.assertNotIn("delayed", saved)
self.assertEqual(saved["score"], 3)
self.assertNotIn("B?", saved["error"])
(workspace.groups_dir / "B").mkdir(parents=True)
args = correction.build_parser().parse_args([str(workspace.root), "--refaire"])
correction.configure_runtime(workspace, [], args, api_client=Mock())
with patch.object(correction.grouping, "get_pdf_height", return_value=400), \
patch.object(correction.grouping, "create_jpg"), \
patch.object(correction, "process_single_task", return_value=[]):
self.assertEqual(correction.run_configured(args), 0)
self.assertFalse(workspace.manual_resolutions_file.exists())
self.assertNotIn("delayed", correction.results["A"][0][1]["result"])
def test_deleting_all_lines_can_be_applied_and_survives_resume(self):
for text in ("", "### Instructions\n", "### Copie01 A x> B|\n"):
with self.subTest(text=text), tempfile.TemporaryDirectory() as temporary:
workspace = self.fixture(Path(temporary), "x>")
source = workspace.copies_dir / "Copie01"
before = {path.name: path.read_bytes() for path in source.iterdir()}
workspace.manual_resolutions_file.write_text(text)
self.assertTrue(has_manual_conflicts(workspace.manual_resolutions_file))
resolve_manual.resolve_manual(workspace)
self.assertEqual(before, {path.name: path.read_bytes() for path in source.iterdir()})
self.assertFalse(has_manual_conflicts(workspace.manual_resolutions_file))
atomic_write_json(workspace.refaire_file, [])
args = correction.build_parser().parse_args([str(workspace.root), "--refaire"])
correction.configure_runtime(workspace, [], args, api_client=Mock())
self.assertEqual(correction.run_configured(args), 0)
self.assertFalse(workspace.manual_resolutions_file.exists())
def test_invalid_remaining_instruction_does_not_commit_deletions(self):
with tempfile.TemporaryDirectory() as temporary:
workspace = self.fixture(Path(temporary), "x>")
before = workspace.correction_file.read_bytes()
workspace.manual_resolutions_file.write_text("invalid instruction\n")
with self.assertRaises(CliError):
resolve_manual.resolve_manual(workspace)
self.assertEqual(workspace.correction_file.read_bytes(), before)
def test_generated_first_comment_documents_deleting_lines(self):
with tempfile.TemporaryDirectory() as temporary:
workspace = self.fixture(Path(temporary), "x>")
atomic_write_json(workspace.refaire_file, [])
args = correction.build_parser().parse_args([str(workspace.root), "--refaire"])
correction.configure_runtime(workspace, [], args, api_client=Mock())
self.assertEqual(correction.run_configured(args), 0)
first_line = workspace.manual_resolutions_file.read_text().splitlines()[0]
self.assertTrue(first_line.startswith("###"))
self.assertIn("delete a line", first_line)
self.assertIn("resolve-manual", first_line)
+36
View File
@@ -134,6 +134,42 @@ class SelectiveGeminiTests(unittest.TestCase):
gemini.refine_existing(self.workspace, "persp", api_client=self.client) gemini.refine_existing(self.workspace, "persp", api_client=self.client)
self.assertEqual(self.snapshot(), before) self.assertEqual(self.snapshot(), before)
def test_rubric_latex_and_line_breaks_survive_json_and_file_writes(self):
expected = (
r'Barème : $f \circ g \rightarrow \frac{1}{2} \not= 0$.'
'\n' + r'$\begin{matrix}a & b \\ c & d\end{matrix}$, $\theta$, $\beta$, $\nu$.'
'\nUne "justification" vaut 1 point.'
)
self.client.models.generate_content.side_effect = [
self.response({'rubrics': [{'label': label, 'rubric_content': expected} for label in group]})
for group in (self.labels[:2], self.labels[2:])
]
gemini.refine_existing(self.workspace, 'persp', api_client=self.client)
for label in self.labels:
self.assertEqual((self.root / 'Persp' / label).read_bytes(),
(label + '\n' + expected).encode('utf-8'))
def test_invalid_latex_escapes_are_repaired_without_retry(self):
for command in (r'\circ', r'\rightarrow', r'\frac', r'\not', r'\theta', r'\beta'):
with self.subTest(command=command):
# Deliberately malformed wire JSON: do not json.dumps the command.
broken = SimpleNamespace(text='{"rubrics":[{"label":"Ex 2","rubric_content":"$'
+ command + '$"}]}')
self.client.models.generate_content.reset_mock()
self.client.models.generate_content.side_effect = [broken]
self.assertEqual(gemini.generate_rubrics(self.client, 'context'),
{'Ex 2': '$' + command + '$'})
self.assertEqual(self.client.models.generate_content.call_count, 1)
def test_unrepairable_control_character_preserves_all_existing_rubrics(self):
before = self.snapshot()
broken = self.response({'rubrics': [{'label': 'Ex 2', 'rubric_content': 'invalid \x01 control'}]})
self.client.models.generate_content.side_effect = [
self.rubric_response(self.labels[:2]), broken]
with self.assertRaisesRegex(ValueError, 'control characters'):
gemini.refine_existing(self.workspace, 'persp', api_client=self.client)
self.assertEqual(self.snapshot(), before)
def test_cli_dispatches_selective_modes_without_full_extraction(self): def test_cli_dispatches_selective_modes_without_full_extraction(self):
with patch.object(gemini, "refine_existing", return_value=ExitCode.SUCCESS) as refine, patch.object( with patch.object(gemini, "refine_existing", return_value=ExitCode.SUCCESS) as refine, patch.object(
gemini, "process_exam" gemini, "process_exam"
+18 -1
View File
@@ -27,9 +27,26 @@ class CompileToPdfTests(unittest.TestCase):
utils.compile_to_pdf("broken", output) utils.compile_to_pdf("broken", output)
self.assertEqual(output.read_bytes(), b"partial") self.assertEqual(output.read_bytes(), b"partial")
self.assertIn("Warning: LaTeX compilation failed", console.getvalue()) self.assertIn("Warning: LaTeX compilation reported errors", console.getvalue())
self.assertIn("a PDF was generated", console.getvalue())
self.assertIn("missing \\item", console.getvalue()) self.assertIn("missing \\item", console.getvalue())
def test_says_no_pdf_was_generated_when_latex_fails(self):
with tempfile.TemporaryDirectory() as temporary:
output = Path(temporary) / "result.pdf"
result = subprocess.CompletedProcess(
args=[], returncode=1, stdout="! Undefined control sequence.\n"
)
console = io.StringIO()
with patch.object(utils.subprocess, "run", return_value=result), \
contextlib.redirect_stdout(console):
utils.compile_to_pdf("broken", output)
self.assertFalse(output.exists())
self.assertIn("no PDF was generated", console.getvalue())
self.assertIn("Undefined control sequence", console.getvalue())
def test_warns_when_no_pdf_is_produced(self): def test_warns_when_no_pdf_is_produced(self):
with tempfile.TemporaryDirectory() as temporary: with tempfile.TemporaryDirectory() as temporary:
output = Path(temporary) / "result.pdf" output = Path(temporary) / "result.pdf"