Compare commits
6
Commits
0a453403cf
...
5009831c7c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5009831c7c | ||
|
|
a9c0703f03 | ||
|
|
c8ce04cf50 | ||
|
|
55f9a29677 | ||
|
|
6a6ed178ef | ||
|
|
32764c101b |
No files matched your search
@@ -27,7 +27,7 @@ from copienator.commands import annotating, annotating_with_checks
|
||||
from copienator.filesystem import staged_directory
|
||||
from copienator.utils import natural_key
|
||||
|
||||
MAX_HEIGHT_PX = 25000
|
||||
MAX_HEIGHT_PX = 32000
|
||||
|
||||
|
||||
def render_item(item):
|
||||
@@ -63,6 +63,11 @@ def save_batch(batch, prefix, group_id, output_root: Path) -> None:
|
||||
print(f"Generating group PDF: {prefix} G{group_id} ({len(batch)} elements)")
|
||||
max_width = max(item[2].width for item in batch)
|
||||
total_height = sum(item[2].height for item in batch)
|
||||
if total_height > MAX_HEIGHT_PX:
|
||||
raise CliError(
|
||||
f"{prefix} G{group_id} would be {total_height} px high; "
|
||||
f"the maximum is {MAX_HEIGHT_PX} px."
|
||||
)
|
||||
concatenated = Image.new("RGB", (max_width, total_height), "white")
|
||||
draw = ImageDraw.Draw(concatenated)
|
||||
checkbox_map: list[dict[str, Any]] = []
|
||||
@@ -268,32 +273,44 @@ def _existing_group_state(
|
||||
|
||||
|
||||
def split_batches(rendered):
|
||||
def split(maximum_height: float):
|
||||
batches = []
|
||||
current = []
|
||||
current_height = 0
|
||||
previous_student = None
|
||||
for item in rendered:
|
||||
student_id = item[0]
|
||||
image_height = item[2].height
|
||||
if (
|
||||
current
|
||||
and current_height + image_height > maximum_height
|
||||
and student_id != previous_student
|
||||
):
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal current, current_height
|
||||
if current:
|
||||
batches.append(current)
|
||||
current = []
|
||||
current_height = 0
|
||||
current.append(item)
|
||||
current_height += image_height
|
||||
previous_student = student_id
|
||||
if current:
|
||||
batches.append(current)
|
||||
return batches
|
||||
|
||||
strict = split(MAX_HEIGHT_PX)
|
||||
relaxed = split(1.1 * MAX_HEIGHT_PX)
|
||||
return relaxed if len(relaxed) < len(strict) else strict
|
||||
index = 0
|
||||
while index < len(rendered):
|
||||
student_id = rendered[index][0]
|
||||
end = index + 1
|
||||
while end < len(rendered) and rendered[end][0] == student_id:
|
||||
end += 1
|
||||
student_items = rendered[index:end]
|
||||
student_height = sum(item[2].height for item in student_items)
|
||||
|
||||
if student_height > MAX_HEIGHT_PX:
|
||||
labels = ", ".join(item[1] for item in student_items)
|
||||
raise CliError(
|
||||
f"Copie{student_id} ({labels}) is {student_height} px high; "
|
||||
f"one student's labels cannot be split across groups and the "
|
||||
f"maximum group height is {MAX_HEIGHT_PX} px."
|
||||
)
|
||||
|
||||
# Consider the student's complete block before adding its first image.
|
||||
# Otherwise later images from that same student could overflow the batch.
|
||||
if current and current_height + student_height > MAX_HEIGHT_PX:
|
||||
flush()
|
||||
current.extend(student_items)
|
||||
current_height += student_height
|
||||
index = end
|
||||
|
||||
flush()
|
||||
return batches
|
||||
|
||||
|
||||
def _generate_groups(
|
||||
|
||||
@@ -1026,7 +1026,9 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
|
||||
manual_path = INPUT_DIR / "manual_resolutions.txt"
|
||||
atomic_write_text(
|
||||
manual_path,
|
||||
"### Use -> x>, -x, ss, sx, xx, xs, c{43}1>, c{43}2x\n"
|
||||
"### Use -> x>, -x, ss, sx, xx, xs, c{43}1>, c{43}2x; "
|
||||
"delete a line to dismiss its conflict and keep its PDFs unchanged, "
|
||||
"then run resolve-manual (even if all lines were deleted).\n"
|
||||
+ "\n".join(unresolved_delayed)
|
||||
+ "\n",
|
||||
)
|
||||
|
||||
@@ -195,6 +195,7 @@ class ImageReviewer:
|
||||
self.root.bind("N", lambda _event: self.on_shift(2 * CROP_SHIFT_STEP))
|
||||
self.root.bind("t", lambda _event: self.on_shift(-CROP_SHIFT_STEP))
|
||||
self.root.bind("l", lambda _event: self.on_enlarge(CROP_WIDTH_STEP))
|
||||
self.root.bind("L", lambda _event: self.on_enlarge(-CROP_WIDTH_STEP))
|
||||
self.root.bind("1", lambda _event: self.on_set_max_pages(1))
|
||||
|
||||
Thread(target=self.prefetch_worker, daemon=True).start()
|
||||
@@ -290,7 +291,7 @@ class ImageReviewer:
|
||||
f"Files: {schema['number_of_files']} | "
|
||||
f"Cols: {schema['columns_per_file']}\n"
|
||||
"Enter: Save and next | s: flag error and skip | n: +50 | N: +100 | t: -50 | "
|
||||
"l: widen by 50 | 1: use full pages"
|
||||
"l: widen by 50 | Shift+l: undo widening by 50 | 1: use full pages"
|
||||
),
|
||||
fg="black",
|
||||
)
|
||||
@@ -305,7 +306,7 @@ class ImageReviewer:
|
||||
def on_enlarge(self, amount: int) -> None:
|
||||
if self.is_processing:
|
||||
return
|
||||
self.current_width_offset += amount
|
||||
self.current_width_offset = max(0, self.current_width_offset + amount)
|
||||
print(f"Applying extra width: {self.current_width_offset}")
|
||||
self.trigger_processing(self.files[self.index], self.current_shift)
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from copienator import (
|
||||
workspace_from_args,
|
||||
)
|
||||
from copienator.platform import validate_windows_labels
|
||||
from copienator.latex_json import repair_latex_json
|
||||
from copienator.filesystem import staged_directory
|
||||
from copienator.utils import compile_to_pdf
|
||||
|
||||
@@ -69,7 +70,7 @@ class ExamContext(BaseModel):
|
||||
# --- Modèles pour la Requête 4 (Barèmes) ---
|
||||
class RubricItem(BaseModel):
|
||||
label: str = Field(description="Label exact de la question, à conserver sans traduction.")
|
||||
rubric_content: str = Field(description="Barème détaillé sur 4 points : toutes les consignes, explications et justifications doivent être rédigées en français.")
|
||||
rubric_content: str = Field(description="Barème détaillé sur 4 points, en français, avec formules LaTeX. Échapper chaque antislash LaTeX conformément à la syntaxe JSON.")
|
||||
|
||||
class GroupRubrics(BaseModel):
|
||||
rubrics: list[RubricItem]
|
||||
@@ -101,6 +102,27 @@ question fournie, cette liste contient un objet avec son `label` exact et
|
||||
son barème en français dans `rubric_content`.
|
||||
"""
|
||||
|
||||
LATEX_JSON_INSTRUCTION = r"""
|
||||
IMPORTANT — Échappement JSON du LaTeX :
|
||||
Dans les chaînes JSON, chaque antislash LaTeX doit être doublé.
|
||||
Exemple d’échappement dans un barème (adapte les clés au schéma demandé) :
|
||||
{"rubrics":[{"label":"1)","rubric_content":"1 point : $\\frac{1}{2}$ et $f \\circ g$.\n1 point : $x \\rightarrow 0$ et $x \\not=0$."}]}
|
||||
Ne confonds pas les commandes LaTeX avec les échappements JSON :
|
||||
\\frac, \\rightarrow, \\not, \\theta et \\begin sont des commandes LaTeX ;
|
||||
\n est réservé à un véritable saut de ligne du texte.
|
||||
"""
|
||||
|
||||
|
||||
def validate_rubric_content(content: str) -> str:
|
||||
"""Catch decoded control escapes; ordinary JSON newlines remain valid.
|
||||
|
||||
A misescaped LaTeX command starting with n is indistinguishable from
|
||||
an intentional newline. Do not guess or rewrite those after decoding.
|
||||
"""
|
||||
if any((ord(char) < 32 and char != "\n") or ord(char) == 127 for char in content):
|
||||
raise ValueError("Rubric contains control characters, likely from unescaped LaTeX")
|
||||
return content
|
||||
|
||||
# --- Modèle fusionné (pour le reste du script) ---
|
||||
class QuestionItem(BaseModel):
|
||||
label: str
|
||||
@@ -178,6 +200,12 @@ traduire, et conserve le LaTeX ainsi que les labels exacts.
|
||||
Renvoie le résultat dans la liste `contexts` de l'objet JSON attendu.
|
||||
"""
|
||||
|
||||
PROMPT_1 += LATEX_JSON_INSTRUCTION
|
||||
PROMPT_2 += LATEX_JSON_INSTRUCTION
|
||||
PROMPT_3 += LATEX_JSON_INSTRUCTION
|
||||
PROMPT_4 += LATEX_JSON_INSTRUCTION
|
||||
|
||||
|
||||
def find_file(folder: Path, base_name: str) -> Path | None:
|
||||
for ext in [".org", ".tex"]:
|
||||
path = folder / f"{base_name}{ext}"
|
||||
@@ -202,10 +230,10 @@ def generate_rubrics(client, group_context_text: str) -> dict[str, str]:
|
||||
response_json_schema=GroupRubrics.model_json_schema(),
|
||||
),
|
||||
)
|
||||
rubrics = GroupRubrics.model_validate_json(response.text).rubrics
|
||||
rubrics = GroupRubrics.model_validate_json(repair_latex_json(response.text)).rubrics
|
||||
if len({item.label for item in rubrics}) != len(rubrics):
|
||||
raise ValueError("Gemini returned duplicate rubric labels")
|
||||
return {item.label: item.rubric_content for item in rubrics}
|
||||
return {item.label: validate_rubric_content(item.rubric_content) for item in rubrics}
|
||||
|
||||
|
||||
def validate_groups(groups: list[list[str]], labels: list[str]) -> None:
|
||||
@@ -364,7 +392,7 @@ def process_exam(
|
||||
print("Saving questions to cache...")
|
||||
atomic_write_text(cache_q_file, response_q_text)
|
||||
|
||||
questions_data = ExamQuestions.model_validate_json(response_q_text)
|
||||
questions_data = ExamQuestions.model_validate_json(repair_latex_json(response_q_text))
|
||||
|
||||
# ==========================================
|
||||
# REQUÊTE 2 : Extraction des Corrections
|
||||
@@ -403,7 +431,7 @@ def process_exam(
|
||||
print("Saving solutions to cache...")
|
||||
atomic_write_text(cache_s_file, response_s_text)
|
||||
|
||||
solutions_data = ExamSolutions.model_validate_json(response_s_text)
|
||||
solutions_data = ExamSolutions.model_validate_json(repair_latex_json(response_s_text))
|
||||
|
||||
# ==========================================
|
||||
# REQUÊTE 3 : Extraction du Contexte (Notations, etc.)
|
||||
@@ -440,7 +468,7 @@ def process_exam(
|
||||
print("Saving context to cache...")
|
||||
atomic_write_text(cache_c_file, response_c_text)
|
||||
|
||||
context_data = ExamContext.model_validate_json(response_c_text)
|
||||
context_data = ExamContext.model_validate_json(repair_latex_json(response_c_text))
|
||||
|
||||
# ==========================================
|
||||
# FUSION des trois résultats
|
||||
|
||||
@@ -4,9 +4,10 @@ import argparse
|
||||
import re
|
||||
import time
|
||||
import typing
|
||||
from threading import Lock
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import ThreadPoolExecutor, wait
|
||||
from pathlib import Path
|
||||
|
||||
from google import genai
|
||||
@@ -271,6 +272,10 @@ def generate_request(
|
||||
]
|
||||
|
||||
generate_content_config = types.GenerateContentConfig(
|
||||
http_options=types.HttpOptions(
|
||||
timeout=120_000,
|
||||
retry_options=types.HttpRetryOptions(attempts=1),
|
||||
),
|
||||
automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
|
||||
temperature=1.0,
|
||||
top_p=0.95,
|
||||
@@ -282,9 +287,17 @@ def generate_request(
|
||||
return (contents, generate_content_config)
|
||||
|
||||
TARGET_INTERVAL = 3.5
|
||||
MAX_ATTEMPTS = 3
|
||||
PROGRESS_INTERVAL = 30
|
||||
Sleep = Callable[[float], None]
|
||||
|
||||
|
||||
class IncompleteCopyError(Exception):
|
||||
def __init__(self, images: list[Path]):
|
||||
self.images = images
|
||||
super().__init__(f"Label detection incomplete for {images[0].name}")
|
||||
|
||||
|
||||
def selected_images(
|
||||
workspace: EvaluationWorkspace,
|
||||
targets: list[Path],
|
||||
@@ -395,11 +408,12 @@ def process_copy_group(
|
||||
overwrite: bool,
|
||||
sleep: Sleep = time.sleep,
|
||||
target_interval: float = TARGET_INTERVAL,
|
||||
progress: Callable[[str, str, float], None] | None = None,
|
||||
) -> int:
|
||||
"""Process one student's image parts sequentially to preserve context."""
|
||||
accumulated_labels: list[str] = []
|
||||
generated = 0
|
||||
for image_file in files:
|
||||
for image_index, image_file in enumerate(files):
|
||||
started = time.monotonic()
|
||||
output_json = workspace.copies_dir / f"{image_file.stem}.json"
|
||||
if output_json.exists() and not overwrite:
|
||||
@@ -414,9 +428,13 @@ def process_copy_group(
|
||||
attempt = 0
|
||||
label_retry_count = 0
|
||||
wrong_labels: list[str] = []
|
||||
while True:
|
||||
while attempt < MAX_ATTEMPTS:
|
||||
if progress:
|
||||
progress(group_key, f"{image_file.name}, attempt {attempt + 1}/{MAX_ATTEMPTS}", started)
|
||||
if attempt > 0:
|
||||
print(f"[{group_key}] Retrying {image_file.name} in {10 * attempt}s...")
|
||||
sleep(10 * attempt)
|
||||
print(f"[{group_key}] {image_file.name}: attempt {attempt + 1}/{MAX_ATTEMPTS} (120s timeout).")
|
||||
try:
|
||||
text_context_file, text_context = closest_text_context(
|
||||
workspace, wrong_labels
|
||||
@@ -486,15 +504,23 @@ def process_copy_group(
|
||||
atomic_write_json(output_json, annotation.model_dump())
|
||||
accumulated_labels.extend(box.label for box in annotation.list)
|
||||
generated += 1
|
||||
print(f"[{group_key}] Saved {image_file.name} after {time.monotonic() - started:.0f}s.")
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - remote API retry boundary
|
||||
print(
|
||||
f"Error processing {image_file.name}: {exc}\n"
|
||||
"\tIt will be retried."
|
||||
f"Error processing {image_file.name} "
|
||||
f"(attempt {attempt + 1}/{MAX_ATTEMPTS}): {type(exc).__name__}: {exc}"
|
||||
)
|
||||
attempt += 1
|
||||
else:
|
||||
# Later pages need the context of this page; defer them until resume.
|
||||
incomplete = [path for path in files[image_index:]
|
||||
if overwrite or not (workspace.copies_dir / f"{path.stem}.json").exists()]
|
||||
print(f"[{group_key}] Failed after {MAX_ATTEMPTS} attempts; "
|
||||
"deferring remaining pages of this copy until the next run.")
|
||||
raise IncompleteCopyError(incomplete)
|
||||
sleep(max(0.0, target_interval - (time.monotonic() - started)))
|
||||
return generated
|
||||
|
||||
@@ -528,8 +554,17 @@ def run(
|
||||
client = genai.Client(api_key=api_key)
|
||||
|
||||
groups = group_images(images)
|
||||
active: dict[str, tuple[str, float]] = {}
|
||||
progress_lock = Lock()
|
||||
incomplete: list[Path] = []
|
||||
run_started = time.monotonic()
|
||||
|
||||
def report_progress(group: str, description: str, started: float) -> None:
|
||||
with progress_lock:
|
||||
active[group] = (description, started)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = [
|
||||
futures = {
|
||||
executor.submit(
|
||||
process_copy_group,
|
||||
workspace,
|
||||
@@ -542,12 +577,33 @@ def run(
|
||||
valid_names=valid_names,
|
||||
overwrite=overwrite,
|
||||
sleep=sleep,
|
||||
)
|
||||
progress=report_progress,
|
||||
): group_key
|
||||
for group_key, files in groups.items()
|
||||
]
|
||||
for future in futures:
|
||||
}
|
||||
pending = set(futures)
|
||||
while pending:
|
||||
done, pending = wait(pending, timeout=PROGRESS_INTERVAL)
|
||||
for future in done:
|
||||
with progress_lock:
|
||||
active.pop(futures[future], None)
|
||||
try:
|
||||
future.result()
|
||||
return ExitCode.PARTIAL if warnings else ExitCode.SUCCESS
|
||||
except IncompleteCopyError as exc:
|
||||
incomplete.extend(exc.images)
|
||||
if pending:
|
||||
now = time.monotonic()
|
||||
with progress_lock:
|
||||
snapshot = list(active.items())
|
||||
print(f"Label detection: {now - run_started:.0f}s elapsed; "
|
||||
f"{len(pending)} copies outstanding ({len(pending) - len(snapshot)} queued).")
|
||||
for group, (description, started) in snapshot:
|
||||
print(f" [{group}] {description}: {now - started:.0f}s elapsed for image.")
|
||||
if incomplete:
|
||||
print("Incomplete images (rerun label detection to resume):")
|
||||
for path in sorted(incomplete):
|
||||
print(f" {path.name}")
|
||||
return ExitCode.PARTIAL if warnings or incomplete else ExitCode.SUCCESS
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
|
||||
@@ -187,7 +187,6 @@ class PDFPreviewer:
|
||||
action_map = {
|
||||
"move_left": self.move_line_left,
|
||||
"move_right": self.move_line_right,
|
||||
"confirm_next": self.confirm_and_next_page,
|
||||
"rotate_page": self.rotate_page,
|
||||
"rotate_all_pages": self.rotate_all_pages,
|
||||
"rotate_all_files": self.rotate_all_files,
|
||||
@@ -209,7 +208,6 @@ class PDFPreviewer:
|
||||
|
||||
# self.master.bind("<Left>", self.move_line_left)
|
||||
# self.master.bind("<Right>", self.move_line_right)
|
||||
# self.master.bind("<Return>", self.confirm_and_next_page)
|
||||
# self.master.bind("c", self.rotate_page)
|
||||
# self.master.bind("C", self.rotate_all_pages)
|
||||
# self.master.bind(",", self.rotate_all_files)
|
||||
@@ -260,7 +258,9 @@ class PDFPreviewer:
|
||||
return
|
||||
|
||||
page = self.doc.load_page(self.current_page_index)
|
||||
self.page_label.config(text=f"Page {self.current_page_index + 1} of {len(self.doc)}")
|
||||
self.page_label.config(
|
||||
text=f"{self.pdf_path.name} — Page {self.current_page_index + 1} of {len(self.doc)}"
|
||||
)
|
||||
|
||||
# --- Calculate Scaling ---
|
||||
canvas_width = self.canvas.winfo_width()
|
||||
|
||||
@@ -132,6 +132,18 @@ def set_suffix_and_clean_error(
|
||||
result["error"] = error
|
||||
|
||||
|
||||
def pending_manual_conflicts(results: dict[str, Any]) -> set[tuple[str, str, str]]:
|
||||
"""Return the copy/source/target conflicts listed by correction's manual file."""
|
||||
return {
|
||||
(item["id"], label, target)
|
||||
for label, batches in results.items()
|
||||
for batch in batches
|
||||
for item in batch
|
||||
for kind, target in item.get("result", {}).get("delayed", [])
|
||||
if kind in {"wrong-label", "add-label"}
|
||||
}
|
||||
|
||||
|
||||
def get_actual_pdf(copies_dir: Path, copy_id: str, label: str) -> Path:
|
||||
base = copies_dir / f"Copie{copy_id}" / f"{label}.pdf"
|
||||
for candidate in (
|
||||
@@ -157,13 +169,7 @@ def _validate_pdf_inputs(
|
||||
missing: set[Path] = set()
|
||||
for instruction in instructions:
|
||||
source = initial_paths[(instruction.copy_id, instruction.old_label)]
|
||||
destination = initial_paths[(instruction.copy_id, instruction.new_label)]
|
||||
if instruction.should_merge:
|
||||
if not source.exists():
|
||||
missing.add(source)
|
||||
if not destination.exists():
|
||||
missing.add(destination)
|
||||
elif instruction.should_copy and not source.exists():
|
||||
if (instruction.should_merge or instruction.should_copy) and not source.exists():
|
||||
missing.add(source)
|
||||
if missing:
|
||||
rendered = ", ".join(str(path) for path in sorted(missing))
|
||||
@@ -178,6 +184,10 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
|
||||
raise CliError("correction.json must contain a JSON object")
|
||||
results: dict[str, Any] = loaded
|
||||
instructions = parse_instructions(workspace.manual_resolutions_file)
|
||||
retained_conflicts = {
|
||||
(item.copy_id, item.old_label, item.new_label) for item in instructions
|
||||
}
|
||||
dismissed_conflicts = pending_manual_conflicts(results) - retained_conflicts
|
||||
cut_sources = [(item.copy_id, item.old_label) for item in instructions if item.cut]
|
||||
if len(set(cut_sources)) != len(cut_sources):
|
||||
raise CliError("Une seule coupe par label source est autorisée dans une résolution.")
|
||||
@@ -226,8 +236,10 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
|
||||
try:
|
||||
if instruction.pipe_first:
|
||||
writer.append(source)
|
||||
if destination.exists():
|
||||
writer.append(destination)
|
||||
else:
|
||||
if destination.exists():
|
||||
writer.append(destination)
|
||||
writer.append(source)
|
||||
writer.write(temp_output)
|
||||
@@ -285,6 +297,9 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
|
||||
if temporary not in used_temps:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
for copy_id, label, target in dismissed_conflicts:
|
||||
# A removed instruction acknowledges the conflict without touching PDFs.
|
||||
set_suffix_and_clean_error(results, copy_id, label, None, target)
|
||||
atomic_write_json(workspace.correction_file, results)
|
||||
refaire_tasks = [[copy_name, labels] for copy_name, labels in refaire_by_copy.items()]
|
||||
if refaire_tasks:
|
||||
@@ -292,6 +307,8 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
|
||||
workspace.manual_resolutions_file.unlink()
|
||||
|
||||
print("Manual resolutions successfully applied.")
|
||||
if dismissed_conflicts:
|
||||
print(f"{len(dismissed_conflicts)} removed conflict(s) dismissed; their PDFs were kept unchanged.")
|
||||
if refaire_tasks:
|
||||
print(
|
||||
f"File {workspace.refaire_file.name} generated. Run "
|
||||
|
||||
@@ -19,11 +19,7 @@ from copienator import (
|
||||
)
|
||||
|
||||
|
||||
def run(workspace: EvaluationWorkspace, *, client=None) -> ExitCode:
|
||||
if client is None:
|
||||
if not config.API_KEY:
|
||||
raise CliError("GEMINI_API_KEY is not configured")
|
||||
client = genai.Client(api_key=config.API_KEY)
|
||||
def run(workspace: EvaluationWorkspace, *, client=None, overwrite: bool = False) -> ExitCode:
|
||||
batches = (
|
||||
(
|
||||
"flash",
|
||||
@@ -45,16 +41,32 @@ def run(workspace: EvaluationWorkspace, *, client=None) -> ExitCode:
|
||||
}
|
||||
if workspace.batch_jobs_file.is_file():
|
||||
previous = read_json(workspace.batch_jobs_file)
|
||||
if isinstance(previous, dict) and isinstance(previous.get("jobs"), dict):
|
||||
if not isinstance(previous, dict) or not isinstance(previous.get("jobs"), dict):
|
||||
raise CliError(f"Invalid batch manifest: {workspace.batch_jobs_file}")
|
||||
manifest["jobs"] = previous["jobs"]
|
||||
started = 0
|
||||
skipped = 0
|
||||
for tier, file_path, model_id, display_name in batches:
|
||||
existing = manifest["jobs"].get(tier)
|
||||
if tier in manifest["jobs"]:
|
||||
if (not isinstance(existing, dict)
|
||||
or not isinstance(existing.get("name"), str)
|
||||
or not existing["name"].strip()):
|
||||
raise CliError(f"Invalid batch job in {workspace.batch_jobs_file}: {tier}")
|
||||
if not overwrite:
|
||||
print(f"Skipping {tier}: already submitted as {existing['name']}.")
|
||||
skipped += 1
|
||||
continue
|
||||
if not file_path.is_file():
|
||||
print(f"Skipping {model_id}: {file_path.name} does not exist.")
|
||||
continue
|
||||
if file_path.stat().st_size == 0:
|
||||
print(f"Skipping {model_id}: {file_path.name} is empty.")
|
||||
continue
|
||||
if client is None:
|
||||
if not config.API_KEY:
|
||||
raise CliError("GEMINI_API_KEY is not configured")
|
||||
client = genai.Client(api_key=config.API_KEY)
|
||||
print(f"Uploading {file_path.name} for model {model_id}...")
|
||||
uploaded = client.files.upload(
|
||||
file=str(file_path),
|
||||
@@ -77,19 +89,26 @@ def run(workspace: EvaluationWorkspace, *, client=None) -> ExitCode:
|
||||
}
|
||||
atomic_write_json(workspace.batch_jobs_file, manifest)
|
||||
print(f"Started batch job: {job.name}")
|
||||
if not started:
|
||||
if not started and not skipped:
|
||||
print("No non-empty batch request files were found.")
|
||||
return ExitCode.PARTIAL
|
||||
return ExitCode.SUCCESS
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
return evaluation_parser("Upload correction JSONL files and start Gemini batches")
|
||||
parser = evaluation_parser("Upload correction JSONL files and start Gemini batches")
|
||||
parser.add_argument(
|
||||
"--overwrite", action="store_true",
|
||||
help="Submit new jobs even for recorded tiers, replacing their saved job IDs",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
return execute(parser, argv, lambda args: run(workspace_from_args(args)))
|
||||
return execute(parser, argv, lambda args: run(
|
||||
workspace_from_args(args), overwrite=args.overwrite,
|
||||
))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+9
-7
@@ -177,21 +177,23 @@ def compile_to_pdf(text, output_pdf_path):
|
||||
text=True,
|
||||
check=False)
|
||||
|
||||
generated_pdf = os.path.join(temp_dir, pdf_filename)
|
||||
pdf_generated = os.path.exists(generated_pdf)
|
||||
if pdf_generated:
|
||||
shutil.move(generated_pdf, output_pdf_path)
|
||||
|
||||
if result.returncode != 0:
|
||||
error_lines = [
|
||||
line.strip() for line in result.stdout.splitlines()
|
||||
if line.lstrip().startswith("!")
|
||||
]
|
||||
detail = f": {error_lines[0]}" if error_lines else ""
|
||||
pdf_status = "a PDF was generated" if pdf_generated else "no PDF was generated"
|
||||
print(
|
||||
f"Warning: LaTeX compilation failed for {output_pdf_path} "
|
||||
f"(exit code {result.returncode}){detail}"
|
||||
f"Warning: LaTeX compilation reported errors for {output_pdf_path} "
|
||||
f"(exit code {result.returncode}; {pdf_status}){detail}"
|
||||
)
|
||||
|
||||
generated_pdf = os.path.join(temp_dir, pdf_filename)
|
||||
if os.path.exists(generated_pdf):
|
||||
shutil.move(generated_pdf, output_pdf_path)
|
||||
else:
|
||||
elif not pdf_generated:
|
||||
print(f"Warning: LaTeX compilation produced no PDF for {output_pdf_path}")
|
||||
except Exception as e:
|
||||
print(f"Compilation error for {output_pdf_path}: {e}")
|
||||
+75
-20
@@ -10,7 +10,8 @@ from pathlib import Path
|
||||
from tkinter import filedialog, messagebox, ttk
|
||||
from typing import Any
|
||||
|
||||
from copienator import CliError, EvaluationWorkspace, ExitCode, atomic_write_json
|
||||
from copienator import CliError, EvaluationWorkspace, ExitCode, atomic_write_json, read_json
|
||||
from copienator.commands.resolve_manual import pending_manual_conflicts
|
||||
from copienator.copy_errors import copy_errors
|
||||
from copienator.filesystem import staged_files
|
||||
from copienator.platform import (
|
||||
@@ -244,7 +245,14 @@ def has_manual_conflicts(path: Path) -> bool:
|
||||
# If the file cannot be inspected, keep the step visible rather than
|
||||
# silently claiming that there is nothing to resolve.
|
||||
return True
|
||||
return any(line.strip() and not line.lstrip().startswith("###") for line in lines)
|
||||
if any(line.strip() and not line.lstrip().startswith("###") for line in lines):
|
||||
return True
|
||||
# An emptied manual file still needs execution to acknowledge deletions.
|
||||
try:
|
||||
results = read_json(path.parent / "correction.json", default={})
|
||||
return bool(pending_manual_conflicts(results))
|
||||
except (OSError, ValueError, TypeError, KeyError, AttributeError):
|
||||
return True
|
||||
|
||||
|
||||
class CopienatorApp(tk.Tk):
|
||||
@@ -591,7 +599,10 @@ class CopienatorApp(tk.Tk):
|
||||
opened = {self.tree.item(item, "text"): self.tree.item(item, "open") for item in self.tree.get_children()}
|
||||
self.tree.delete(*self.tree.get_children())
|
||||
section_items: dict[str, str] = {}
|
||||
show_batches = self._has_batch_correction()
|
||||
for step in self.steps:
|
||||
if step.skip_for_live_correction and not show_batches:
|
||||
continue
|
||||
if step.section not in section_items:
|
||||
section_id = f"section:{len(section_items)}"
|
||||
section_items[step.section] = section_id
|
||||
@@ -607,6 +618,25 @@ class CopienatorApp(tk.Tk):
|
||||
)
|
||||
if selected and self.tree.exists(selected):
|
||||
self.tree.selection_set(selected)
|
||||
elif selected and self.tree.exists("correction"):
|
||||
self.tree.selection_set("correction")
|
||||
|
||||
def _has_batch_correction(self) -> bool:
|
||||
evaluation = self.state_store.evaluation
|
||||
if evaluation is None:
|
||||
return False
|
||||
correction = self.state_store.step("correction")
|
||||
if correction.get("batch_started") or correction.get("last_run_variant") in {"batch", "hybrid", "integrate"}:
|
||||
return True
|
||||
for name in ("batch_requests_flash.jsonl", "batch_requests_pro.jsonl",
|
||||
"batch_jobs.json", "batched_correction_result.jsonl"):
|
||||
path = evaluation / name
|
||||
try:
|
||||
if path.is_file() and path.stat().st_size > 0:
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
return False
|
||||
|
||||
def _step_status(self, step: StepDefinition) -> str:
|
||||
if self.state_store.evaluation:
|
||||
@@ -721,16 +751,6 @@ class CopienatorApp(tk.Tk):
|
||||
# trigger the same automatic behavior twice.
|
||||
self.state_store.update_step(step.id, visited=True)
|
||||
|
||||
if step.skip_for_live_correction:
|
||||
correction = self.state_store.step("correction")
|
||||
if correction.get("variant", "live") == "live":
|
||||
self.after_idle(
|
||||
lambda step_id=step.id: self._automatic_skip(
|
||||
step_id, "correction immédiate sélectionnée"
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
if step.skip_without_manual_conflicts:
|
||||
evaluation = self.evaluation
|
||||
conflicts = evaluation / "manual_resolutions.txt" if evaluation else None
|
||||
@@ -1217,7 +1237,9 @@ class CopienatorApp(tk.Tk):
|
||||
|
||||
def _progression_ids(self, step_id: str) -> list[str]:
|
||||
redo = step_id.startswith("refaire_")
|
||||
return [step.id for step in self.steps if step.id.startswith("refaire_") == redo]
|
||||
show_batches = self._has_batch_correction()
|
||||
return [step.id for step in self.steps if step.id.startswith("refaire_") == redo
|
||||
and (not step.skip_for_live_correction or show_batches)]
|
||||
|
||||
def _refaire_selection(self) -> list[list]:
|
||||
evaluation = self.evaluation
|
||||
@@ -1445,6 +1467,10 @@ class CopienatorApp(tk.Tk):
|
||||
self._save_current_form()
|
||||
run_values = self._values()
|
||||
ordered_ids = self._progression_ids(step.id)
|
||||
if step.id == "correction" and variant.id == "integrate":
|
||||
# Integration consumes the retrieved batches; it does not invalidate them.
|
||||
ordered_ids = [step_id for step_id in ordered_ids
|
||||
if not self.step_by_id[step_id].skip_for_live_correction]
|
||||
self.state_store.invalidate_after(ordered_ids, step.id)
|
||||
self.state_store.update_step(
|
||||
step.id,
|
||||
@@ -1473,6 +1499,8 @@ class CopienatorApp(tk.Tk):
|
||||
self._append_console(f"\n$ {command_display(command)}\n")
|
||||
try:
|
||||
self.runner.start(command, self.repository, environment, log_path)
|
||||
if step.id == "correction" and variant.id in {"batch", "hybrid"}:
|
||||
self.state_store.update_step("correction", batch_started=True)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
self.state_store.update_step(step.id, status="failed")
|
||||
self.state_store.add_history(
|
||||
@@ -1551,7 +1579,6 @@ class CopienatorApp(tk.Tk):
|
||||
if self.current_step and self.current_step.optional:
|
||||
step_id = self.current_step.id
|
||||
self._mark_step("skipped")
|
||||
if step_id.startswith("refaire_"):
|
||||
self._move_selection_from(step_id, 1)
|
||||
|
||||
def _poll_runner(self) -> None:
|
||||
@@ -1615,15 +1642,18 @@ class CopienatorApp(tk.Tk):
|
||||
self._populate_tree()
|
||||
message = f"{evaluation.name} : tous les résultats batch sont prêts à récupérer."
|
||||
self.info_var.set(message)
|
||||
try:
|
||||
notify_desktop("Copienator — batchs terminés", message)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
self.bell()
|
||||
self._append_console(f"Notification de bureau indisponible : {exc}\n{message}\n")
|
||||
self._notify_completion("Copienator — batchs terminés", message)
|
||||
if (self.current_step and self.current_step.id == "batch_status"
|
||||
and not self.active_step_id and not self.runner.running):
|
||||
self._move_selection_from("batch_status", 1)
|
||||
|
||||
def _notify_completion(self, title: str, message: str) -> None:
|
||||
try:
|
||||
notify_desktop(title, message)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
self.bell()
|
||||
self._append_console(f"Notification de bureau indisponible : {exc}\n{message}\n")
|
||||
|
||||
def _finish_process(self, return_code: int, interrupted: bool) -> None:
|
||||
step_id = self.active_step_id
|
||||
if not step_id:
|
||||
@@ -1710,12 +1740,28 @@ class CopienatorApp(tk.Tk):
|
||||
"sur « Marquer terminée »."
|
||||
)
|
||||
return
|
||||
notified_completions = {
|
||||
"labels": "détection des labels terminée",
|
||||
"correction": "étape de correction terminée",
|
||||
"annotation": "génération des copies annotées terminée",
|
||||
"read_annotations": "lecture des annotations manuscrites terminée",
|
||||
}
|
||||
if step_id in notified_completions and self.state_store.evaluation:
|
||||
completion = notified_completions[step_id]
|
||||
outcome = STATUS_LABELS[status].lower()
|
||||
self._notify_completion(
|
||||
f"Copienator — {completion} ({outcome})",
|
||||
f"{self.state_store.evaluation.name} : {completion} — {outcome} "
|
||||
f"(code {return_code}).",
|
||||
)
|
||||
if status == "success":
|
||||
if step_id == "batch_status" and self.batch_monitor.active:
|
||||
self.batch_monitor.stop()
|
||||
self._batch_results_ready()
|
||||
return
|
||||
if step_id == "manual_resolution":
|
||||
if step_id == "fetch_batches":
|
||||
self.after_idle(self._select_batch_integration)
|
||||
elif step_id == "manual_resolution":
|
||||
self.after_idle(self._select_refaire_correction)
|
||||
else:
|
||||
self.after_idle(
|
||||
@@ -1813,6 +1859,15 @@ class CopienatorApp(tk.Tk):
|
||||
self.tree.selection_set(ids[target])
|
||||
self.tree.see(ids[target])
|
||||
|
||||
def _select_batch_integration(self) -> None:
|
||||
"""Prepare integration after batch results have been retrieved."""
|
||||
if self.runner.running or self.active_step_id or not self.state_store.evaluation:
|
||||
return
|
||||
self._save_current_form()
|
||||
self.state_store.update_step("correction", variant="integrate")
|
||||
self.tree.selection_set("correction")
|
||||
self.tree.see("correction")
|
||||
|
||||
def _select_refaire_correction(self) -> None:
|
||||
"""Return from manual conflict resolution to the main redo correction."""
|
||||
if self.runner.running or self.active_step_id or not self.state_store.evaluation:
|
||||
|
||||
@@ -29,7 +29,8 @@ c{43}2x : couper la source à 43 %, conserver la fin et remplacer la cible par l
|
||||
Cut permet de placer la coupe, avec accrochage entre les pages. Entrée ferme l’aperçu et affiche la commande à recopier ci-dessus : aucun fichier n’est modifié. Une séparation entre pages conserve les pages entières. La source complète est archivée en _old ; les deux labels modifiés sont générés en _new et ajoutés à refaire.json.
|
||||
|
||||
Pour les fusions : « Source x> Cible| » place la cible avant la source ; « Source x> |Cible » place la source avant la cible. Même règle avec -> et c{…}1>/c{…}2> (pour la partie transférée) ; sans |, la cible vient en premier.
|
||||
Vous pouvez changer l’opérateur, déplacer |, corriger les labels ou retirer une instruction. Les lignes vides et celles commençant par ### sont ignorées. Retirer/commenter une ligne ne résout pas son conflit.
|
||||
Si le PDF cible n’existe pas, il est considéré comme vide : la source (ou la partie transférée après une coupe) devient le nouveau PDF cible.
|
||||
Vous pouvez changer l’opérateur, déplacer |, corriger les labels ou retirer une instruction. Les lignes vides et celles commençant par ### sont ignorées. Retirer/commenter une ligne signifie conserver les PDF sans modification et considérer ce conflit comme résolu après Exécuter. Exécutez cette étape même si vous avez retiré toutes les instructions : ces conflits ne seront plus proposés lors de la reprise avec refaire.json.
|
||||
|
||||
Enregistrez dans l’éditeur, puis cliquez sur Recharger et enfin sur Exécuter. Seul le fichier enregistré est appliqué. L’archivage utilise le suffixe _old ; les PDF créés utilisent _new. Après succès, manual_resolutions.txt est supprimé et correction.json est mis à jour. Si des PDF sont créés, refaire.json est généré : relancez la correction avec --refaire."""
|
||||
|
||||
@@ -46,7 +47,10 @@ class ManualResolutionPanel(ttk.Frame):
|
||||
actions, text="Ouvrir dans un éditeur de texte", command=self.open_editor
|
||||
)
|
||||
self.editor_button.pack(side="left")
|
||||
ttk.Button(actions, text="Recharger", command=self.reload).pack(side="left", padx=6)
|
||||
self.reload_button = ttk.Button(
|
||||
actions, text="Recharger", command=self.reload
|
||||
)
|
||||
self.reload_button.pack(side="left", padx=6)
|
||||
self.cut_result = tk.StringVar()
|
||||
result = ttk.Frame(self)
|
||||
result.pack(fill="x", pady=4)
|
||||
@@ -116,10 +120,10 @@ class ManualResolutionPanel(ttk.Frame):
|
||||
else:
|
||||
if instructions:
|
||||
instruction = instructions[0]
|
||||
for title, label in (("PDF source", instruction.old_label),
|
||||
("PDF cible", instruction.new_label)):
|
||||
for title, label in (("Source", instruction.old_label),
|
||||
("Cible", instruction.new_label)):
|
||||
button = ttk.Button(
|
||||
self.text, text=title,
|
||||
self.text, text=f"{title} : {label}",
|
||||
command=lambda label=label, copy_id=instruction.copy_id, root=path.parent: self.open_pdf(root, copy_id, label),
|
||||
)
|
||||
self.pdf_buttons.append(button)
|
||||
|
||||
@@ -44,7 +44,6 @@ MODEL_PRO_ID = "gemini-3.8-flash"
|
||||
PAGE_SPLITTER_KB = {
|
||||
"move_left": "<Left>",
|
||||
"move_right": "<Right>",
|
||||
"confirm_next": "<Return>",
|
||||
"rotate_page": "r",
|
||||
"rotate_all_pages": "R",
|
||||
"rotate_all_files": ",",
|
||||
|
||||
@@ -133,7 +133,8 @@ class GuiConvenienceTests(unittest.TestCase):
|
||||
self.assertNotEqual(self.app.current_step.id, "giving_names")
|
||||
|
||||
def test_batch_status_waits_until_all_jobs_are_ready(self):
|
||||
self.app.state_store.update_step("correction", variant="batch")
|
||||
self.app.state_store.update_step("correction", variant="batch", last_run_variant="batch")
|
||||
self.app._populate_tree()
|
||||
self.app.state_store.update_step("batch_status", visited=True)
|
||||
self.app.tree.selection_set("batch_status")
|
||||
self.app.update()
|
||||
@@ -149,8 +150,59 @@ class GuiConvenienceTests(unittest.TestCase):
|
||||
self.app.update()
|
||||
self.assertEqual(self.app.current_step.id, "fetch_batches")
|
||||
|
||||
def test_batch_retrieval_success_selects_integration_without_starting_it(self):
|
||||
self.app.state_store.update_step("correction", variant="batch", last_run_variant="batch")
|
||||
self.app._populate_tree()
|
||||
self.app.tree.selection_set("fetch_batches")
|
||||
self.app.update()
|
||||
self.app.active_step_id = "fetch_batches"
|
||||
with patch.object(self.app.runner, "start") as start:
|
||||
self.app._finish_process(0, False)
|
||||
self.app.update()
|
||||
start.assert_not_called()
|
||||
self.assertEqual(self.app.current_step.id, "correction")
|
||||
self.assertEqual(self.app.variant_var.get(), "integrate")
|
||||
self.assertEqual(self.app.state_store.step("correction")["variant"], "integrate")
|
||||
self.assertIn("--deal-with-batched", self.app._make_command())
|
||||
|
||||
def test_integration_preserves_batch_statuses_but_invalidates_downstream_steps(self):
|
||||
for variant in ("integrate", "batch"):
|
||||
with self.subTest(variant=variant):
|
||||
self.app.active_step_id = None
|
||||
self.app.current_step = None
|
||||
self.app.state_store.update_step("correction", variant=variant, batch_started=True)
|
||||
statuses = {"submit_batches": "success", "batch_status": "detected",
|
||||
"fetch_batches": "success", "annotation": "success"}
|
||||
for step_id, status in statuses.items():
|
||||
self.app.state_store.update_step(step_id, status=status)
|
||||
self.app._populate_tree()
|
||||
self.app.tree.selection_set("correction")
|
||||
self.app.update()
|
||||
with patch.object(self.app.runner, "start"), \
|
||||
patch.object(self.app, "_missing_requirements", return_value=[]):
|
||||
self.app._run_current_step()
|
||||
self.assertEqual(self.app.active_step_id, "correction")
|
||||
for step_id, status in statuses.items():
|
||||
expected = status if variant == "integrate" and step_id != "annotation" else "stale"
|
||||
self.assertEqual(self.app.state_store.step(step_id)["status"], expected)
|
||||
self.app.active_step_id = None
|
||||
|
||||
def test_unsuccessful_batch_retrieval_stays_on_retrieval(self):
|
||||
self.app.state_store.update_step("correction", variant="batch", last_run_variant="batch")
|
||||
self.app._populate_tree()
|
||||
self.app.tree.selection_set("fetch_batches")
|
||||
self.app.update()
|
||||
for code, interrupted in ((4, False), (1, False), (0, True)):
|
||||
with self.subTest(code=code, interrupted=interrupted):
|
||||
self.app.active_step_id = "fetch_batches"
|
||||
self.app._finish_process(code, interrupted)
|
||||
self.app.update()
|
||||
self.assertEqual(self.app.current_step.id, "fetch_batches")
|
||||
self.assertEqual(self.app.state_store.step("correction")["variant"], "batch")
|
||||
|
||||
def test_batch_watch_buttons_use_loaded_evaluation_and_stop_on_reload(self):
|
||||
self.app.state_store.update_step("correction", variant="batch")
|
||||
self.app.state_store.update_step("correction", variant="batch", last_run_variant="batch")
|
||||
self.app._populate_tree()
|
||||
self.app.tree.selection_set("batch_status")
|
||||
self.app.update()
|
||||
with patch.object(self.app.batch_monitor, "start") as start:
|
||||
@@ -175,6 +227,97 @@ class GuiConvenienceTests(unittest.TestCase):
|
||||
self.assertEqual(self.app.current_step.id, "inputs")
|
||||
self.assertEqual(self.app.state_store.step("batch_status")["status"], "success")
|
||||
|
||||
def test_batch_steps_hidden_until_a_correction_is_started(self):
|
||||
batch_ids = ("submit_batches", "batch_status", "fetch_batches")
|
||||
self.app.state_store.update_step("correction", variant="batch")
|
||||
self.app._populate_tree()
|
||||
for step_id in batch_ids:
|
||||
self.assertFalse(self.app.tree.exists(step_id))
|
||||
self.assertNotIn(step_id, self.app._progression_ids("correction"))
|
||||
self.app.state_store.update_step("correction", last_run_variant="hybrid", status="running")
|
||||
self.app._populate_tree()
|
||||
for step_id in batch_ids:
|
||||
self.assertTrue(self.app.tree.exists(step_id))
|
||||
self.assertIn(step_id, self.app._progression_ids("correction"))
|
||||
|
||||
def test_existing_batch_artifacts_reveal_steps_even_in_live_mode(self):
|
||||
for filename in ("batch_requests_flash.jsonl", "batch_requests_pro.jsonl",
|
||||
"batch_jobs.json", "batched_correction_result.jsonl"):
|
||||
with self.subTest(filename=filename):
|
||||
path = self.evaluation / filename
|
||||
path.write_text("")
|
||||
self.app._populate_tree()
|
||||
self.assertFalse(self.app.tree.exists("batch_status"))
|
||||
path.write_text('{"jobs":{"flash":{"name":"batches/1"}}}\n')
|
||||
self.app._populate_tree()
|
||||
self.assertTrue(self.app.tree.exists("batch_status"))
|
||||
self.app._handle_first_visit(self.app.step_by_id["batch_status"])
|
||||
self.app.update()
|
||||
self.assertNotEqual(self.app.state_store.step("batch_status").get("status"), "skipped")
|
||||
path.unlink()
|
||||
|
||||
def test_started_batches_remain_visible_after_mode_change_and_reload(self):
|
||||
self.app.state_store.update_step("correction", batch_started=True, last_run_variant="live")
|
||||
self.app.state_store.load(self.evaluation)
|
||||
self.app._populate_tree()
|
||||
self.assertTrue(self.app.tree.exists("batch_status"))
|
||||
fresh = self.repository / "Fresh"
|
||||
fresh.mkdir()
|
||||
self.app.state_store.load(fresh)
|
||||
self.app._populate_tree()
|
||||
self.assertFalse(self.app.tree.exists("batch_status"))
|
||||
|
||||
def test_navigation_skips_hidden_batch_steps(self):
|
||||
with patch.object(self.app.tree, "selection_set") as select:
|
||||
self.app._move_selection_from("correction", 1)
|
||||
select.assert_called_once_with("manual_resolution")
|
||||
|
||||
def test_long_running_steps_notify_once_on_success(self):
|
||||
expected_text = {
|
||||
"labels": "labels",
|
||||
"correction": "correction",
|
||||
"annotation": "copies annotées",
|
||||
"read_annotations": "annotations manuscrites",
|
||||
}
|
||||
for step_id, completion_text in expected_text.items():
|
||||
with self.subTest(step=step_id), patch("copienator_gui.app.notify_desktop") as notify:
|
||||
self.app.active_step_id = step_id
|
||||
self.app._finish_process(0, False)
|
||||
self.app._finish_process(0, False)
|
||||
notify.assert_called_once()
|
||||
self.assertIn(self.evaluation.name, notify.call_args.args[1])
|
||||
self.assertIn(completion_text, notify.call_args.args[1])
|
||||
|
||||
def test_incomplete_failed_and_interrupted_steps_notify_the_outcome_once(self):
|
||||
for step_id in ("labels", "correction", "annotation", "read_annotations"):
|
||||
for code, interrupted, outcome in (
|
||||
(4, False, "partielle"), (1, False, "échouée"),
|
||||
(0, True, "interrompue"), (130, False, "interrompue"),
|
||||
):
|
||||
with self.subTest(step=step_id, code=code, interrupted=interrupted), \
|
||||
patch("copienator_gui.app.notify_desktop") as notify:
|
||||
self.app.active_step_id = step_id
|
||||
self.app._finish_process(code, interrupted)
|
||||
self.app._finish_process(code, interrupted)
|
||||
notify.assert_called_once()
|
||||
title, message = notify.call_args.args
|
||||
self.assertIn(outcome, title)
|
||||
self.assertIn(outcome, message)
|
||||
self.assertIn(self.evaluation.name, message)
|
||||
self.assertIn(f"code {code}", message)
|
||||
self.assertNotIn("réussie", message)
|
||||
|
||||
def test_notification_failure_keeps_success_and_uses_existing_fallback(self):
|
||||
self.app.active_step_id = "labels"
|
||||
with patch("copienator_gui.app.notify_desktop", side_effect=RuntimeError("unavailable")), \
|
||||
patch.object(self.app, "bell") as bell, \
|
||||
patch.object(self.app, "_append_console") as console:
|
||||
self.app._finish_process(0, False)
|
||||
bell.assert_called_once()
|
||||
self.assertTrue(any("Notification de bureau indisponible" in call.args[0]
|
||||
for call in console.call_args_list))
|
||||
self.assertEqual(self.app.state_store.step("labels")["status"], "success")
|
||||
|
||||
def test_optional_blank_crop_can_target_a_copy_or_be_skipped(self):
|
||||
copies = self.evaluation/"Copies"
|
||||
copies.mkdir()
|
||||
@@ -190,7 +333,9 @@ class GuiConvenienceTests(unittest.TestCase):
|
||||
command = self.app._make_command()
|
||||
self.assertEqual(command[-4:], ["crop-margins", str(source), "--workers", "5"])
|
||||
self.app._skip_step()
|
||||
self.app.update()
|
||||
self.assertEqual(self.app.state_store.step("crop_blank_margins")["status"], "skipped")
|
||||
self.assertNotEqual(self.app.current_step.id, "crop_blank_margins")
|
||||
|
||||
def test_optional_answer_bottom_crop_uses_parallel_workers_and_can_be_skipped(self):
|
||||
answers = self.evaluation / "Copies" / "Copie01"
|
||||
@@ -207,9 +352,11 @@ class GuiConvenienceTests(unittest.TestCase):
|
||||
["crop-answer-bottoms", self.app._evaluation_arg(), "--workers", "5"],
|
||||
)
|
||||
self.app._skip_step()
|
||||
self.app.update()
|
||||
self.assertEqual(
|
||||
self.app.state_store.step("crop_exercise_bottoms")["status"], "skipped"
|
||||
)
|
||||
self.assertNotEqual(self.app.current_step.id, "crop_exercise_bottoms")
|
||||
|
||||
def test_redo_targets_selected_copy_and_copies_runnable_command(self):
|
||||
for folder in ("Copies", "Copies Originales"):
|
||||
|
||||
+28
-4
@@ -19,6 +19,7 @@ from PIL import Image
|
||||
from pypdf import PdfReader, PdfWriter
|
||||
|
||||
from copienator import (
|
||||
CliError,
|
||||
EvaluationWorkspace,
|
||||
WorkspaceNotFoundError,
|
||||
WorkspaceValidationError,
|
||||
@@ -2058,7 +2059,23 @@ class StandardCliTests(unittest.TestCase):
|
||||
self.assertEqual(groups, [["Ex 1"], ["Ex 2"]])
|
||||
editor.assert_not_called()
|
||||
|
||||
def test_grouped_batching_does_not_split_one_student(self) -> None:
|
||||
def test_grouped_batching_looks_ahead_to_keep_below_the_hard_limit(self) -> None:
|
||||
module = self.modules["annotating_by_label"]
|
||||
large = Image.new("RGB", (10, 80), "white")
|
||||
small = Image.new("RGB", (10, 15), "white")
|
||||
rendered = [
|
||||
("01", "Ex 1", large, 0, []),
|
||||
("02", "Ex 1", small, 0, []),
|
||||
("02", "Ex 2", small, 0, []),
|
||||
]
|
||||
with patch.object(module, "MAX_HEIGHT_PX", 100):
|
||||
batches = module.split_batches(rendered)
|
||||
self.assertEqual([len(batch) for batch in batches], [1, 2])
|
||||
self.assertTrue(
|
||||
all(sum(item[2].height for item in batch) <= 100 for batch in batches)
|
||||
)
|
||||
|
||||
def test_grouped_batching_never_splits_one_student(self) -> None:
|
||||
module = self.modules["annotating_by_label"]
|
||||
image = Image.new("RGB", (10, 60), "white")
|
||||
rendered = [
|
||||
@@ -2066,9 +2083,16 @@ class StandardCliTests(unittest.TestCase):
|
||||
("01", "Ex 2", image, 0, []),
|
||||
("02", "Ex 1", image, 0, []),
|
||||
]
|
||||
with patch.object(module, "MAX_HEIGHT_PX", 100):
|
||||
batches = module.split_batches(rendered)
|
||||
self.assertEqual([len(batch) for batch in batches], [2, 1])
|
||||
with patch.object(module, "MAX_HEIGHT_PX", 100), self.assertRaises(
|
||||
CliError
|
||||
) as raised:
|
||||
module.split_batches(rendered)
|
||||
self.assertIn("Copie01", str(raised.exception))
|
||||
self.assertIn("cannot be split", str(raised.exception))
|
||||
|
||||
def test_grouped_annotation_height_limit_is_32000_pixels(self) -> None:
|
||||
module = self.modules["annotating_by_label"]
|
||||
self.assertEqual(module.MAX_HEIGHT_PX, 32000)
|
||||
|
||||
|
||||
class WorkflowTests(unittest.TestCase):
|
||||
|
||||
@@ -38,13 +38,31 @@ class ManualResolutionPanelTests(unittest.TestCase):
|
||||
self.assertEqual(panel.text.get("1.0", "end-1c"), content)
|
||||
self.assertEqual(len(panel.pdf_buttons), 4)
|
||||
self.assertEqual([button.cget("text") for button in panel.pdf_buttons],
|
||||
["PDF source", "PDF cible"] * 2)
|
||||
["Source : Ex 1", "Cible : Ex 2",
|
||||
"Source : Ex 3", "Cible : Ex 4"])
|
||||
with patch("copienator_gui.manual_resolution.open_path") as opened:
|
||||
panel.editor_button.invoke()
|
||||
for button in panel.pdf_buttons:
|
||||
button.invoke()
|
||||
self.assertEqual(opened.call_args_list, [call(self.path), *map(call, paths)])
|
||||
|
||||
def test_reload_updates_the_displayed_target_and_its_pdf_action(self):
|
||||
self.path.write_text("Copie01 A -> B|\n", encoding="utf-8")
|
||||
copies = self.evaluation / "Copies" / "Copie01"
|
||||
copies.mkdir(parents=True)
|
||||
(copies / "A.pdf").touch()
|
||||
(copies / "C.pdf").touch()
|
||||
panel = self.panel()
|
||||
|
||||
self.path.write_text("Copie01 A -> C|\n", encoding="utf-8")
|
||||
panel.reload_button.invoke()
|
||||
|
||||
self.assertEqual(panel.text.get("1.0", "end-1c"), "Copie01 A -> C|\n")
|
||||
self.assertEqual(panel.pdf_buttons[1].cget("text"), "Cible : C")
|
||||
with patch("copienator_gui.manual_resolution.open_path") as opened:
|
||||
panel.pdf_buttons[1].invoke()
|
||||
opened.assert_called_once_with(copies / "C.pdf")
|
||||
|
||||
def test_reload_keeps_invalid_lines_visible_and_removes_stale_actions(self):
|
||||
self.path.write_text("Copie01 A -> B|\n", encoding="utf-8")
|
||||
panel = self.panel()
|
||||
|
||||
@@ -6,11 +6,40 @@ from unittest.mock import Mock, patch
|
||||
|
||||
import pymupdf
|
||||
|
||||
from copienator import EvaluationWorkspace, atomic_write_json, read_json
|
||||
from copienator import CliError, EvaluationWorkspace, atomic_write_json, read_json
|
||||
from copienator.commands import correction, resolve_manual
|
||||
from copienator_gui.app import has_manual_conflicts
|
||||
|
||||
|
||||
class ManualResolutionStateTests(unittest.TestCase):
|
||||
def test_merge_into_missing_target_uses_source_and_schedules_redo(self):
|
||||
for operator in ("->", "x>"):
|
||||
for target in ("B|", "|B"):
|
||||
with self.subTest(operator=operator, target=target), tempfile.TemporaryDirectory() as temporary:
|
||||
workspace = self.fixture(Path(temporary), operator)
|
||||
copies = workspace.copies_dir / "Copie01"
|
||||
(copies / "B.pdf").unlink()
|
||||
workspace.manual_resolutions_file.write_text(f"Copie01 A {operator} {target}\n")
|
||||
resolve_manual.resolve_manual(workspace)
|
||||
with pymupdf.open(copies / "B_new.pdf") as document:
|
||||
self.assertEqual(len(document), 1)
|
||||
self.assertIn("A", document[0].get_text())
|
||||
self.assertEqual((copies / "A_old.pdf").exists(), operator == "x>")
|
||||
self.assertEqual((copies / "A.pdf").exists(), operator == "->")
|
||||
self.assertFalse((copies / "B_old.pdf").exists())
|
||||
self.assertEqual(read_json(workspace.refaire_file), [["Copie01", ["B"]]])
|
||||
self.assertNotIn("delayed", read_json(workspace.correction_file)["A"][0][0]["result"])
|
||||
|
||||
def test_merge_still_requires_source_without_changing_state(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
workspace = self.fixture(Path(temporary), "x>")
|
||||
(workspace.copies_dir / "Copie01" / "A.pdf").unlink()
|
||||
before = workspace.correction_file.read_bytes()
|
||||
with self.assertRaises(CliError):
|
||||
resolve_manual.resolve_manual(workspace)
|
||||
self.assertEqual(workspace.correction_file.read_bytes(), before)
|
||||
self.assertTrue(workspace.manual_resolutions_file.exists())
|
||||
|
||||
def test_acknowledgement_clears_only_this_target_and_copy(self):
|
||||
for error in ("wrg-lbl:B?", "wrg-lbl:B?delayed", "wrg-lbl:B?exists", "al:(->)B?(->)C?", "al:(delayed)B"):
|
||||
with self.subTest(error=error):
|
||||
@@ -63,3 +92,75 @@ class ManualResolutionStateTests(unittest.TestCase):
|
||||
self.assertEqual(correction.run_configured(args), 0)
|
||||
self.assertFalse(workspace.manual_resolutions_file.exists())
|
||||
self.assertNotIn("delayed", correction.results["A"][0][0]["result"])
|
||||
|
||||
def test_deleted_conflict_stays_resolved_when_other_resolution_generates_refaire(self):
|
||||
for kind in ("wrong-label", "add-label"):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temporary:
|
||||
workspace = self.fixture(Path(temporary), "x>")
|
||||
other = workspace.copies_dir / "Copie02"
|
||||
other.mkdir()
|
||||
for label in ("A", "B"):
|
||||
(other / f"{label}.pdf").write_bytes(
|
||||
(workspace.copies_dir / "Copie01" / f"{label}.pdf").read_bytes())
|
||||
before = {path.name: path.read_bytes() for path in other.iterdir()}
|
||||
results = read_json(workspace.correction_file)
|
||||
results["A"][0].append({"id": "02", "result": {
|
||||
"error": "wrg-lbl:B?" if kind == "wrong-label" else "al:(->)B?",
|
||||
"delayed": [[kind, "B"]], "score": 3,
|
||||
}})
|
||||
atomic_write_json(workspace.correction_file, results)
|
||||
# Only Copie01 remains in the edited file; Copie02 was deleted.
|
||||
resolve_manual.resolve_manual(workspace)
|
||||
self.assertEqual(read_json(workspace.refaire_file), [["Copie01", ["B"]]])
|
||||
self.assertEqual(before, {path.name: path.read_bytes() for path in other.iterdir()})
|
||||
saved = read_json(workspace.correction_file)["A"][0][1]["result"]
|
||||
self.assertNotIn("delayed", saved)
|
||||
self.assertEqual(saved["score"], 3)
|
||||
self.assertNotIn("B?", saved["error"])
|
||||
(workspace.groups_dir / "B").mkdir(parents=True)
|
||||
args = correction.build_parser().parse_args([str(workspace.root), "--refaire"])
|
||||
correction.configure_runtime(workspace, [], args, api_client=Mock())
|
||||
with patch.object(correction.grouping, "get_pdf_height", return_value=400), \
|
||||
patch.object(correction.grouping, "create_jpg"), \
|
||||
patch.object(correction, "process_single_task", return_value=[]):
|
||||
self.assertEqual(correction.run_configured(args), 0)
|
||||
self.assertFalse(workspace.manual_resolutions_file.exists())
|
||||
self.assertNotIn("delayed", correction.results["A"][0][1]["result"])
|
||||
|
||||
def test_deleting_all_lines_can_be_applied_and_survives_resume(self):
|
||||
for text in ("", "### Instructions\n", "### Copie01 A x> B|\n"):
|
||||
with self.subTest(text=text), tempfile.TemporaryDirectory() as temporary:
|
||||
workspace = self.fixture(Path(temporary), "x>")
|
||||
source = workspace.copies_dir / "Copie01"
|
||||
before = {path.name: path.read_bytes() for path in source.iterdir()}
|
||||
workspace.manual_resolutions_file.write_text(text)
|
||||
self.assertTrue(has_manual_conflicts(workspace.manual_resolutions_file))
|
||||
resolve_manual.resolve_manual(workspace)
|
||||
self.assertEqual(before, {path.name: path.read_bytes() for path in source.iterdir()})
|
||||
self.assertFalse(has_manual_conflicts(workspace.manual_resolutions_file))
|
||||
atomic_write_json(workspace.refaire_file, [])
|
||||
args = correction.build_parser().parse_args([str(workspace.root), "--refaire"])
|
||||
correction.configure_runtime(workspace, [], args, api_client=Mock())
|
||||
self.assertEqual(correction.run_configured(args), 0)
|
||||
self.assertFalse(workspace.manual_resolutions_file.exists())
|
||||
|
||||
def test_invalid_remaining_instruction_does_not_commit_deletions(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
workspace = self.fixture(Path(temporary), "x>")
|
||||
before = workspace.correction_file.read_bytes()
|
||||
workspace.manual_resolutions_file.write_text("invalid instruction\n")
|
||||
with self.assertRaises(CliError):
|
||||
resolve_manual.resolve_manual(workspace)
|
||||
self.assertEqual(workspace.correction_file.read_bytes(), before)
|
||||
|
||||
def test_generated_first_comment_documents_deleting_lines(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
workspace = self.fixture(Path(temporary), "x>")
|
||||
atomic_write_json(workspace.refaire_file, [])
|
||||
args = correction.build_parser().parse_args([str(workspace.root), "--refaire"])
|
||||
correction.configure_runtime(workspace, [], args, api_client=Mock())
|
||||
self.assertEqual(correction.run_configured(args), 0)
|
||||
first_line = workspace.manual_resolutions_file.read_text().splitlines()[0]
|
||||
self.assertTrue(first_line.startswith("###"))
|
||||
self.assertIn("delete a line", first_line)
|
||||
self.assertIn("resolve-manual", first_line)
|
||||
@@ -134,6 +134,42 @@ class SelectiveGeminiTests(unittest.TestCase):
|
||||
gemini.refine_existing(self.workspace, "persp", api_client=self.client)
|
||||
self.assertEqual(self.snapshot(), before)
|
||||
|
||||
def test_rubric_latex_and_line_breaks_survive_json_and_file_writes(self):
|
||||
expected = (
|
||||
r'Barème : $f \circ g \rightarrow \frac{1}{2} \not= 0$.'
|
||||
'\n' + r'$\begin{matrix}a & b \\ c & d\end{matrix}$, $\theta$, $\beta$, $\nu$.'
|
||||
'\nUne "justification" vaut 1 point.'
|
||||
)
|
||||
self.client.models.generate_content.side_effect = [
|
||||
self.response({'rubrics': [{'label': label, 'rubric_content': expected} for label in group]})
|
||||
for group in (self.labels[:2], self.labels[2:])
|
||||
]
|
||||
gemini.refine_existing(self.workspace, 'persp', api_client=self.client)
|
||||
for label in self.labels:
|
||||
self.assertEqual((self.root / 'Persp' / label).read_bytes(),
|
||||
(label + '\n' + expected).encode('utf-8'))
|
||||
|
||||
def test_invalid_latex_escapes_are_repaired_without_retry(self):
|
||||
for command in (r'\circ', r'\rightarrow', r'\frac', r'\not', r'\theta', r'\beta'):
|
||||
with self.subTest(command=command):
|
||||
# Deliberately malformed wire JSON: do not json.dumps the command.
|
||||
broken = SimpleNamespace(text='{"rubrics":[{"label":"Ex 2","rubric_content":"$'
|
||||
+ command + '$"}]}')
|
||||
self.client.models.generate_content.reset_mock()
|
||||
self.client.models.generate_content.side_effect = [broken]
|
||||
self.assertEqual(gemini.generate_rubrics(self.client, 'context'),
|
||||
{'Ex 2': '$' + command + '$'})
|
||||
self.assertEqual(self.client.models.generate_content.call_count, 1)
|
||||
|
||||
def test_unrepairable_control_character_preserves_all_existing_rubrics(self):
|
||||
before = self.snapshot()
|
||||
broken = self.response({'rubrics': [{'label': 'Ex 2', 'rubric_content': 'invalid \x01 control'}]})
|
||||
self.client.models.generate_content.side_effect = [
|
||||
self.rubric_response(self.labels[:2]), broken]
|
||||
with self.assertRaisesRegex(ValueError, 'control characters'):
|
||||
gemini.refine_existing(self.workspace, 'persp', api_client=self.client)
|
||||
self.assertEqual(self.snapshot(), before)
|
||||
|
||||
def test_cli_dispatches_selective_modes_without_full_extraction(self):
|
||||
with patch.object(gemini, "refine_existing", return_value=ExitCode.SUCCESS) as refine, patch.object(
|
||||
gemini, "process_exam"
|
||||
|
||||
@@ -27,9 +27,26 @@ class CompileToPdfTests(unittest.TestCase):
|
||||
utils.compile_to_pdf("broken", output)
|
||||
|
||||
self.assertEqual(output.read_bytes(), b"partial")
|
||||
self.assertIn("Warning: LaTeX compilation failed", console.getvalue())
|
||||
self.assertIn("Warning: LaTeX compilation reported errors", console.getvalue())
|
||||
self.assertIn("a PDF was generated", console.getvalue())
|
||||
self.assertIn("missing \\item", console.getvalue())
|
||||
|
||||
def test_says_no_pdf_was_generated_when_latex_fails(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
output = Path(temporary) / "result.pdf"
|
||||
result = subprocess.CompletedProcess(
|
||||
args=[], returncode=1, stdout="! Undefined control sequence.\n"
|
||||
)
|
||||
|
||||
console = io.StringIO()
|
||||
with patch.object(utils.subprocess, "run", return_value=result), \
|
||||
contextlib.redirect_stdout(console):
|
||||
utils.compile_to_pdf("broken", output)
|
||||
|
||||
self.assertFalse(output.exists())
|
||||
self.assertIn("no PDF was generated", console.getvalue())
|
||||
self.assertIn("Undefined control sequence", console.getvalue())
|
||||
|
||||
def test_warns_when_no_pdf_is_produced(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
output = Path(temporary) / "result.pdf"
|
||||
|
||||
Reference in new issue
Block a user