Standardisation 2

This commit is contained in:
2026-08-20 14:17:32 +02:00
parent 0a95afacdd
commit 63f690b353
8 changed files with 578 additions and 308 deletions
+10 -2
View File
@@ -167,8 +167,11 @@ Les codes de sortie communs sont :
| 130 | interruption par l'utilisateur | | 130 | interruption par l'utilisateur |
Le GUI distingue notamment un traitement partiel d'un échec. Les Le GUI distingue notamment un traitement partiel d'un échec. Les
premiers scripts migrés vers cette convention sont =export.py=, scripts migrés vers cette convention sont actuellement :
=import.py=, =giving_names.py= et =grouping.py=.
- =copies_tools.py=, =grouping.py= et =verify_groups.py= ;
- =post-correction.py= et =resolve_manual.py= ;
- =export.py=, =import.py= et =giving_names.py=.
** Correction d'un paquet de copies ** Correction d'un paquet de copies
@@ -289,6 +292,11 @@ Optional : Set proxy with ~export HTTPS_PROXY="http://10.0.0.1:3128"~
Regroupe les mêmes questions de différentes copies en groupes de Regroupe les mêmes questions de différentes copies en groupes de
tailles raisonnables. tailles raisonnables.
5. Facultatif : =python verify_groups.py Interro=
Vérifie que chaque réponse PDF apparaît bien dans les métadonnées
des groupes. La commande renvoie un code non nul si une réponse est
absente ou si la vérification est incomplète.
** Correction et annotation ** Correction et annotation
+2
View File
@@ -6,6 +6,7 @@ from .cli import (
evaluation_parser, evaluation_parser,
evaluation_workspace, evaluation_workspace,
execute, execute,
standard_parser,
workspace_from_args, workspace_from_args,
) )
from .json_io import ( from .json_io import (
@@ -35,5 +36,6 @@ __all__ = [
"evaluation_workspace", "evaluation_workspace",
"execute", "execute",
"read_json", "read_json",
"standard_parser",
"workspace_from_args", "workspace_from_args",
] ]
+11 -6
View File
@@ -30,13 +30,8 @@ class CliError(Exception):
super().__init__(message) super().__init__(message)
def evaluation_parser(description: str) -> argparse.ArgumentParser: def standard_parser(description: str) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=description) parser = argparse.ArgumentParser(description=description)
parser.add_argument(
"evaluation",
type=Path,
help="Evaluation directory",
)
parser.add_argument( parser.add_argument(
"--verbose", "--verbose",
action="store_true", action="store_true",
@@ -45,6 +40,16 @@ def evaluation_parser(description: str) -> argparse.ArgumentParser:
return parser return parser
def evaluation_parser(description: str) -> argparse.ArgumentParser:
parser = standard_parser(description)
parser.add_argument(
"evaluation",
type=Path,
help="Evaluation directory",
)
return parser
def evaluation_workspace( def evaluation_workspace(
path: str | Path, path: str | Path,
*, *,
+10
View File
@@ -212,6 +212,16 @@ def build_workflow(show_personal_steps: bool) -> list[StepDefinition]:
requires=("Copies",), requires=("Copies",),
artifacts=("Par label",), artifacts=("Par label",),
), ),
StepDefinition(
"verify_groups",
"Labels et regroupement",
"Vérifier les groupes produits",
"Vérifie que chaque réponse PDF apparaît dans les métadonnées des groupes.",
(python("default", "Vérification des groupes", "verify_groups.py"),),
arguments=(arg_target("Dossier de l’évaluation"),),
optional=True,
requires=("Copies", "Par label"),
),
StepDefinition( StepDefinition(
"correction", "correction",
"Correction", "Correction",
+35 -8
View File
@@ -2,10 +2,19 @@ from __future__ import annotations
import argparse import argparse
import uuid import uuid
from collections.abc import Sequence
from pathlib import Path from pathlib import Path
from pypdf import PdfReader, PdfWriter from pypdf import PdfReader, PdfWriter
from copienator import (
EvaluationWorkspace,
ExitCode,
execute,
standard_parser,
workspace_from_args,
)
def copy_pdfs(directory: Path) -> list[Path]: def copy_pdfs(directory: Path) -> list[Path]:
directory = directory.expanduser().resolve() directory = directory.expanduser().resolve()
@@ -91,19 +100,37 @@ def rename_all(directory: Path) -> list[tuple[Path, Path]]:
return [(source, destination) for source, _temporary, destination in staged] return [(source, destination) for source, _temporary, destination in staged]
def main() -> None: def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Prepare scanned PDF copies.") parser = standard_parser("Prepare scanned PDF copies.")
subparsers = parser.add_subparsers(dest="operation", required=True) subparsers = parser.add_subparsers(dest="operation", required=True)
for operation in ("rotate", "rename"): for operation in ("rotate", "rename"):
subparser = subparsers.add_parser(operation) subparser = subparsers.add_parser(operation)
subparser.add_argument("directory", type=Path) subparser.add_argument("evaluation", type=Path, help="Evaluation directory")
args = parser.parse_args() subparser.add_argument(
"--verbose",
action="store_true",
default=argparse.SUPPRESS,
help="Show a traceback when an unexpected error occurs",
)
return parser
if args.operation == "rotate":
rotate_all(args.directory) def run(workspace: EvaluationWorkspace, *, operation: str) -> ExitCode:
if operation == "rotate":
rotate_all(workspace.root)
else: else:
rename_all(args.directory) rename_all(workspace.root)
return ExitCode.SUCCESS
def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
return execute(
parser,
argv,
lambda args: run(workspace_from_args(args), operation=args.operation),
)
if __name__ == "__main__": if __name__ == "__main__":
main() raise SystemExit(main())
+92 -129
View File
@@ -1,155 +1,118 @@
import sys from __future__ import annotations
import os
import time
from pathlib import Path
import argparse import argparse
import re
from collections.abc import Sequence
from pathlib import Path
from typing import Any
from copienator import atomic_write_json from copienator import (
EvaluationWorkspace,
if len(sys.argv) < 2: ExitCode,
sys.exit("Usage: python script.py <InputDir>") atomic_write_json,
evaluation_parser,
def escape_latex_underscores(text): execute,
r""" read_json,
Escape '_' outside LaTeX math environments. workspace_from_args,
Supports:
- $...$
- $$...$$
- \( ... \)
- \[ ... \]
"""
# Regex matching LaTeX math blocks
math_pattern = re.compile(
r'(\$\$.*?\$\$|' # $$...$$
r'\$.*?\$|' # $...$
r'\\\(.*?\\\)|' # \( ... \)
r'\\\[.*?\\\])', # \[ ... \]
re.DOTALL
) )
parts = [] WORD_LIST_FILE = Path(__file__).with_name("liste_francais.txt")
last_end = 0 ACCENT_PATTERN = re.compile(r"[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]")
def build_parser() -> argparse.ArgumentParser:
return evaluation_parser("Clean encoding and LaTeX issues in correction.json.")
def escape_latex_underscores(text: str) -> str:
r"""Escape underscores outside LaTeX math environments."""
math_pattern = re.compile(
r"(\$\$.*?\$\$|\$.*?\$|\\\(.*?\\\)|\\\[.*?\\\])",
re.DOTALL,
)
parts: list[str] = []
last_end = 0
for match in math_pattern.finditer(text): for match in math_pattern.finditer(text):
start, end = match.span() start, end = match.span()
parts.append(text[last_end:start].replace("_", r"\_"))
# Escape underscores outside math
outside = text[last_end:start].replace('_', r'\_')
parts.append(outside)
# Keep math block unchanged
parts.append(match.group(0)) parts.append(match.group(0))
last_end = end last_end = end
parts.append(text[last_end:].replace("_", r"\_"))
return "".join(parts)
# Remaining text after last math block
outside = text[last_end:].replace('_', r'\_')
parts.append(outside)
return ''.join(parts) def build_lookup_map(word_list_path: Path = WORD_LIST_FILE) -> dict[str, str]:
words = word_list_path.read_text(encoding="utf-8").splitlines()
lookup: dict[str, str] = {}
for word in words:
broken_key = ACCENT_PATTERN.sub("\x00", word)
if "\x00" in broken_key:
lookup[broken_key.lower()] = word
return lookup
arg_path = Path(sys.argv[1])
tasks = [] # List of tuples: (filepath_str, label_str)
results = {}
INPUT_DIR = str(arg_path) def fast_fix(text: str, lookup: dict[str, str]) -> str:
if not arg_path.exists(): def replacer(match: re.Match[str]) -> str:
sys.exit(f"Directory {INPUT_DIR} not found.")
import json
import ftfy
import re
import urllib.request
with open('liste_francais.txt', 'r') as f:
french_words = f.read().splitlines()
# 2. Pre-compute an O(1) lookup dictionary
# We simulate the corruption by replacing accents with null bytes (\x00)
lookup_map = {}
for word in french_words:
# Replace all French accents with \x00 to create the "broken" key
broken_key = re.sub(r'[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]', '\x00', word)
if '\x00' in broken_key:
lookup_map[broken_key] = word # e.g., "\x00cole" -> "école"
# 3. Fast replace function
def fast_fix(text):
# Find words containing regular letters and null bytes
def replacer(match):
broken_word = match.group(0) broken_word = match.group(0)
# Return the fixed word from our map, or leave it if not found return lookup.get(broken_word.lower(), broken_word)
# (Handles case-insensitivity by falling back to lowercase map)
fixed = lookup_map.get(broken_word.lower())
# if not fixed:
# print(f"No match found for: {repr(broken_word)}")
return fixed or broken_word
return re.sub(r'[a-zA-Z\x00]+', replacer, text) return re.sub(r"[a-zA-Z\x00]+", replacer, text)
# return text
INPUT_FILE = Path(INPUT_DIR) / "correction.json" def fix_hex_corruption_safe(text: str) -> str:
OUTPUT_FILE = Path(INPUT_DIR) / "correction.json" return re.sub(
r"\x00([eEfF][0-9a-fA-F])",
def fix_hex_corruption_safe(text): lambda match: chr(int(match.group(1), 16)),
# Only matches \x00 followed by hex if it results in an accented character text,
# or common Latin-1 symbols )
return re.sub(r'\x00([eEfF][0-9a-fA-F])',
lambda m: chr(int(m.group(1), 16)),
text)
def some_other_replacements(s):
s = s.replace("\neq", "\\neq")
s = s.replace("\not", "\\not")
return s
def clean_string(s: str) -> str: def some_other_replacements(text: str) -> str:
# fix encoding issues return text.replace("\neq", "\\neq").replace("\not", "\\not")
# s = ftfy.fix_text(s)
# print(s)
s = fix_hex_corruption_safe(s)
s = s.replace('\x19', '\x00')
s = s.replace('\x18', '\x00')
s = s.replace('\x00\x00', '\x00')
s = re.sub(r' \x00{1,2} ', ' à ', s)
if '\x00' in s:
s = fast_fix(s)
s = s.replace('\x00', '')
s = some_other_replacements(s)
return escape_latex_underscores(s)
def clean_obj(obj): def clean_string(text: str, lookup: dict[str, str]) -> str:
if isinstance(obj, str): text = fix_hex_corruption_safe(text)
return clean_string(obj) text = text.replace("\x19", "\x00")
text = text.replace("\x18", "\x00")
elif isinstance(obj, list): text = text.replace("\x00\x00", "\x00")
return [clean_obj(x) for x in obj] text = re.sub(r" \x00{1,2} ", " à ", text)
if "\x00" in text:
elif isinstance(obj, dict): text = fast_fix(text, lookup).replace("\x00", "")
r = {} return escape_latex_underscores(some_other_replacements(text))
for k, v in obj.items():
if k != "suffix":
r[k] = clean_obj(v)
else:
r[k] = v
return r
else:
return obj
with open(INPUT_FILE, "r", encoding="utf-8") as f: def clean_obj(value: Any, lookup: dict[str, str]) -> Any:
data = json.load(f) if isinstance(value, str):
return clean_string(value, lookup)
if isinstance(value, list):
return [clean_obj(item, lookup) for item in value]
if isinstance(value, dict):
return {
key: item if key == "suffix" else clean_obj(item, lookup)
for key, item in value.items()
}
return value
data = clean_obj(data)
atomic_write_json(OUTPUT_FILE, data) def run(
workspace: EvaluationWorkspace,
*,
word_list_path: Path = WORD_LIST_FILE,
) -> ExitCode:
workspace.require_files("correction.json")
lookup = build_lookup_map(word_list_path)
data = read_json(workspace.correction_file)
cleaned = clean_obj(data, lookup)
atomic_write_json(workspace.correction_file, cleaned)
print(f"Fixed JSON saved to {workspace.correction_file}")
return ExitCode.SUCCESS
print("Fixed JSON saved to", OUTPUT_FILE)
def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
return execute(parser, argv, lambda args: run(workspace_from_args(args)))
if __name__ == "__main__":
raise SystemExit(main())
+236 -152
View File
@@ -1,196 +1,280 @@
import sys from __future__ import annotations
import os
import json import argparse
import re import re
import shutil import shutil
from collections.abc import Sequence
from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any
from pypdf import PdfWriter from pypdf import PdfWriter
from copienator import atomic_write_json from copienator import (
CliError,
EvaluationWorkspace,
ExitCode,
atomic_write_json,
evaluation_parser,
execute,
read_json,
workspace_from_args,
)
if len(sys.argv) < 2: OPERATORS = ("-x", "->", "x>", "ss", "sx", "xx", "xs")
sys.exit("Usage: python resolve_manual.py <InputDir>") OPERATOR_PATTERN = re.compile(r"\s+(-x|->|x>|ss|sx|xx|xs)\s+")
COPY_PATTERN = re.compile(r"Copie(\d+)\s+(.+)")
input_dir = Path(sys.argv[1])
manual_file = input_dir / "manual_resolutions.txt"
correction_file = input_dir / "correction.json"
refaire_file = input_dir / "refaire.json"
copies_dir = input_dir / "Copies"
if not manual_file.exists(): @dataclass(frozen=True, slots=True)
sys.exit(f"No {manual_file.name} found. Nothing to resolve.") class ManualInstruction:
copy_id: str
old_label: str
operator: str
new_label: str
pipe_first: bool
with open(correction_file, "r", encoding="utf-8") as f: @property
results = json.load(f) def should_merge(self) -> bool:
return self.operator.endswith(">")
def set_suffix_and_clean_error(pid, label, suffix, new_lbl_target=None): @property
"""Updates correction.json to set suffixes and clear resolved delayed tags.""" def should_copy(self) -> bool:
if label in results: return not self.should_merge and "s" not in self.operator
for batch in results[label]:
def build_parser() -> argparse.ArgumentParser:
return evaluation_parser("Apply the instructions from manual_resolutions.txt.")
def parse_instructions(path: Path) -> list[ManualInstruction]:
instructions: list[ManualInstruction] = []
malformed: list[int] = []
for line_number, raw_line in enumerate(
path.read_text(encoding="utf-8").splitlines(), start=1
):
line = raw_line.strip()
if not line or line.startswith("###"):
continue
operator_match = OPERATOR_PATTERN.search(line)
if operator_match is None:
malformed.append(line_number)
continue
left = line[: operator_match.start()].strip()
right = line[operator_match.end() :].strip()
copy_match = COPY_PATTERN.fullmatch(left)
new_label = right.strip("|").strip()
if copy_match is None or not new_label:
malformed.append(line_number)
continue
instructions.append(
ManualInstruction(
copy_id=copy_match.group(1),
old_label=copy_match.group(2).strip(),
operator=operator_match.group(1),
new_label=new_label,
pipe_first=right.startswith("|"),
)
)
if malformed:
lines = ", ".join(str(number) for number in malformed)
raise CliError(f"Malformed manual resolution instruction at line(s): {lines}")
return instructions
def set_suffix_and_clean_error(
results: dict[str, Any],
copy_id: str,
label: str,
suffix: str | None,
new_label_target: str | None = None,
) -> None:
for batch in results.get(label, []):
for item in batch: for item in batch:
if item["id"] == pid: if item["id"] != copy_id:
continue
if suffix: if suffix:
item["result"]["suffix"] = suffix item["result"]["suffix"] = suffix
err = item["result"].get("error", "") error = item["result"].get("error", "")
if new_lbl_target: if new_label_target:
if f"wrg-lbl:{new_lbl_target}?delayed" in err: if f"wrg-lbl:{new_label_target}?delayed" in error:
item["result"]["error"] = f"wrg-lbl-moved-to:{new_lbl_target}" item["result"]["error"] = (
if f"(delayed){new_lbl_target}" in err: f"wrg-lbl-moved-to:{new_label_target}"
item["result"]["error"] = err.replace(f"(delayed){new_lbl_target}", f"(->){new_lbl_target}") )
if f"(delayed){new_label_target}" in error:
item["result"]["error"] = error.replace(
f"(delayed){new_label_target}",
f"(->){new_label_target}",
)
def get_actual_pdf(pid, label):
"""Finds the effective PDF considering possible suffixes.""" def get_actual_pdf(copies_dir: Path, copy_id: str, label: str) -> Path:
base = copies_dir / f"Copie{pid}" / f"{label}.pdf" base = copies_dir / f"Copie{copy_id}" / f"{label}.pdf"
if base.exists(): return base for candidate in (
if base.with_name(f"{label}_new.pdf").exists(): return base.with_name(f"{label}_new.pdf") base,
if base.with_name(f"{label}_old.pdf").exists(): return base.with_name(f"{label}_old.pdf") base.with_name(f"{label}_new.pdf"),
base.with_name(f"{label}_old.pdf"),
):
if candidate.exists():
return candidate
return base return base
def safe_strip_suffix(stem):
if stem.endswith("_new"): return stem[:-4] def safe_strip_suffix(stem: str) -> str:
if stem.endswith("_old"): return stem[:-4] if stem.endswith(("_new", "_old")):
return stem[:-4]
return stem return stem
instructions = []
with open(manual_file, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line or line.startswith("###"): continue
# Regex to split on the operator properly handles spaces in labels def _validate_pdf_inputs(
match = re.search(r'\s+(-x|->|x>)\s+', line) instructions: list[ManualInstruction],
if not match: initial_paths: dict[tuple[str, str], Path],
print(f"Skipping malformed line: {line}") ) -> None:
continue missing: set[Path] = set()
for instruction in instructions:
source = initial_paths[(instruction.copy_id, instruction.old_label)]
destination = initial_paths[(instruction.copy_id, instruction.new_label)]
if instruction.should_merge:
if not source.exists():
missing.add(source)
if not destination.exists():
missing.add(destination)
elif instruction.should_copy and not source.exists():
missing.add(source)
if missing:
rendered = ", ".join(str(path) for path in sorted(missing))
raise CliError(f"PDF input(s) required by manual resolutions not found: {rendered}")
op = match.group(1)
left = line[:match.start()].strip()
right = line[match.end():].strip()
m_left = re.match(r'Copie(\d+)\s+(.+)', left) def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
if not m_left: workspace.require_files("manual_resolutions.txt", "correction.json")
continue workspace.require_directories("Copies")
loaded = read_json(workspace.correction_file)
if not isinstance(loaded, dict):
raise CliError("correction.json must contain a JSON object")
results: dict[str, Any] = loaded
instructions = parse_instructions(workspace.manual_resolutions_file)
pid = m_left.group(1) initial_paths: dict[tuple[str, str], Path] = {}
old_label = m_left.group(2).strip() current_paths: dict[tuple[str, str], Path] = {}
new_part = right for instruction in instructions:
for label in (instruction.old_label, instruction.new_label):
key = (instruction.copy_id, label)
if key not in initial_paths:
path = get_actual_pdf(workspace.copies_dir, *key)
initial_paths[key] = path
current_paths[key] = path
_validate_pdf_inputs(instructions, initial_paths)
pipe_first = new_part.startswith("|") files_to_old: set[Path] = set()
# pipe_last = new_part.endswith("|") temp_files: list[Path] = []
new_label = new_part.strip("|").strip() try:
for instruction in instructions:
key_old = (instruction.copy_id, instruction.old_label)
key_new = (instruction.copy_id, instruction.new_label)
source = initial_paths[key_old]
destination = current_paths[key_new]
temp_output = (
workspace.copies_dir
/ f"Copie{instruction.copy_id}"
/ f"temp_{len(temp_files)}.pdf"
)
instructions.append((pid, old_label, op, new_label, pipe_first)) if instruction.operator.startswith("x"):
files_to_old.add(initial_paths[key_old])
if instruction.operator.endswith("x"):
files_to_old.add(initial_paths[key_new])
# State trackers if instruction.should_merge:
initial_paths = {} # Tracks the source files at script start (enables safe Swapping) writer = PdfWriter()
current_paths = {} # Tracks the "latest active file" (which may be a temp file for chained merges) try:
files_to_old = set() if instruction.pipe_first:
temp_files = [] writer.append(source)
writer.append(destination)
# Pre-load existing paths
for pid, old_label, _, new_label, _ in instructions:
for lbl in (old_label, new_label):
if (pid, lbl) not in initial_paths:
p = get_actual_pdf(pid, lbl)
initial_paths[(pid, lbl)] = p
current_paths[(pid, lbl)] = p
# Evaluate instructions
for pid, old_label, op, new_label, pipe_first in instructions:
should_merge = op[1] == ">"
src_pdf = initial_paths[(pid, old_label)]
dest_pdf = current_paths[(pid, new_label)]
temp_out = copies_dir / f"Copie{pid}" / f"temp_{len(temp_files)}.pdf"
if op[0] == "x":
files_to_old.add(initial_paths[(pid, old_label)])
if op[1] == "x":
files_to_old.add(initial_paths[(pid, new_label)])
if should_merge:
if not dest_pdf.exists() or not src_pdf.exists():
print("Debug : should_merge but, {} or {} doesn't exist"
.format(src_pdf, dest_pdf))
input("You should Ctrl-C and fix.")
# MERGE
merger = PdfWriter()
if pipe_first:
merger.append(src_pdf)
merger.append(dest_pdf)
else: else:
merger.append(dest_pdf) writer.append(destination)
merger.append(src_pdf) writer.append(source)
merger.write(temp_out) writer.write(temp_output)
merger.close() except Exception:
current_paths[(pid, new_label)] = temp_out temp_output.unlink(missing_ok=True)
temp_files.append(temp_out) raise
# Original destination is now embedded in the merge, back it up finally:
files_to_old.add(initial_paths[(pid, new_label)]) writer.close()
else: current_paths[key_new] = temp_output
if op[1] != "s" and op[0] != "s": # xx or -x temp_files.append(temp_output)
shutil.copy(src_pdf, temp_out) files_to_old.add(initial_paths[key_new])
current_paths[(pid, new_label)] = temp_out elif instruction.should_copy:
temp_files.append(temp_out) shutil.copy(source, temp_output)
current_paths[key_new] = temp_output
temp_files.append(temp_output)
except Exception:
for temporary in temp_files:
temporary.unlink(missing_ok=True)
raise
# Commit Filesystem Changes
# 1. Archive deprecated files to _old.pdf
for pdf in files_to_old: for pdf in files_to_old:
if pdf.exists(): if not pdf.exists():
pid_str = pdf.parent.name.replace("Copie", "") continue
copy_id = pdf.parent.name.removeprefix("Copie")
label = safe_strip_suffix(pdf.stem) label = safe_strip_suffix(pdf.stem)
old_name = pdf.with_name(f"{label}_old.pdf") old_name = pdf.with_name(f"{label}_old.pdf")
if pdf != old_name: if pdf != old_name:
if old_name.exists(): old_name.unlink() old_name.unlink(missing_ok=True)
shutil.move(str(pdf), str(old_name)) shutil.move(str(pdf), str(old_name))
set_suffix_and_clean_error(results, copy_id, label, "_old")
set_suffix_and_clean_error(pid_str, label, "_old") for instruction in instructions:
set_suffix_and_clean_error(
results,
instruction.copy_id,
instruction.old_label,
None,
instruction.new_label,
)
# 2. Clear all delayed errors for the instructions refaire_by_copy: dict[str, list[str]] = {}
for pid, old_label, op, new_label, pf in instructions: for (copy_id, label), current_path in current_paths.items():
set_suffix_and_clean_error(pid, old_label, None, new_label) if not current_path.name.startswith("temp_"):
continue
refaire_tasks = [] final_name = workspace.copies_dir / f"Copie{copy_id}" / f"{label}_new.pdf"
final_name.unlink(missing_ok=True)
# 3. Rename active temp files to _new.pdf and queue for --refaire
for (pid, label), current_path in current_paths.items():
if "temp_" in current_path.name:
final_name = copies_dir / f"Copie{pid}" / f"{label}_new.pdf"
if final_name.exists(): final_name.unlink()
shutil.move(str(current_path), str(final_name)) shutil.move(str(current_path), str(final_name))
set_suffix_and_clean_error(results, copy_id, label, "_new")
labels = refaire_by_copy.setdefault(f"Copie{copy_id}", [])
if label not in labels:
labels.append(label)
set_suffix_and_clean_error(pid, label, "_new")
# Push to refaire_tasks uniquely
added = False
for t in refaire_tasks:
if t[0] == f"Copie{pid}":
if label not in t[1]:
t[1].append(label)
added = True
break
if not added:
refaire_tasks.append([f"Copie{pid}", [label]])
# 4. Clean up any unused temp files (overwritten by chained replacements)
used_temps = set(current_paths.values()) used_temps = set(current_paths.values())
for temp in temp_files: for temporary in temp_files:
if temp not in used_temps and temp.exists(): if temporary not in used_temps:
temp.unlink() temporary.unlink(missing_ok=True)
# Finalize JSONs
atomic_write_json(correction_file, results)
atomic_write_json(workspace.correction_file, results)
refaire_tasks = [[copy_name, labels] for copy_name, labels in refaire_by_copy.items()]
if refaire_tasks: if refaire_tasks:
atomic_write_json(refaire_file, refaire_tasks) atomic_write_json(workspace.refaire_file, refaire_tasks)
workspace.manual_resolutions_file.unlink()
manual_file.unlink(missing_ok=True)
print("Manual resolutions successfully applied.") print("Manual resolutions successfully applied.")
if refaire_tasks: if refaire_tasks:
print(f"File {refaire_file.name} generated. Run `python correction.py \"{input_dir}\" --refaire` to process updates.") print(
f"File {workspace.refaire_file.name} generated. Run "
f'`python correction.py "{workspace.command_argument()}" --refaire` '
"to process updates."
)
else: else:
print("No new corrections required.") print("No new corrections required.")
return ExitCode.SUCCESS
def run(workspace: EvaluationWorkspace) -> ExitCode:
return resolve_manual(workspace)
def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
return execute(parser, argv, lambda args: run(workspace_from_args(args)))
if __name__ == "__main__":
raise SystemExit(main())
+171
View File
@@ -45,7 +45,12 @@ def load_script_module(filename: str, module_name: str):
if spec is None or spec.loader is None: if spec is None or spec.loader is None:
raise RuntimeError(f"Could not load {filename}") raise RuntimeError(f"Could not load {filename}")
module = importlib.util.module_from_spec(spec) module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
try:
spec.loader.exec_module(module) spec.loader.exec_module(module)
except Exception:
sys.modules.pop(module_name, None)
raise
return module return module
@@ -155,12 +160,24 @@ class StandardCliTests(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls) -> None: def setUpClass(cls) -> None:
cls.modules = { cls.modules = {
"copies_tools": load_script_module(
"copies_tools.py", "copienator_copies_tools_test"
),
"export": load_script_module("export.py", "copienator_export_test"), "export": load_script_module("export.py", "copienator_export_test"),
"import": load_script_module("import.py", "copienator_import_test"), "import": load_script_module("import.py", "copienator_import_test"),
"giving_names": load_script_module( "giving_names": load_script_module(
"giving_names.py", "copienator_giving_names_test" "giving_names.py", "copienator_giving_names_test"
), ),
"grouping": load_script_module("grouping.py", "copienator_grouping_test"), "grouping": load_script_module("grouping.py", "copienator_grouping_test"),
"post_correction": load_script_module(
"post-correction.py", "copienator_post_correction_test"
),
"resolve_manual": load_script_module(
"resolve_manual.py", "copienator_resolve_manual_test"
),
"verify_groups": load_script_module(
"verify_groups.py", "copienator_verify_groups_test"
),
} }
def test_missing_evaluation_has_standard_exit_code(self) -> None: def test_missing_evaluation_has_standard_exit_code(self) -> None:
@@ -171,6 +188,10 @@ class StandardCliTests(unittest.TestCase):
"import": [missing], "import": [missing],
"giving_names": [missing, "BGnot"], "giving_names": [missing, "BGnot"],
"grouping": [missing], "grouping": [missing],
"post_correction": [missing],
"resolve_manual": [missing],
"verify_groups": [missing],
"copies_tools": ["rotate", missing],
} }
for name, arguments in invocations.items(): for name, arguments in invocations.items():
with self.subTest(script=name), redirect_stderr(io.StringIO()): with self.subTest(script=name), redirect_stderr(io.StringIO()):
@@ -223,6 +244,21 @@ class StandardCliTests(unittest.TestCase):
{"target": evaluation, "annotation_dir": "BGnot"}, {"target": evaluation, "annotation_dir": "BGnot"},
), ),
"grouping": ("grouping", "default", {"target": evaluation}), "grouping": ("grouping", "default", {"target": evaluation}),
"post_correction": (
"post_correction",
"default",
{"target": evaluation},
),
"resolve_manual": (
"manual_resolution",
"default",
{"target": evaluation},
),
"verify_groups": (
"verify_groups",
"default",
{"target": evaluation},
),
} }
for module_name, (step_id, variant_id, values) in cases.items(): for module_name, (step_id, variant_id, values) in cases.items():
step = steps[step_id] step = steps[step_id]
@@ -232,6 +268,23 @@ class StandardCliTests(unittest.TestCase):
parsed = self.modules[module_name].build_parser().parse_args(command[3:]) parsed = self.modules[module_name].build_parser().parse_args(command[3:])
self.assertEqual(str(parsed.evaluation), evaluation) self.assertEqual(str(parsed.evaluation), evaluation)
for step_id in ("rotate", "rename"):
step = steps[step_id]
variant = step.variants[0]
command = build_command(
REPOSITORY,
step,
variant,
{"target": evaluation},
evaluation,
)
with self.subTest(script=f"copies_tools:{step_id}"):
parsed = self.modules["copies_tools"].build_parser().parse_args(
command[3:]
)
self.assertEqual(parsed.operation, step_id)
self.assertEqual(str(parsed.evaluation), evaluation)
def test_export_main_copies_outputs(self) -> None: def test_export_main_copies_outputs(self) -> None:
module = self.modules["export"] module = self.modules["export"]
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
@@ -287,6 +340,124 @@ class StandardCliTests(unittest.TestCase):
self.assertEqual(module.main([str(evaluation)]), 0) self.assertEqual(module.main([str(evaluation)]), 0)
self.assertTrue((evaluation / "Par label").is_dir()) self.assertTrue((evaluation / "Par label").is_dir())
def test_post_correction_main_cleans_json_atomically(self) -> None:
module = self.modules["post_correction"]
with tempfile.TemporaryDirectory() as directory:
evaluation = Path(directory) / "Exam"
evaluation.mkdir()
atomic_write_json(
evaluation / "correction.json",
{"text": "outside_name and $math_name$", "suffix": "_new"},
)
self.assertEqual(module.main([str(evaluation)]), 0)
self.assertEqual(
read_json(evaluation / "correction.json"),
{"text": r"outside\_name and $math_name$", "suffix": "_new"},
)
def test_manual_resolution_with_no_actions_is_safe(self) -> None:
module = self.modules["resolve_manual"]
with tempfile.TemporaryDirectory() as directory:
evaluation = Path(directory) / "Exam"
(evaluation / "Copies").mkdir(parents=True)
atomic_write_json(evaluation / "correction.json", {})
manual = evaluation / "manual_resolutions.txt"
manual.write_text("### Nothing to do\n", encoding="utf-8")
self.assertEqual(module.main([str(evaluation)]), 0)
self.assertFalse(manual.exists())
self.assertEqual(read_json(evaluation / "correction.json"), {})
def test_malformed_manual_resolution_is_not_deleted(self) -> None:
module = self.modules["resolve_manual"]
with tempfile.TemporaryDirectory() as directory:
evaluation = Path(directory) / "Exam"
(evaluation / "Copies").mkdir(parents=True)
correction = evaluation / "correction.json"
atomic_write_json(correction, {})
manual = evaluation / "manual_resolutions.txt"
manual.write_text("this is malformed\n", encoding="utf-8")
before = correction.read_bytes()
with redirect_stderr(io.StringIO()):
self.assertEqual(module.main([str(evaluation)]), 1)
self.assertTrue(manual.exists())
self.assertEqual(correction.read_bytes(), before)
def test_documented_manual_resolution_operators_are_parsed(self) -> None:
module = self.modules["resolve_manual"]
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "manual_resolutions.txt"
path.write_text(
"\n".join(
f"Copie01 Old label {operator} New label"
for operator in module.OPERATORS
),
encoding="utf-8",
)
instructions = module.parse_instructions(path)
self.assertEqual(
[instruction.operator for instruction in instructions],
list(module.OPERATORS),
)
def test_replace_manual_resolution_creates_refaire_state(self) -> None:
module = self.modules["resolve_manual"]
with tempfile.TemporaryDirectory() as directory:
evaluation = Path(directory) / "Exam"
copy_dir = evaluation / "Copies" / "Copie01"
copy_dir.mkdir(parents=True)
(copy_dir / "Old.pdf").write_bytes(b"old answer")
(copy_dir / "New.pdf").write_bytes(b"replaced answer")
atomic_write_json(
evaluation / "correction.json",
{
"Old": [
[
{
"id": "01",
"result": {"error": "wrg-lbl:New?delayed"},
}
]
],
"New": [[{"id": "01", "result": {"error": ""}}]],
},
)
(evaluation / "manual_resolutions.txt").write_text(
"Copie01 Old -x New\n", encoding="utf-8"
)
self.assertEqual(module.main([str(evaluation)]), 0)
self.assertEqual((copy_dir / "New_old.pdf").read_bytes(), b"replaced answer")
self.assertEqual((copy_dir / "New_new.pdf").read_bytes(), b"old answer")
self.assertEqual(
read_json(evaluation / "refaire.json"),
[["Copie01", ["New"]]],
)
correction = read_json(evaluation / "correction.json")
self.assertEqual(
correction["Old"][0][0]["result"]["error"],
"wrg-lbl-moved-to:New",
)
self.assertEqual(correction["New"][0][0]["result"]["suffix"], "_new")
def test_verify_groups_reports_success_and_missing_answers(self) -> None:
module = self.modules["verify_groups"]
with tempfile.TemporaryDirectory() as directory:
evaluation = Path(directory) / "Exam"
copy_dir = evaluation / "Copies" / "Copie01"
group_dir = evaluation / "Par label" / "Ex 1"
copy_dir.mkdir(parents=True)
group_dir.mkdir(parents=True)
(copy_dir / "Ex 1.pdf").write_bytes(b"pdf")
self.assertEqual(module.main([str(evaluation)]), 1)
atomic_write_json(
group_dir / "Group_1.json",
[["01", 0, 100, 1.0, "Ex 1"]],
)
self.assertEqual(module.main([str(evaluation)]), 0)
class WorkflowTests(unittest.TestCase): class WorkflowTests(unittest.TestCase):
def setUp(self) -> None: def setUp(self) -> None: