sebastien/Copies
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

miscs (Interro02) : horizontal cutting resolution

Browse Source
This commit is contained in:
sebastien
2026-09-14 22:20:46 +02:00
parent 5080274e8f
commit 0a86403ca6
28 changed files with 1447 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
copienator/commands/annotating.py
+11
View File
@@ -32,6 +32,7 @@ from copienator import (
from copienator.annotation_data import load_annotation_data
from copienator.answer_info import build_answer_info
from copienator.filesystem import staged_directory
from copienator.feedback_boxes import valid_feedback_box
from copienator.utils import natural_key
MARGIN_LEFT = 300
@@ -326,6 +327,16 @@ def compose_label_image(base_img, label, result, hmin,
# Filter deleted items (used by reading_annotations.py)
feedbacks = [f for f in feedbacks if "to_delete" not in f]
# Never guess where an invalid rectangle belongs, or lose its comment.
# Use a copy so rendering cannot mutate saved correction data.
normalized = []
for feedback in feedbacks:
box = feedback.get("box_2d")
if box is not None and not valid_feedback_box(box):
print(f"Warning: Copie{with_id or ''} {label}: invalid feedback box {box!r}; displaying the comment without a rectangle.")
feedback = {**feedback, "box_2d": None}
normalized.append(feedback)
feedbacks = normalized
global_fb = [f for f in feedbacks if not f.get('box_2d')]
local_fb = [f for f in feedbacks if f.get('box_2d')]
copienator/commands/batch_status.py
+45
View File
@@ -9,10 +9,13 @@ from google import genai
from copienator import configuration as config
from copienator import (
CliError,
EvaluationWorkspace,
ExitCode,
atomic_write_bytes,
execute,
read_json,
standard_parser,
workspace_from_args,
)
@@ -43,6 +46,41 @@ def list_jobs(*, client=None) -> ExitCode:
return ExitCode.SUCCESS
def check_evaluation_jobs(workspace: EvaluationWorkspace, *, client=None) -> ExitCode:
"""Only report readiness after checking every job recorded for this evaluation."""
if not workspace.batch_jobs_file.is_file():
print("Impossible de vérifier les batchs : batch_jobs.json est absent.")
return ExitCode.PARTIAL
manifest = read_json(workspace.batch_jobs_file)
jobs = manifest.get("jobs") if isinstance(manifest, dict) else None
if not isinstance(jobs, dict):
raise CliError(f"Invalid batch manifest: {workspace.batch_jobs_file}")
if not jobs:
print("Aucun job enregistré pour cette évaluation.")
return ExitCode.PARTIAL
if any(not isinstance(entry, dict) or not isinstance(entry.get("name"), str)
or not entry["name"].strip() for entry in jobs.values()):
raise CliError(f"Invalid batch job in {workspace.batch_jobs_file}")
client = client or _client()
ready = True
for tier, entry in jobs.items():
job = client.batches.get(name=entry["name"])
state = job.state.name if hasattr(job.state, "name") else job.state
print(f"{tier} — {entry['name']}: {state}")
if state != "JOB_STATE_SUCCEEDED":
ready = False
if getattr(job, "error", None):
print(f" Erreur : {job.error}")
elif not getattr(getattr(job, "dest", None), "file_name", None):
ready = False
print(" Le fichier de résultats n’est pas encore disponible.")
if ready:
print("Tous les batchs de l’évaluation ont réussi. Les résultats sont prêts à récupérer.")
return ExitCode.SUCCESS
print("Les résultats ne sont pas tous prêts. Consultez à nouveau cette étape plus tard.")
return ExitCode.PARTIAL
def download_job(
job_name: str,
*,
@@ -73,6 +111,8 @@ def build_parser() -> argparse.ArgumentParser:
parser = standard_parser("List or download Gemini correction batch jobs")
parser.add_argument("--download", metavar="JOB_NAME")
parser.add_argument("--output", type=Path, help="Downloaded JSONL destination")
parser.add_argument("--evaluation", type=Path,
help="Check readiness of jobs recorded in this evaluation's batch_jobs.json")
return parser
@@ -80,6 +120,11 @@ def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
def handle(args: argparse.Namespace) -> ExitCode:
if args.evaluation is not None:
if args.download or args.output is not None:
raise CliError("--evaluation cannot be combined with --download or --output",
ExitCode.INVALID_ARGUMENTS)
return check_evaluation_jobs(workspace_from_args(args))
if args.output is not None and not args.download:
raise CliError("--output requires --download", ExitCode.INVALID_ARGUMENTS)
if args.download:
copienator/commands/correction.py
+13 -4
View File
@@ -18,6 +18,7 @@ from google import genai
from copienator import configuration as config
from copienator.commands import grouping
from copienator import prompting
from copienator.feedback_boxes import valid_feedback_box
from copienator import (
CliError,
EvaluationWorkspace,
@@ -267,6 +268,9 @@ def correct_boxes_with_gemini(pid, label, pdf_path, original_feedbacks,
for f in corrected_feedbacks:
b = f.get("box_2d")
if b:
if not valid_feedback_box(b) or any(value < 0 or value > 1000 for value in b):
f["box_2d"] = None
continue
ymin_s, xmin_s, ymax_s, xmax_s = b
# Y mapping: Add the group Y-offset (yming), then normalize to total_height
@@ -475,6 +479,9 @@ def process_single_task(task_tuple, precomputed_response=None):
for (i,f) in enumerate(res["feedback"]):
b = f.get("box_2d")
if b:
if not valid_feedback_box(b):
needs_correction.append(i)
continue
ymin, _xmin, ymax, xmax = b
ymin = ymin * total_height // 1000
ymax = ymax * total_height // 1000
@@ -484,9 +491,11 @@ def process_single_task(task_tuple, precomputed_response=None):
pid, label, group_name)
continue
if (ymin < yming - 50 or ymax > ymaxg + 50 or xmax / 1000 > width_r):
if (ymin < yming - 50 or ymax > ymaxg + 50
or ymin > ymaxg + 50 or ymax < yming - 50
or _xmin < 0 or xmax / 1000 > width_r):
needs_correction.append(i)
break
continue
if ymin < yming - 5:
ymin = yming - 5
b[0] = ymin * 1000 // total_height
@@ -852,7 +861,7 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
# Check for remaining unresolved delayed tasks
unresolved_delayed = []
with io_lock:
for label, batches in results.items():
for label, batches in sorted(results.items()):
for batch in batches:
for p in batch:
res = p.get("result", {})
@@ -869,7 +878,7 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
manual_path = INPUT_DIR / "manual_resolutions.txt"
atomic_write_text(
manual_path,
"### Use -> x>, -x, ss, sx, xx, xs\n"
"### Use -> x>, -x, ss, sx, xx, xs, c{43}1>, c{43}2x\n"
+ "\n".join(unresolved_delayed)
+ "\n",
)
copienator/commands/cutleft.py
+3 -1
View File
@@ -65,7 +65,9 @@ def stitch_images(image_list: list[Image.Image]) -> Image.Image | None:
@lru_cache(maxsize=3)
def _get_pdf_pages_cached(pdf_path: Path) -> list[Image.Image]:
return convert_from_path(pdf_path)
# Label coordinates use the full, displayed MediaBox, including on PDFs
# previously cropped by crop-margins. Keep this in sync with split-answers.
return convert_from_path(pdf_path, use_cropbox=False)
def get_pdf_pages(pdf_path: Path) -> list[Image.Image]:
copienator/commands/post_correction.py
+4
View File
@@ -9,6 +9,7 @@ from typing import Any
from copienator import (
EvaluationWorkspace,
ExitCode,
atomic_write_bytes,
atomic_write_json,
evaluation_parser,
execute,
@@ -104,6 +105,9 @@ def run(
lookup = build_lookup_map(word_list_path)
data = read_json(workspace.correction_file)
cleaned = clean_obj(data, lookup)
backup = workspace.root / "correction_precleanup.json"
atomic_write_bytes(backup, workspace.correction_file.read_bytes())
print(f"Original JSON backed up to {backup}")
atomic_write_json(workspace.correction_file, cleaned)
print(f"Fixed JSON saved to {workspace.correction_file}")
return ExitCode.SUCCESS
copienator/commands/resolve_manual.py
+48 -12
View File
@@ -9,6 +9,7 @@ from pathlib import Path
from typing import Any
from pypdf import PdfWriter
from copienator.pdf_cut import split_pdf
from copienator import (
CliError,
@@ -22,7 +23,8 @@ from copienator import (
)
OPERATORS = ("-x", "->", "x>", "ss", "sx", "xx", "xs")
OPERATOR_PATTERN = re.compile(r"\s+(-x|->|x>|ss|sx|xx|xs)\s+")
CUT_PATTERN = re.compile(r"c\{(\d+(?:\.\d+)?)\}([12])([x>])")
OPERATOR_PATTERN = re.compile(r"\s+(-x|->|x>|ss|sx|xx|xs|c\{\d+(?:\.\d+)?\}[12][x>])\s+")
COPY_PATTERN = re.compile(r"Copie(\d+)\s+(.+)")
@@ -34,6 +36,11 @@ class ManualInstruction:
new_label: str
pipe_first: bool
@property
def cut(self) -> tuple[float, int] | None:
match = CUT_PATTERN.fullmatch(self.operator)
return (float(match[1]), int(match[2])) if match else None
@property
def should_merge(self) -> bool:
return self.operator.endswith(">")
@@ -48,10 +55,14 @@ def build_parser() -> argparse.ArgumentParser:
def parse_instructions(path: Path) -> list[ManualInstruction]:
return parse_instruction_text(path.read_text(encoding="utf-8"))
def parse_instruction_text(text: str) -> list[ManualInstruction]:
instructions: list[ManualInstruction] = []
malformed: list[int] = []
for line_number, raw_line in enumerate(
path.read_text(encoding="utf-8").splitlines(), start=1
text.splitlines(), start=1
):
line = raw_line.strip()
if not line or line.startswith("###"):
@@ -64,7 +75,10 @@ def parse_instructions(path: Path) -> list[ManualInstruction]:
right = line[operator_match.end() :].strip()
copy_match = COPY_PATTERN.fullmatch(left)
new_label = right.strip("|").strip()
if copy_match is None or not new_label:
cut_match = CUT_PATTERN.fullmatch(operator_match.group(1))
if (copy_match is None or not new_label
or (cut_match and (not 0 < float(cut_match[1]) < 100
or copy_match.group(2).strip() == new_label))):
malformed.append(line_number)
continue
instructions.append(
@@ -97,15 +111,25 @@ def set_suffix_and_clean_error(
item["result"]["suffix"] = suffix
error = item["result"].get("error", "")
if new_label_target:
if f"wrg-lbl:{new_label_target}?delayed" in error:
item["result"]["error"] = (
f"wrg-lbl-moved-to:{new_label_target}"
)
if f"(delayed){new_label_target}" in error:
item["result"]["error"] = error.replace(
f"(delayed){new_label_target}",
f"(->){new_label_target}",
)
# This instruction acknowledges this source/target conflict,
# including decisions to keep/discard PDFs without merging.
# Leave other pending targets (and other copies) untouched.
result = item["result"]
if "delayed" in result:
pending = [entry for entry in result["delayed"]
if entry not in (["wrong-label", new_label_target],
["add-label", new_label_target])]
if pending:
result["delayed"] = pending
else:
result.pop("delayed")
if error in {f"wrg-lbl:{new_label_target}?",
f"wrg-lbl:{new_label_target}?delayed",
f"wrg-lbl:{new_label_target}?exists"}:
error = f"wrg-lbl-moved-to:{new_label_target}"
error = error.replace(f"(delayed){new_label_target}", f"(->){new_label_target}")
error = error.replace(f"(->){new_label_target}?", f"(->){new_label_target}")
result["error"] = error
def get_actual_pdf(copies_dir: Path, copy_id: str, label: str) -> Path:
@@ -154,6 +178,9 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
raise CliError("correction.json must contain a JSON object")
results: dict[str, Any] = loaded
instructions = parse_instructions(workspace.manual_resolutions_file)
cut_sources = [(item.copy_id, item.old_label) for item in instructions if item.cut]
if len(set(cut_sources)) != len(cut_sources):
raise CliError("Une seule coupe par label source est autorisée dans une résolution.")
initial_paths: dict[tuple[str, str], Path] = {}
current_paths: dict[tuple[str, str], Path] = {}
@@ -174,6 +201,15 @@ def resolve_manual(workspace: EvaluationWorkspace) -> ExitCode:
key_new = (instruction.copy_id, instruction.new_label)
source = initial_paths[key_old]
destination = current_paths[key_new]
if instruction.cut:
percent, keep = instruction.cut
first = source.parent / f"temp_{len(temp_files)}.pdf"
second = source.parent / f"temp_{len(temp_files) + 1}.pdf"
temp_files.extend((first, second))
split_pdf(source, percent, first, second)
retained, source = (first, second) if keep == 1 else (second, first)
current_paths[key_old] = retained
files_to_old.add(initial_paths[key_old])
temp_output = (
workspace.copies_dir
/ f"Copie{instruction.copy_id}"
copienator/commands/splitting_int.py
+33 -17
View File
@@ -80,15 +80,9 @@ def _save_cropped_page(
y1: float,
output_path: Path,
) -> None:
page = document[page_number]
rotated_rectangle = page.rect * page.transformation_matrix
visual_crop = pymupdf.Rect(
rotated_rectangle.x0 + x0,
y0,
rotated_rectangle.x0 + x1,
y1,
)
unrotated_clip = visual_crop * page.derotation_matrix
# The source has been normalized by _prepare_split_pages: no rotation or
# CropBox translation remains in the coordinates passed to show_pdf_page.
visual_crop = pymupdf.Rect(x0, y0, x1, y1)
cropped = pymupdf.open()
try:
target_page = cropped.new_page(width=visual_crop.width, height=visual_crop.height)
@@ -96,14 +90,32 @@ def _save_cropped_page(
target_page.rect,
document,
page_number,
rotate=-page.rotation,
clip=unrotated_clip,
clip=visual_crop,
)
cropped.save(output_path)
finally:
cropped.close()
def _prepare_split_pages(document: pymupdf.Document) -> list[pymupdf.Rect]:
"""Use the label preview's full-page coordinates; retain visible bounds.
Work only on the in-memory document. Baking rotation into the content after
restoring the MediaBox avoids show_pdf_page's rotated CropBox offsets.
Intersecting with the saved bounds later preserves prior margin cropping.
"""
visible_bounds = []
for page in document:
crop = page.cropbox
media = page.mediabox
full_crop = pymupdf.Rect(media.x0, 0, media.x1, media.height)
page.set_cropbox(full_crop)
crop -= (full_crop.x0, full_crop.y0, full_crop.x0, full_crop.y0)
visible_bounds.append(crop * page.rotation_matrix)
page.remove_rotation()
return visible_bounds
def _render_split_outputs(
input_pdf: Path,
coords_list: list[Coordinate],
@@ -112,6 +124,7 @@ def _render_split_outputs(
"""Render every current answer into an otherwise empty staging directory."""
document = pymupdf.open(input_pdf)
try:
visible_bounds = _prepare_split_pages(document)
parsed = _parse_coordinates(coords_list)
parts_by_label: defaultdict[str, list[Path]] = defaultdict(list)
with tempfile.TemporaryDirectory(prefix="copienator-split-") as temp_directory:
@@ -160,16 +173,20 @@ def _render_split_outputs(
page = document[page_number]
y0 = (y_start / 1000) * page.rect.height if page_number == start_page else 0
y1 = (end_y / 1000) * page.rect.height if page_number == end_page else page.rect.height
if y1 <= y0 + 1:
clip = pymupdf.Rect(
fraction_x0 * page.rect.width, y0,
fraction_x1 * page.rect.width, y1,
) & visible_bounds[page_number] & page.rect
if clip.is_empty or clip.height <= 1 or clip.width <= 1:
continue
part_path = temporary / f"part-{index}-{page_number}.pdf"
_save_cropped_page(
document,
page_number,
fraction_x0 * page.rect.width,
y0,
fraction_x1 * page.rect.width,
y1,
clip.x0,
clip.y0,
clip.x1,
clip.y1,
part_path,
)
parts_by_label[clean_label].append(part_path)
@@ -264,4 +281,3 @@ def main(argv: Sequence[str] | None = None) -> int:
if __name__ == "__main__":
raise SystemExit(main())