améliorations diverses

This commit is contained in:
2026-09-17 22:19:05 +02:00
parent 5b8215e7f5
commit 0a453403cf
11 changed files with 1072 additions and 81 deletions
+176 -21
View File
@@ -5,6 +5,7 @@ import base64
import concurrent.futures
import json
import os
import signal
import shlex
import shutil
import sys
@@ -45,15 +46,22 @@ COPIES_DIR = Path()
GROUPS_DIR = Path()
output_path = Path()
progress_path = Path()
pending_responses_path = Path()
tasks: list[tuple] = []
tasks_to_process: list[tuple] = []
results: dict = {}
completed_tasks: list = []
errors_summary: list = []
pending_responses: dict[str, str] = {}
overwrite = False
limit = None
client = None
start_time = 0.0
stop_requested = threading.Event()
class CorrectionStopRequested(Exception):
"""Raised in a worker before it starts another Gemini request."""
# --- Thread-safe Logging ---
log_lock = threading.Lock()
@@ -82,6 +90,25 @@ def flush_thread_log(tid=None):
f.write("\n".join(thread_logs[tid]) + "\n\n")
thread_logs[tid].clear()
def report_group_progress(completed: int, total: int) -> None:
"""Emit a stable, human-readable progress line for the GUI and CLI."""
print(
f"[Progression correction] Groupes traités : {completed}/{total}",
flush=True,
)
def request_graceful_stop(_signum=None, _frame=None) -> None:
"""Stop scheduling Gemini calls while allowing in-flight calls to finish."""
if not stop_requested.is_set():
stop_requested.set()
print(
"\n[Interruption] Arrêt des nouveaux appels Gemini demandé. "
"Attente des appels en cours et sauvegarde de leurs résultats…",
flush=True,
)
# --- Lock for thread-safe file writing ---
io_lock = threading.Lock()
pro_lock = threading.Lock()
@@ -140,6 +167,7 @@ def configure_runtime(
api_client=None,
) -> None:
global INPUT_DIR, COPIES_DIR, GROUPS_DIR, output_path, progress_path
global pending_responses_path, pending_responses
global tasks, tasks_to_process, results, completed_tasks, errors_summary
global overwrite, limit, client, start_time
global pro_count, flash_count, pro_quota_exhausted
@@ -149,11 +177,13 @@ def configure_runtime(
GROUPS_DIR = workspace.groups_dir
output_path = workspace.correction_file
progress_path = workspace.correction_progress_file
pending_responses_path = workspace.correction_pending_responses_file
tasks = list(discovered_tasks)
overwrite = bool(args.overwrite)
limit = args.limit
start_time = time.time()
errors_summary = []
pending_responses = {}
completed_tasks = []
results = {label: [] for _file, label in tasks}
thread_logs.clear()
@@ -161,6 +191,7 @@ def configure_runtime(
pro_count = 0
flash_count = 0
pro_quota_exhausted = False
stop_requested.clear()
if not overwrite:
if progress_path.is_file():
@@ -174,6 +205,20 @@ def configure_runtime(
raise TypeError("correction.json must contain a JSON object")
results = loaded_results
# A response saved during a graceful interruption is not a completed
# correction. Resume its auxiliary checks even with --overwrite instead of
# paying for the same primary request again.
if pending_responses_path.is_file():
loaded_pending = read_json(pending_responses_path)
if not isinstance(loaded_pending, dict) or not all(
isinstance(key, str) and isinstance(value, str)
for key, value in loaded_pending.items()
):
raise TypeError(
"correction_pending_responses.json must contain a JSON object"
)
pending_responses = loaded_pending
completed_set = {(str(file_path), label) for file_path, label in completed_tasks}
tasks_to_process = [
task for task in tasks if (str(task[0]), task[1]) not in completed_set
@@ -184,7 +229,11 @@ def configure_runtime(
def reset_workspace(workspace: EvaluationWorkspace) -> None:
"""Apply the explicitly requested correction reset."""
print("--- Running Reset ---")
for path in (workspace.correction_file, workspace.correction_progress_file):
for path in (
workspace.correction_file,
workspace.correction_progress_file,
workspace.correction_pending_responses_file,
):
if path.exists():
path.unlink()
print(f"Deleted: {path}")
@@ -213,6 +262,8 @@ def call_gemini_with_retries(model_id, contents, config,
delays = [60, 300]
for attempt in range(3):
if stop_requested.is_set():
raise CorrectionStopRequested
# Switch to fallback immediately if quota was exhausted by another thread
if model_id == MODEL_ID_pro and pro_quota_exhausted and fallback_model_id:
model_id = fallback_model_id
@@ -228,6 +279,8 @@ def call_gemini_with_retries(model_id, contents, config,
full_response_text += chunk.text
return full_response_text
except Exception as e:
if stop_requested.is_set():
raise CorrectionStopRequested from e
error_msg = str(e).lower()
is_quota_error = "429" in error_msg or "quota" in error_msg or "exhausted" in error_msg
is_minute_limit = "minute" in error_msg or "rpm" in error_msg or "tpm" in error_msg
@@ -239,7 +292,8 @@ def call_gemini_with_retries(model_id, contents, config,
wait_time = float(retry_match.group(1)) + 1.0 if retry_match else delays[attempt]
tprint(f"\tGemini Pro minute limit hit. Waiting {wait_time:.1f}s...")
time.sleep(wait_time)
if stop_requested.wait(wait_time):
raise CorrectionStopRequested
continue # Retry same model
# Immediately fallback to Flash without waiting if it's a Pro quota error
@@ -251,7 +305,8 @@ def call_gemini_with_retries(model_id, contents, config,
if attempt < 2:
tprint(f"\tGemini API failure: {e}. Retrying in {delays[attempt]} seconds...")
time.sleep(delays[attempt])
if stop_requested.wait(delays[attempt]):
raise CorrectionStopRequested
else:
tprint(f"\tGemini API failure: {e}. Maximum retries reached.")
raise
@@ -359,6 +414,8 @@ def handle_label_errors(pid, label, res, pdf_path):
tprint(f"\tHandling additional-answer for {pid} {label}")
try:
add_labels = json.loads(call_gemini_with_retries(MODEL_ID_flash, contents, config))
except CorrectionStopRequested:
raise
except Exception: # noqa: BLE001 - invalid auxiliary model response
add_labels = []
@@ -417,6 +474,9 @@ def process_single_task(task_tuple, precomputed_response=None):
total_height = group_data[-1][2]
use_flash = n >= 4 or total_height <= 500
if precomputed_response is None and stop_requested.is_set():
raise CorrectionStopRequested
# Only apply limits and counts if we are making a live call
if precomputed_response is None:
if not use_flash:
@@ -437,9 +497,11 @@ def process_single_task(task_tuple, precomputed_response=None):
model_to_use = MODEL_ID_flash if use_flash else MODEL_ID_pro
if precomputed_response:
tprint(f"Using batched response for: {label} {group_name}")
tprint(f"Using saved response for: {label} {group_name}")
full_response_text = precomputed_response
else:
if stop_requested.is_set():
raise CorrectionStopRequested
tprint(f"Asking Gemini {'Flash' if use_flash else 'Pro '}: {label} {group_name}")
full_response_text = call_gemini_with_retries(model_to_use, contents, config)
@@ -478,8 +540,28 @@ def process_single_task(task_tuple, precomputed_response=None):
if res["error"] != "":
tprint("\tError :", res["error"], "for Copie", pid, group_name)
if can_spawn_tasks and res.get("error") in ["wrong-label", "additional-answer"]:
new_tasks.extend(handle_label_errors(pid, label, res, pdf_path))
if can_spawn_tasks and res.get("error") in [
"wrong-label",
"additional-answer",
]:
if stop_requested.is_set():
with io_lock:
pending_responses[file_path] = json.dumps(json_data)
atomic_write_json(
pending_responses_path, pending_responses
)
raise CorrectionStopRequested
try:
new_tasks.extend(
handle_label_errors(pid, label, res, pdf_path)
)
except CorrectionStopRequested:
with io_lock:
pending_responses[file_path] = json.dumps(json_data)
atomic_write_json(
pending_responses_path, pending_responses
)
raise
# Si "wrong-label" a déplacé le fichier courant vers _old
if res.get("error", "").startswith("wrg-lbl-moved-to:"):
current_suffix = "_old"
@@ -520,6 +602,8 @@ def process_single_task(task_tuple, precomputed_response=None):
if needs_correction:
tprint(f"\tBox anomalies detected for Copie {pid} {group_name}. \n\tRequesting isolated correction from Gemini Flash...")
try:
if stop_requested.is_set():
raise CorrectionStopRequested
# Pensez à passer pdf_path à la fonction modifiée !
res["feedback"] = correct_boxes_with_gemini(
pid, label, pdf_path, res["feedback"],
@@ -542,7 +626,12 @@ def process_single_task(task_tuple, precomputed_response=None):
# To track progress
completed_tasks.append((file_path, label))
atomic_write_json(progress_path, completed_tasks)
if file_path in pending_responses:
del pending_responses[file_path]
atomic_write_json(pending_responses_path, pending_responses)
except CorrectionStopRequested:
raise
except json.JSONDecodeError:
tprint(f"Error decoding JSON for {file_path}", file=sys.stderr)
with io_lock:
@@ -836,36 +925,82 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
else:
print(f"Warning: Batch results file {batch_results_path} not found.", file=sys.stderr)
report_live_progress = not any(
(args.batch, args.batch_from, args.deal_with_batched, args.refaire)
)
progress_total = len(tasks)
progress_completed = max(0, progress_total - len(tasks_to_process))
if report_live_progress:
report_group_progress(progress_completed, progress_total)
made_progress = True
while tasks_to_process or made_progress:
if tasks_to_process:
print(f"Starting processing on {len(tasks_to_process)} tasks with {NB_THREADS} threads...")
with concurrent.futures.ThreadPoolExecutor(max_workers=NB_THREADS) as executor:
waiting_tasks = list(tasks_to_process)
futures = {}
for task in tasks_to_process:
file_path = task[0]
precomp = batched_responses.get(file_path)
futures[executor.submit(process_single_task, task, precomp)] = task
for future in concurrent.futures.as_completed(futures):
try:
new_generated_tasks = future.result()
if new_generated_tasks:
for new_task in new_generated_tasks:
futures[executor.submit(process_single_task, new_task)] = new_task
except Exception as e: # noqa: BLE001 - future boundary
print(f"Exception during task execution: {e}", file=sys.stderr)
failed_task = futures[future]
with io_lock:
errors_summary.append((str(e), failed_task[0]))
def submit_available_tasks() -> None:
while (
waiting_tasks
and len(futures) < NB_THREADS
and not stop_requested.is_set()
):
task = waiting_tasks.pop(0)
file_path = task[0]
precomp = pending_responses.get(
file_path, batched_responses.get(file_path)
)
futures[
executor.submit(process_single_task, task, precomp)
] = task
submit_available_tasks()
while futures:
completed_futures, _pending = concurrent.futures.wait(
tuple(futures),
return_when=concurrent.futures.FIRST_COMPLETED,
)
for future in completed_futures:
failed_task = futures.pop(future)
task_completed = False
try:
new_generated_tasks = future.result()
task_completed = True
if new_generated_tasks and not stop_requested.is_set():
if report_live_progress:
progress_total += len(new_generated_tasks)
waiting_tasks.extend(new_generated_tasks)
except CorrectionStopRequested:
pass
except Exception as e: # noqa: BLE001 - future boundary
print(
f"Exception during task execution: {e}",
file=sys.stderr,
)
with io_lock:
errors_summary.append((str(e), failed_task[0]))
if report_live_progress and task_completed:
progress_completed += 1
report_group_progress(
progress_completed, progress_total
)
submit_available_tasks()
tasks_to_process = [] # Vider la liste une fois traitée
# Après avoir traité toutes les tâches actuelles (live ou batched),
# on tente de débloquer les mouvements qui étaient en attente
if stop_requested.is_set():
break
delayed_tasks = resolve_delayed_moves()
if delayed_tasks:
print(f"Resolved {len(delayed_tasks)} delayed moves! Running executor for new tasks...")
if report_live_progress:
progress_total += len(delayed_tasks)
report_group_progress(progress_completed, progress_total)
tasks_to_process.extend(delayed_tasks)
made_progress = True
else:
@@ -907,6 +1042,12 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
print(err, file=sys.stderr)
escaped_path = shlex.quote(str(file))
print(f"Run : python -m copienator correct {escaped_path}")
if stop_requested.is_set():
print(
"[Interruption] Appels en cours terminés ; résultats disponibles sauvegardés.",
flush=True,
)
return ExitCode.INTERRUPTED
return ExitCode.PARTIAL if errors_summary else ExitCode.SUCCESS
@@ -931,9 +1072,17 @@ def run(
configure_runtime(workspace, discovered, args, api_client=api_client)
if not discovered and not args.refaire:
return ExitCode.PARTIAL
previous_handlers = {}
for signal_name in ("SIGINT", "SIGBREAK"):
interrupt_signal = getattr(signal, signal_name, None)
if interrupt_signal is not None:
previous_handlers[interrupt_signal] = signal.getsignal(interrupt_signal)
signal.signal(interrupt_signal, request_graceful_stop)
try:
status = run_configured(args)
finally:
for interrupt_signal, previous_handler in previous_handlers.items():
signal.signal(interrupt_signal, previous_handler)
for thread_id in list(thread_logs):
flush_thread_log(thread_id)
if warnings and status == ExitCode.SUCCESS:
@@ -983,6 +1132,12 @@ def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
def handle(args: argparse.Namespace) -> ExitCode:
if args.refaire and args.overwrite:
raise CliError(
"--overwrite cannot be used with --refaire; --refaire already "
"replaces the corrections selected in refaire.json",
ExitCode.INVALID_ARGUMENTS,
)
workspace, target = workspace_from_target(args)
targets = [target]
for additional in args.additional_targets:
+31 -7
View File
@@ -28,6 +28,8 @@ from copienator.copy_errors import clear_copy_error, mark_copy_error, marked_cop
DELIMITER_WIDTH = 5
DELIMITER_COLOR = (0, 0, 0)
OUTPUT_SIZE = (1800, 1000)
CROP_SHIFT_STEP = 50
CROP_WIDTH_STEP = 50
pdf_cache_lock = threading.Lock()
@@ -80,6 +82,7 @@ def process_single_pdf(
pdf_path: Path,
shift_offset: int = 0,
max_per_file: int = 5,
width_offset: int = 0,
) -> tuple[Image.Image, list[Image.Image], dict[str, object]] | None:
"""Convert one PDF into a preview, full-resolution splits and metadata."""
try:
@@ -90,7 +93,10 @@ def process_single_pdf(
left, right = 0, width
else:
left = max(0, 100 + shift_offset)
right = min(width, width // 3 + 100 + shift_offset)
right = min(
width,
width // 3 + 100 + shift_offset + max(0, width_offset),
)
if right > left:
cropped_images.append(image.crop((left, 0, right, height)))
if not cropped_images:
@@ -166,6 +172,7 @@ class ImageReviewer:
self.current_result = None
self.index = 0
self.current_shift = 0
self.current_width_offset = 0
self.default_max_per_file = default_max_per_file
self.current_max_per_file = default_max_per_file
self.current_preview: Image.Image | None = None
@@ -184,9 +191,10 @@ class ImageReviewer:
self.root.bind("<Return>", self.on_next)
self.root.bind("s", self.on_skip)
self.root.protocol("WM_DELETE_WINDOW", self.on_close)
self.root.bind("n", lambda _event: self.on_shift(50))
self.root.bind("N", lambda _event: self.on_shift(100))
self.root.bind("t", lambda _event: self.on_shift(-50))
self.root.bind("n", lambda _event: self.on_shift(CROP_SHIFT_STEP))
self.root.bind("N", lambda _event: self.on_shift(2 * CROP_SHIFT_STEP))
self.root.bind("t", lambda _event: self.on_shift(-CROP_SHIFT_STEP))
self.root.bind("l", lambda _event: self.on_enlarge(CROP_WIDTH_STEP))
self.root.bind("1", lambda _event: self.on_set_max_pages(1))
Thread(target=self.prefetch_worker, daemon=True).start()
@@ -222,6 +230,7 @@ class ImageReviewer:
return
self.is_processing = False
self.current_shift = 0
self.current_width_offset = 0
self.current_result = None
self.trigger_processing(self.files[self.index], self.current_shift)
@@ -234,7 +243,12 @@ class ImageReviewer:
def worker() -> None:
self.manual_queue.put(
process_single_pdf(pdf_path, shift, self.current_max_per_file)
process_single_pdf(
pdf_path,
shift,
self.current_max_per_file,
self.current_width_offset,
)
)
Thread(target=worker, daemon=True).start()
@@ -271,10 +285,12 @@ class ImageReviewer:
self.label_info.configure(
text=(
f"[{self.index + 1}/{len(self.files)}] {filename} | "
f"Shift: {self.current_shift}px\nFiles: {schema['number_of_files']} | "
f"Shift: {self.current_shift}px | "
f"Extra width: {self.current_width_offset}px\n"
f"Files: {schema['number_of_files']} | "
f"Cols: {schema['columns_per_file']}\n"
"Enter: Save and next | s: flag error and skip | n: +50 | N: +100 | t: -50 | "
"1: use single column"
"l: widen by 50 | 1: use full pages"
),
fg="black",
)
@@ -286,6 +302,13 @@ class ImageReviewer:
print(f"Applying shift: {self.current_shift}")
self.trigger_processing(self.files[self.index], self.current_shift)
def on_enlarge(self, amount: int) -> None:
if self.is_processing:
return
self.current_width_offset += amount
print(f"Applying extra width: {self.current_width_offset}")
self.trigger_processing(self.files[self.index], self.current_shift)
def on_next(self, _event: object) -> None:
if self.is_processing or self.current_result is None:
return
@@ -317,6 +340,7 @@ class ImageReviewer:
def _advance(self) -> None:
self.index += 1
self.current_shift = 0
self.current_width_offset = 0
self.current_max_per_file = self.default_max_per_file
self.load_current_image()
+112 -21
View File
@@ -1,7 +1,9 @@
from __future__ import annotations
import argparse
import copy
import queue
import re
import threading
import tkinter as tk
from collections.abc import Sequence
@@ -24,6 +26,9 @@ from copienator.utils import natural_key, read_all_labels
# --- Configuration & Globals ---
padding = 60
MISSING_LABEL_COLOR = "orange"
COMMON_MISSING_LABEL_COLOR = "#403a00"
COMMON_MISSING_THRESHOLD = 0.66
try:
font = ImageFont.truetype("DejaVuSans.ttf", size=30)
@@ -81,13 +86,81 @@ def normalized_labels(entries):
if str(value["label"]) != "_"
]
def prepare_image(image_path: str, bounding_boxes, all_labels, nb_pages, last_label_index):
def frequently_missing_labels(
copies_dir: Path,
all_labels: list[str],
threshold: float = COMMON_MISSING_THRESHOLD,
) -> set[str]:
"""Return labels absent from at least ``threshold`` of detected copies."""
labels_by_copy: dict[str, set[str]] = {}
for json_path in copies_dir.glob("*.json"):
match = re.fullmatch(r"(.+)_\d+", json_path.stem)
if match is None:
continue
try:
data = read_json(json_path)
entries = data["list"]
if not isinstance(entries, list):
continue
present = set(normalized_labels(entries))
except (OSError, KeyError, TypeError, ValueError):
continue
labels_by_copy.setdefault(match.group(1), set()).update(present)
copy_count = len(labels_by_copy)
if copy_count == 0:
return set()
return {
label
for label in all_labels
if sum(label not in present for present in labels_by_copy.values())
>= threshold * copy_count
}
def label_color(
label: str | None,
all_labels: list[str],
last_label_index: int,
common_missing: set[str],
) -> tuple[str, int]:
"""Choose a label color and return the updated chronological index."""
color = "black"
if not label or label not in all_labels:
return color, last_label_index
current_index = all_labels.index(label)
if current_index < last_label_index or (
last_label_index == -1 and current_index != 0
):
color = "red"
elif current_index > last_label_index + 1:
only_previous_is_missing = current_index == last_label_index + 2
previous_label = all_labels[current_index - 1]
color = (
COMMON_MISSING_LABEL_COLOR
if only_previous_is_missing and previous_label in common_missing
else MISSING_LABEL_COLOR
)
return color, current_index
def prepare_image(
image_path: str,
bounding_boxes,
all_labels,
nb_pages,
last_label_index,
common_missing: set[str] | None = None,
):
im = Image.open(image_path)
im.load()
width, height = im.size
new_im = Image.new(im.mode, (width + padding, height), "white")
new_im.paste(im, (0, 0))
draw = ImageDraw.Draw(new_im)
common_missing = common_missing or set()
for bbox in sort_bounding_boxes(bounding_boxes, nb_pages):
raw_y_min = int(bbox["box_2d"][0] * height / 1000)
@@ -99,15 +172,13 @@ def prepare_image(image_path: str, bounding_boxes, all_labels, nb_pages, last_la
abs_y_max = min(height, raw_y_max + 10)
abs_x_max = min(width, raw_x_max + 10)
color = "black"
label = bbox.get("label")
if label and label in all_labels:
current_index = all_labels.index(label)
if current_index < last_label_index or (last_label_index == -1 and current_index != 0):
color = "red"
elif current_index > last_label_index + 1:
color = "orange"
last_label_index = current_index
color, last_label_index = label_color(
label,
all_labels,
last_label_index,
common_missing,
)
draw.rectangle(((abs_x_min, abs_y_min), (abs_x_max, abs_y_max)), outline=color, width=4)
if label:
@@ -126,6 +197,7 @@ def _worker_items(base_dir, files_to_process, all_labels, output_queue):
"""
previous_copie = None
last_label_index = None
common_missing = frequently_missing_labels(base_dir / "Copies", all_labels)
for img_path in files_to_process:
json_path = base_dir / "Copies" / f"{img_path.stem}.json"
copie_part = int(img_path.stem[-2:])
@@ -158,7 +230,14 @@ def _worker_items(base_dir, files_to_process, all_labels, output_queue):
try:
print(f"Buffering {img_path.name}...")
(pil_image, last_label_index) = \
prepare_image(str(img_path), bb_list, all_labels, nb_pages, last_label_index)
prepare_image(
str(img_path),
bb_list,
all_labels,
nb_pages,
last_label_index,
common_missing,
)
error_msg = None
except Exception as e: # noqa: BLE001 - keep the item editable in the GUI
@@ -271,7 +350,6 @@ class ImageViewer:
# Start new batch
self.active_copie_name = metadata["copie"]
self.accumulated_results = {"name": metadata["name"], "list": []}
self.history.clear()
self.display_image(pil_image, json_path, metadata)
except queue.Empty:
@@ -293,16 +371,23 @@ class ImageViewer:
def on_previous(self, event):
if self.is_viewing and self.history:
print("Going back to previous image...")
prev_pil, prev_json, prev_meta, num_added = self.history.pop()
# Undo the accumulation to prevent duplicates when we hit Enter again
if self.accumulated_results and num_added > 0:
self.accumulated_results["list"] = self.accumulated_results["list"][:-num_added]
(
prev_pil,
prev_json,
prev_meta,
previous_copie_name,
previous_results,
) = self.history.pop()
# Push current image to the forward stack so we don't lose it
self.forward_stack.append((self.current_pil_image,
self.current_json_path, self.current_meta))
# Restore the aggregation exactly as it was before validating the
# previous image. This also makes navigation across copies safe.
self.active_copie_name = previous_copie_name
self.accumulated_results = previous_results
# Display the previous image immediately
self.display_image(prev_pil, prev_json, prev_meta)
def display_image(self, pil_image, json_path, metadata):
@@ -329,7 +414,8 @@ class ImageViewer:
def on_enter(self, event):
if self.is_viewing:
print(f"Committing data for {self.current_json_path.name}...")
num_added = 0 # ADD THIS LINE
previous_copie_name = self.active_copie_name
previous_results = copy.deepcopy(self.accumulated_results)
try:
current_data = read_json(self.current_json_path)
@@ -352,8 +438,6 @@ class ImageViewer:
print(msg)
messagebox.showerror("Label Error", msg)
return
num_added = len(converted_items)
# Add to accumulator
if self.accumulated_results:
self.accumulated_results["list"].extend(converted_items)
@@ -368,8 +452,15 @@ class ImageViewer:
messagebox.showerror("JSON Error", msg)
return # Abort advancement
self.history.append((self.current_pil_image, self.current_json_path,
self.current_meta, num_added))
self.history.append(
(
self.current_pil_image,
self.current_json_path,
self.current_meta,
previous_copie_name,
previous_results,
)
)
# Advance UI
self.is_viewing = False