améliorations diverses
This commit is contained in:
@@ -5,6 +5,7 @@ import base64
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import shlex
|
||||
import shutil
|
||||
import sys
|
||||
@@ -45,15 +46,22 @@ COPIES_DIR = Path()
|
||||
GROUPS_DIR = Path()
|
||||
output_path = Path()
|
||||
progress_path = Path()
|
||||
pending_responses_path = Path()
|
||||
tasks: list[tuple] = []
|
||||
tasks_to_process: list[tuple] = []
|
||||
results: dict = {}
|
||||
completed_tasks: list = []
|
||||
errors_summary: list = []
|
||||
pending_responses: dict[str, str] = {}
|
||||
overwrite = False
|
||||
limit = None
|
||||
client = None
|
||||
start_time = 0.0
|
||||
stop_requested = threading.Event()
|
||||
|
||||
|
||||
class CorrectionStopRequested(Exception):
|
||||
"""Raised in a worker before it starts another Gemini request."""
|
||||
|
||||
# --- Thread-safe Logging ---
|
||||
log_lock = threading.Lock()
|
||||
@@ -82,6 +90,25 @@ def flush_thread_log(tid=None):
|
||||
f.write("\n".join(thread_logs[tid]) + "\n\n")
|
||||
thread_logs[tid].clear()
|
||||
|
||||
|
||||
def report_group_progress(completed: int, total: int) -> None:
|
||||
"""Emit a stable, human-readable progress line for the GUI and CLI."""
|
||||
print(
|
||||
f"[Progression correction] Groupes traités : {completed}/{total}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def request_graceful_stop(_signum=None, _frame=None) -> None:
|
||||
"""Stop scheduling Gemini calls while allowing in-flight calls to finish."""
|
||||
if not stop_requested.is_set():
|
||||
stop_requested.set()
|
||||
print(
|
||||
"\n[Interruption] Arrêt des nouveaux appels Gemini demandé. "
|
||||
"Attente des appels en cours et sauvegarde de leurs résultats…",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# --- Lock for thread-safe file writing ---
|
||||
io_lock = threading.Lock()
|
||||
pro_lock = threading.Lock()
|
||||
@@ -140,6 +167,7 @@ def configure_runtime(
|
||||
api_client=None,
|
||||
) -> None:
|
||||
global INPUT_DIR, COPIES_DIR, GROUPS_DIR, output_path, progress_path
|
||||
global pending_responses_path, pending_responses
|
||||
global tasks, tasks_to_process, results, completed_tasks, errors_summary
|
||||
global overwrite, limit, client, start_time
|
||||
global pro_count, flash_count, pro_quota_exhausted
|
||||
@@ -149,11 +177,13 @@ def configure_runtime(
|
||||
GROUPS_DIR = workspace.groups_dir
|
||||
output_path = workspace.correction_file
|
||||
progress_path = workspace.correction_progress_file
|
||||
pending_responses_path = workspace.correction_pending_responses_file
|
||||
tasks = list(discovered_tasks)
|
||||
overwrite = bool(args.overwrite)
|
||||
limit = args.limit
|
||||
start_time = time.time()
|
||||
errors_summary = []
|
||||
pending_responses = {}
|
||||
completed_tasks = []
|
||||
results = {label: [] for _file, label in tasks}
|
||||
thread_logs.clear()
|
||||
@@ -161,6 +191,7 @@ def configure_runtime(
|
||||
pro_count = 0
|
||||
flash_count = 0
|
||||
pro_quota_exhausted = False
|
||||
stop_requested.clear()
|
||||
|
||||
if not overwrite:
|
||||
if progress_path.is_file():
|
||||
@@ -174,6 +205,20 @@ def configure_runtime(
|
||||
raise TypeError("correction.json must contain a JSON object")
|
||||
results = loaded_results
|
||||
|
||||
# A response saved during a graceful interruption is not a completed
|
||||
# correction. Resume its auxiliary checks even with --overwrite instead of
|
||||
# paying for the same primary request again.
|
||||
if pending_responses_path.is_file():
|
||||
loaded_pending = read_json(pending_responses_path)
|
||||
if not isinstance(loaded_pending, dict) or not all(
|
||||
isinstance(key, str) and isinstance(value, str)
|
||||
for key, value in loaded_pending.items()
|
||||
):
|
||||
raise TypeError(
|
||||
"correction_pending_responses.json must contain a JSON object"
|
||||
)
|
||||
pending_responses = loaded_pending
|
||||
|
||||
completed_set = {(str(file_path), label) for file_path, label in completed_tasks}
|
||||
tasks_to_process = [
|
||||
task for task in tasks if (str(task[0]), task[1]) not in completed_set
|
||||
@@ -184,7 +229,11 @@ def configure_runtime(
|
||||
def reset_workspace(workspace: EvaluationWorkspace) -> None:
|
||||
"""Apply the explicitly requested correction reset."""
|
||||
print("--- Running Reset ---")
|
||||
for path in (workspace.correction_file, workspace.correction_progress_file):
|
||||
for path in (
|
||||
workspace.correction_file,
|
||||
workspace.correction_progress_file,
|
||||
workspace.correction_pending_responses_file,
|
||||
):
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
print(f"Deleted: {path}")
|
||||
@@ -213,6 +262,8 @@ def call_gemini_with_retries(model_id, contents, config,
|
||||
delays = [60, 300]
|
||||
|
||||
for attempt in range(3):
|
||||
if stop_requested.is_set():
|
||||
raise CorrectionStopRequested
|
||||
# Switch to fallback immediately if quota was exhausted by another thread
|
||||
if model_id == MODEL_ID_pro and pro_quota_exhausted and fallback_model_id:
|
||||
model_id = fallback_model_id
|
||||
@@ -228,6 +279,8 @@ def call_gemini_with_retries(model_id, contents, config,
|
||||
full_response_text += chunk.text
|
||||
return full_response_text
|
||||
except Exception as e:
|
||||
if stop_requested.is_set():
|
||||
raise CorrectionStopRequested from e
|
||||
error_msg = str(e).lower()
|
||||
is_quota_error = "429" in error_msg or "quota" in error_msg or "exhausted" in error_msg
|
||||
is_minute_limit = "minute" in error_msg or "rpm" in error_msg or "tpm" in error_msg
|
||||
@@ -239,7 +292,8 @@ def call_gemini_with_retries(model_id, contents, config,
|
||||
wait_time = float(retry_match.group(1)) + 1.0 if retry_match else delays[attempt]
|
||||
|
||||
tprint(f"\tGemini Pro minute limit hit. Waiting {wait_time:.1f}s...")
|
||||
time.sleep(wait_time)
|
||||
if stop_requested.wait(wait_time):
|
||||
raise CorrectionStopRequested
|
||||
continue # Retry same model
|
||||
|
||||
# Immediately fallback to Flash without waiting if it's a Pro quota error
|
||||
@@ -251,7 +305,8 @@ def call_gemini_with_retries(model_id, contents, config,
|
||||
|
||||
if attempt < 2:
|
||||
tprint(f"\tGemini API failure: {e}. Retrying in {delays[attempt]} seconds...")
|
||||
time.sleep(delays[attempt])
|
||||
if stop_requested.wait(delays[attempt]):
|
||||
raise CorrectionStopRequested
|
||||
else:
|
||||
tprint(f"\tGemini API failure: {e}. Maximum retries reached.")
|
||||
raise
|
||||
@@ -359,6 +414,8 @@ def handle_label_errors(pid, label, res, pdf_path):
|
||||
tprint(f"\tHandling additional-answer for {pid} {label}")
|
||||
try:
|
||||
add_labels = json.loads(call_gemini_with_retries(MODEL_ID_flash, contents, config))
|
||||
except CorrectionStopRequested:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - invalid auxiliary model response
|
||||
add_labels = []
|
||||
|
||||
@@ -417,6 +474,9 @@ def process_single_task(task_tuple, precomputed_response=None):
|
||||
total_height = group_data[-1][2]
|
||||
use_flash = n >= 4 or total_height <= 500
|
||||
|
||||
if precomputed_response is None and stop_requested.is_set():
|
||||
raise CorrectionStopRequested
|
||||
|
||||
# Only apply limits and counts if we are making a live call
|
||||
if precomputed_response is None:
|
||||
if not use_flash:
|
||||
@@ -437,9 +497,11 @@ def process_single_task(task_tuple, precomputed_response=None):
|
||||
model_to_use = MODEL_ID_flash if use_flash else MODEL_ID_pro
|
||||
|
||||
if precomputed_response:
|
||||
tprint(f"Using batched response for: {label} {group_name}")
|
||||
tprint(f"Using saved response for: {label} {group_name}")
|
||||
full_response_text = precomputed_response
|
||||
else:
|
||||
if stop_requested.is_set():
|
||||
raise CorrectionStopRequested
|
||||
tprint(f"Asking Gemini {'Flash' if use_flash else 'Pro '}: {label} {group_name}")
|
||||
full_response_text = call_gemini_with_retries(model_to_use, contents, config)
|
||||
|
||||
@@ -478,8 +540,28 @@ def process_single_task(task_tuple, precomputed_response=None):
|
||||
if res["error"] != "":
|
||||
tprint("\tError :", res["error"], "for Copie", pid, group_name)
|
||||
|
||||
if can_spawn_tasks and res.get("error") in ["wrong-label", "additional-answer"]:
|
||||
new_tasks.extend(handle_label_errors(pid, label, res, pdf_path))
|
||||
if can_spawn_tasks and res.get("error") in [
|
||||
"wrong-label",
|
||||
"additional-answer",
|
||||
]:
|
||||
if stop_requested.is_set():
|
||||
with io_lock:
|
||||
pending_responses[file_path] = json.dumps(json_data)
|
||||
atomic_write_json(
|
||||
pending_responses_path, pending_responses
|
||||
)
|
||||
raise CorrectionStopRequested
|
||||
try:
|
||||
new_tasks.extend(
|
||||
handle_label_errors(pid, label, res, pdf_path)
|
||||
)
|
||||
except CorrectionStopRequested:
|
||||
with io_lock:
|
||||
pending_responses[file_path] = json.dumps(json_data)
|
||||
atomic_write_json(
|
||||
pending_responses_path, pending_responses
|
||||
)
|
||||
raise
|
||||
# Si "wrong-label" a déplacé le fichier courant vers _old
|
||||
if res.get("error", "").startswith("wrg-lbl-moved-to:"):
|
||||
current_suffix = "_old"
|
||||
@@ -520,6 +602,8 @@ def process_single_task(task_tuple, precomputed_response=None):
|
||||
if needs_correction:
|
||||
tprint(f"\tBox anomalies detected for Copie {pid} {group_name}. \n\tRequesting isolated correction from Gemini Flash...")
|
||||
try:
|
||||
if stop_requested.is_set():
|
||||
raise CorrectionStopRequested
|
||||
# Pensez à passer pdf_path à la fonction modifiée !
|
||||
res["feedback"] = correct_boxes_with_gemini(
|
||||
pid, label, pdf_path, res["feedback"],
|
||||
@@ -542,7 +626,12 @@ def process_single_task(task_tuple, precomputed_response=None):
|
||||
# To track progress
|
||||
completed_tasks.append((file_path, label))
|
||||
atomic_write_json(progress_path, completed_tasks)
|
||||
if file_path in pending_responses:
|
||||
del pending_responses[file_path]
|
||||
atomic_write_json(pending_responses_path, pending_responses)
|
||||
|
||||
except CorrectionStopRequested:
|
||||
raise
|
||||
except json.JSONDecodeError:
|
||||
tprint(f"Error decoding JSON for {file_path}", file=sys.stderr)
|
||||
with io_lock:
|
||||
@@ -836,36 +925,82 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
|
||||
else:
|
||||
print(f"Warning: Batch results file {batch_results_path} not found.", file=sys.stderr)
|
||||
|
||||
report_live_progress = not any(
|
||||
(args.batch, args.batch_from, args.deal_with_batched, args.refaire)
|
||||
)
|
||||
progress_total = len(tasks)
|
||||
progress_completed = max(0, progress_total - len(tasks_to_process))
|
||||
if report_live_progress:
|
||||
report_group_progress(progress_completed, progress_total)
|
||||
|
||||
made_progress = True
|
||||
while tasks_to_process or made_progress:
|
||||
if tasks_to_process:
|
||||
print(f"Starting processing on {len(tasks_to_process)} tasks with {NB_THREADS} threads...")
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=NB_THREADS) as executor:
|
||||
waiting_tasks = list(tasks_to_process)
|
||||
futures = {}
|
||||
for task in tasks_to_process:
|
||||
file_path = task[0]
|
||||
precomp = batched_responses.get(file_path)
|
||||
futures[executor.submit(process_single_task, task, precomp)] = task
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
new_generated_tasks = future.result()
|
||||
if new_generated_tasks:
|
||||
for new_task in new_generated_tasks:
|
||||
futures[executor.submit(process_single_task, new_task)] = new_task
|
||||
except Exception as e: # noqa: BLE001 - future boundary
|
||||
print(f"Exception during task execution: {e}", file=sys.stderr)
|
||||
failed_task = futures[future]
|
||||
with io_lock:
|
||||
errors_summary.append((str(e), failed_task[0]))
|
||||
def submit_available_tasks() -> None:
|
||||
while (
|
||||
waiting_tasks
|
||||
and len(futures) < NB_THREADS
|
||||
and not stop_requested.is_set()
|
||||
):
|
||||
task = waiting_tasks.pop(0)
|
||||
file_path = task[0]
|
||||
precomp = pending_responses.get(
|
||||
file_path, batched_responses.get(file_path)
|
||||
)
|
||||
futures[
|
||||
executor.submit(process_single_task, task, precomp)
|
||||
] = task
|
||||
|
||||
submit_available_tasks()
|
||||
while futures:
|
||||
completed_futures, _pending = concurrent.futures.wait(
|
||||
tuple(futures),
|
||||
return_when=concurrent.futures.FIRST_COMPLETED,
|
||||
)
|
||||
for future in completed_futures:
|
||||
failed_task = futures.pop(future)
|
||||
task_completed = False
|
||||
try:
|
||||
new_generated_tasks = future.result()
|
||||
task_completed = True
|
||||
if new_generated_tasks and not stop_requested.is_set():
|
||||
if report_live_progress:
|
||||
progress_total += len(new_generated_tasks)
|
||||
waiting_tasks.extend(new_generated_tasks)
|
||||
except CorrectionStopRequested:
|
||||
pass
|
||||
except Exception as e: # noqa: BLE001 - future boundary
|
||||
print(
|
||||
f"Exception during task execution: {e}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
with io_lock:
|
||||
errors_summary.append((str(e), failed_task[0]))
|
||||
if report_live_progress and task_completed:
|
||||
progress_completed += 1
|
||||
report_group_progress(
|
||||
progress_completed, progress_total
|
||||
)
|
||||
submit_available_tasks()
|
||||
|
||||
tasks_to_process = [] # Vider la liste une fois traitée
|
||||
|
||||
# Après avoir traité toutes les tâches actuelles (live ou batched),
|
||||
# on tente de débloquer les mouvements qui étaient en attente
|
||||
if stop_requested.is_set():
|
||||
break
|
||||
|
||||
delayed_tasks = resolve_delayed_moves()
|
||||
if delayed_tasks:
|
||||
print(f"Resolved {len(delayed_tasks)} delayed moves! Running executor for new tasks...")
|
||||
if report_live_progress:
|
||||
progress_total += len(delayed_tasks)
|
||||
report_group_progress(progress_completed, progress_total)
|
||||
tasks_to_process.extend(delayed_tasks)
|
||||
made_progress = True
|
||||
else:
|
||||
@@ -907,6 +1042,12 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
|
||||
print(err, file=sys.stderr)
|
||||
escaped_path = shlex.quote(str(file))
|
||||
print(f"Run : python -m copienator correct {escaped_path}")
|
||||
if stop_requested.is_set():
|
||||
print(
|
||||
"[Interruption] Appels en cours terminés ; résultats disponibles sauvegardés.",
|
||||
flush=True,
|
||||
)
|
||||
return ExitCode.INTERRUPTED
|
||||
return ExitCode.PARTIAL if errors_summary else ExitCode.SUCCESS
|
||||
|
||||
|
||||
@@ -931,9 +1072,17 @@ def run(
|
||||
configure_runtime(workspace, discovered, args, api_client=api_client)
|
||||
if not discovered and not args.refaire:
|
||||
return ExitCode.PARTIAL
|
||||
previous_handlers = {}
|
||||
for signal_name in ("SIGINT", "SIGBREAK"):
|
||||
interrupt_signal = getattr(signal, signal_name, None)
|
||||
if interrupt_signal is not None:
|
||||
previous_handlers[interrupt_signal] = signal.getsignal(interrupt_signal)
|
||||
signal.signal(interrupt_signal, request_graceful_stop)
|
||||
try:
|
||||
status = run_configured(args)
|
||||
finally:
|
||||
for interrupt_signal, previous_handler in previous_handlers.items():
|
||||
signal.signal(interrupt_signal, previous_handler)
|
||||
for thread_id in list(thread_logs):
|
||||
flush_thread_log(thread_id)
|
||||
if warnings and status == ExitCode.SUCCESS:
|
||||
@@ -983,6 +1132,12 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
|
||||
def handle(args: argparse.Namespace) -> ExitCode:
|
||||
if args.refaire and args.overwrite:
|
||||
raise CliError(
|
||||
"--overwrite cannot be used with --refaire; --refaire already "
|
||||
"replaces the corrections selected in refaire.json",
|
||||
ExitCode.INVALID_ARGUMENTS,
|
||||
)
|
||||
workspace, target = workspace_from_target(args)
|
||||
targets = [target]
|
||||
for additional in args.additional_targets:
|
||||
|
||||
@@ -28,6 +28,8 @@ from copienator.copy_errors import clear_copy_error, mark_copy_error, marked_cop
|
||||
DELIMITER_WIDTH = 5
|
||||
DELIMITER_COLOR = (0, 0, 0)
|
||||
OUTPUT_SIZE = (1800, 1000)
|
||||
CROP_SHIFT_STEP = 50
|
||||
CROP_WIDTH_STEP = 50
|
||||
pdf_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
@@ -80,6 +82,7 @@ def process_single_pdf(
|
||||
pdf_path: Path,
|
||||
shift_offset: int = 0,
|
||||
max_per_file: int = 5,
|
||||
width_offset: int = 0,
|
||||
) -> tuple[Image.Image, list[Image.Image], dict[str, object]] | None:
|
||||
"""Convert one PDF into a preview, full-resolution splits and metadata."""
|
||||
try:
|
||||
@@ -90,7 +93,10 @@ def process_single_pdf(
|
||||
left, right = 0, width
|
||||
else:
|
||||
left = max(0, 100 + shift_offset)
|
||||
right = min(width, width // 3 + 100 + shift_offset)
|
||||
right = min(
|
||||
width,
|
||||
width // 3 + 100 + shift_offset + max(0, width_offset),
|
||||
)
|
||||
if right > left:
|
||||
cropped_images.append(image.crop((left, 0, right, height)))
|
||||
if not cropped_images:
|
||||
@@ -166,6 +172,7 @@ class ImageReviewer:
|
||||
self.current_result = None
|
||||
self.index = 0
|
||||
self.current_shift = 0
|
||||
self.current_width_offset = 0
|
||||
self.default_max_per_file = default_max_per_file
|
||||
self.current_max_per_file = default_max_per_file
|
||||
self.current_preview: Image.Image | None = None
|
||||
@@ -184,9 +191,10 @@ class ImageReviewer:
|
||||
self.root.bind("<Return>", self.on_next)
|
||||
self.root.bind("s", self.on_skip)
|
||||
self.root.protocol("WM_DELETE_WINDOW", self.on_close)
|
||||
self.root.bind("n", lambda _event: self.on_shift(50))
|
||||
self.root.bind("N", lambda _event: self.on_shift(100))
|
||||
self.root.bind("t", lambda _event: self.on_shift(-50))
|
||||
self.root.bind("n", lambda _event: self.on_shift(CROP_SHIFT_STEP))
|
||||
self.root.bind("N", lambda _event: self.on_shift(2 * CROP_SHIFT_STEP))
|
||||
self.root.bind("t", lambda _event: self.on_shift(-CROP_SHIFT_STEP))
|
||||
self.root.bind("l", lambda _event: self.on_enlarge(CROP_WIDTH_STEP))
|
||||
self.root.bind("1", lambda _event: self.on_set_max_pages(1))
|
||||
|
||||
Thread(target=self.prefetch_worker, daemon=True).start()
|
||||
@@ -222,6 +230,7 @@ class ImageReviewer:
|
||||
return
|
||||
self.is_processing = False
|
||||
self.current_shift = 0
|
||||
self.current_width_offset = 0
|
||||
self.current_result = None
|
||||
self.trigger_processing(self.files[self.index], self.current_shift)
|
||||
|
||||
@@ -234,7 +243,12 @@ class ImageReviewer:
|
||||
|
||||
def worker() -> None:
|
||||
self.manual_queue.put(
|
||||
process_single_pdf(pdf_path, shift, self.current_max_per_file)
|
||||
process_single_pdf(
|
||||
pdf_path,
|
||||
shift,
|
||||
self.current_max_per_file,
|
||||
self.current_width_offset,
|
||||
)
|
||||
)
|
||||
|
||||
Thread(target=worker, daemon=True).start()
|
||||
@@ -271,10 +285,12 @@ class ImageReviewer:
|
||||
self.label_info.configure(
|
||||
text=(
|
||||
f"[{self.index + 1}/{len(self.files)}] {filename} | "
|
||||
f"Shift: {self.current_shift}px\nFiles: {schema['number_of_files']} | "
|
||||
f"Shift: {self.current_shift}px | "
|
||||
f"Extra width: {self.current_width_offset}px\n"
|
||||
f"Files: {schema['number_of_files']} | "
|
||||
f"Cols: {schema['columns_per_file']}\n"
|
||||
"Enter: Save and next | s: flag error and skip | n: +50 | N: +100 | t: -50 | "
|
||||
"1: use single column"
|
||||
"l: widen by 50 | 1: use full pages"
|
||||
),
|
||||
fg="black",
|
||||
)
|
||||
@@ -286,6 +302,13 @@ class ImageReviewer:
|
||||
print(f"Applying shift: {self.current_shift}")
|
||||
self.trigger_processing(self.files[self.index], self.current_shift)
|
||||
|
||||
def on_enlarge(self, amount: int) -> None:
|
||||
if self.is_processing:
|
||||
return
|
||||
self.current_width_offset += amount
|
||||
print(f"Applying extra width: {self.current_width_offset}")
|
||||
self.trigger_processing(self.files[self.index], self.current_shift)
|
||||
|
||||
def on_next(self, _event: object) -> None:
|
||||
if self.is_processing or self.current_result is None:
|
||||
return
|
||||
@@ -317,6 +340,7 @@ class ImageReviewer:
|
||||
def _advance(self) -> None:
|
||||
self.index += 1
|
||||
self.current_shift = 0
|
||||
self.current_width_offset = 0
|
||||
self.current_max_per_file = self.default_max_per_file
|
||||
self.load_current_image()
|
||||
|
||||
|
||||
+112
-21
@@ -1,7 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import queue
|
||||
import re
|
||||
import threading
|
||||
import tkinter as tk
|
||||
from collections.abc import Sequence
|
||||
@@ -24,6 +26,9 @@ from copienator.utils import natural_key, read_all_labels
|
||||
|
||||
# --- Configuration & Globals ---
|
||||
padding = 60
|
||||
MISSING_LABEL_COLOR = "orange"
|
||||
COMMON_MISSING_LABEL_COLOR = "#403a00"
|
||||
COMMON_MISSING_THRESHOLD = 0.66
|
||||
|
||||
try:
|
||||
font = ImageFont.truetype("DejaVuSans.ttf", size=30)
|
||||
@@ -81,13 +86,81 @@ def normalized_labels(entries):
|
||||
if str(value["label"]) != "_"
|
||||
]
|
||||
|
||||
def prepare_image(image_path: str, bounding_boxes, all_labels, nb_pages, last_label_index):
|
||||
|
||||
def frequently_missing_labels(
|
||||
copies_dir: Path,
|
||||
all_labels: list[str],
|
||||
threshold: float = COMMON_MISSING_THRESHOLD,
|
||||
) -> set[str]:
|
||||
"""Return labels absent from at least ``threshold`` of detected copies."""
|
||||
labels_by_copy: dict[str, set[str]] = {}
|
||||
for json_path in copies_dir.glob("*.json"):
|
||||
match = re.fullmatch(r"(.+)_\d+", json_path.stem)
|
||||
if match is None:
|
||||
continue
|
||||
try:
|
||||
data = read_json(json_path)
|
||||
entries = data["list"]
|
||||
if not isinstance(entries, list):
|
||||
continue
|
||||
present = set(normalized_labels(entries))
|
||||
except (OSError, KeyError, TypeError, ValueError):
|
||||
continue
|
||||
labels_by_copy.setdefault(match.group(1), set()).update(present)
|
||||
|
||||
copy_count = len(labels_by_copy)
|
||||
if copy_count == 0:
|
||||
return set()
|
||||
return {
|
||||
label
|
||||
for label in all_labels
|
||||
if sum(label not in present for present in labels_by_copy.values())
|
||||
>= threshold * copy_count
|
||||
}
|
||||
|
||||
|
||||
def label_color(
|
||||
label: str | None,
|
||||
all_labels: list[str],
|
||||
last_label_index: int,
|
||||
common_missing: set[str],
|
||||
) -> tuple[str, int]:
|
||||
"""Choose a label color and return the updated chronological index."""
|
||||
color = "black"
|
||||
if not label or label not in all_labels:
|
||||
return color, last_label_index
|
||||
|
||||
current_index = all_labels.index(label)
|
||||
if current_index < last_label_index or (
|
||||
last_label_index == -1 and current_index != 0
|
||||
):
|
||||
color = "red"
|
||||
elif current_index > last_label_index + 1:
|
||||
only_previous_is_missing = current_index == last_label_index + 2
|
||||
previous_label = all_labels[current_index - 1]
|
||||
color = (
|
||||
COMMON_MISSING_LABEL_COLOR
|
||||
if only_previous_is_missing and previous_label in common_missing
|
||||
else MISSING_LABEL_COLOR
|
||||
)
|
||||
return color, current_index
|
||||
|
||||
|
||||
def prepare_image(
|
||||
image_path: str,
|
||||
bounding_boxes,
|
||||
all_labels,
|
||||
nb_pages,
|
||||
last_label_index,
|
||||
common_missing: set[str] | None = None,
|
||||
):
|
||||
im = Image.open(image_path)
|
||||
im.load()
|
||||
width, height = im.size
|
||||
new_im = Image.new(im.mode, (width + padding, height), "white")
|
||||
new_im.paste(im, (0, 0))
|
||||
draw = ImageDraw.Draw(new_im)
|
||||
common_missing = common_missing or set()
|
||||
|
||||
for bbox in sort_bounding_boxes(bounding_boxes, nb_pages):
|
||||
raw_y_min = int(bbox["box_2d"][0] * height / 1000)
|
||||
@@ -99,15 +172,13 @@ def prepare_image(image_path: str, bounding_boxes, all_labels, nb_pages, last_la
|
||||
abs_y_max = min(height, raw_y_max + 10)
|
||||
abs_x_max = min(width, raw_x_max + 10)
|
||||
|
||||
color = "black"
|
||||
label = bbox.get("label")
|
||||
if label and label in all_labels:
|
||||
current_index = all_labels.index(label)
|
||||
if current_index < last_label_index or (last_label_index == -1 and current_index != 0):
|
||||
color = "red"
|
||||
elif current_index > last_label_index + 1:
|
||||
color = "orange"
|
||||
last_label_index = current_index
|
||||
color, last_label_index = label_color(
|
||||
label,
|
||||
all_labels,
|
||||
last_label_index,
|
||||
common_missing,
|
||||
)
|
||||
|
||||
draw.rectangle(((abs_x_min, abs_y_min), (abs_x_max, abs_y_max)), outline=color, width=4)
|
||||
if label:
|
||||
@@ -126,6 +197,7 @@ def _worker_items(base_dir, files_to_process, all_labels, output_queue):
|
||||
"""
|
||||
previous_copie = None
|
||||
last_label_index = None
|
||||
common_missing = frequently_missing_labels(base_dir / "Copies", all_labels)
|
||||
for img_path in files_to_process:
|
||||
json_path = base_dir / "Copies" / f"{img_path.stem}.json"
|
||||
copie_part = int(img_path.stem[-2:])
|
||||
@@ -158,7 +230,14 @@ def _worker_items(base_dir, files_to_process, all_labels, output_queue):
|
||||
try:
|
||||
print(f"Buffering {img_path.name}...")
|
||||
(pil_image, last_label_index) = \
|
||||
prepare_image(str(img_path), bb_list, all_labels, nb_pages, last_label_index)
|
||||
prepare_image(
|
||||
str(img_path),
|
||||
bb_list,
|
||||
all_labels,
|
||||
nb_pages,
|
||||
last_label_index,
|
||||
common_missing,
|
||||
)
|
||||
error_msg = None
|
||||
|
||||
except Exception as e: # noqa: BLE001 - keep the item editable in the GUI
|
||||
@@ -271,7 +350,6 @@ class ImageViewer:
|
||||
# Start new batch
|
||||
self.active_copie_name = metadata["copie"]
|
||||
self.accumulated_results = {"name": metadata["name"], "list": []}
|
||||
self.history.clear()
|
||||
|
||||
self.display_image(pil_image, json_path, metadata)
|
||||
except queue.Empty:
|
||||
@@ -293,16 +371,23 @@ class ImageViewer:
|
||||
def on_previous(self, event):
|
||||
if self.is_viewing and self.history:
|
||||
print("Going back to previous image...")
|
||||
prev_pil, prev_json, prev_meta, num_added = self.history.pop()
|
||||
|
||||
# Undo the accumulation to prevent duplicates when we hit Enter again
|
||||
if self.accumulated_results and num_added > 0:
|
||||
self.accumulated_results["list"] = self.accumulated_results["list"][:-num_added]
|
||||
(
|
||||
prev_pil,
|
||||
prev_json,
|
||||
prev_meta,
|
||||
previous_copie_name,
|
||||
previous_results,
|
||||
) = self.history.pop()
|
||||
|
||||
# Push current image to the forward stack so we don't lose it
|
||||
self.forward_stack.append((self.current_pil_image,
|
||||
self.current_json_path, self.current_meta))
|
||||
|
||||
# Restore the aggregation exactly as it was before validating the
|
||||
# previous image. This also makes navigation across copies safe.
|
||||
self.active_copie_name = previous_copie_name
|
||||
self.accumulated_results = previous_results
|
||||
|
||||
# Display the previous image immediately
|
||||
self.display_image(prev_pil, prev_json, prev_meta)
|
||||
def display_image(self, pil_image, json_path, metadata):
|
||||
@@ -329,7 +414,8 @@ class ImageViewer:
|
||||
def on_enter(self, event):
|
||||
if self.is_viewing:
|
||||
print(f"Committing data for {self.current_json_path.name}...")
|
||||
num_added = 0 # ADD THIS LINE
|
||||
previous_copie_name = self.active_copie_name
|
||||
previous_results = copy.deepcopy(self.accumulated_results)
|
||||
|
||||
try:
|
||||
current_data = read_json(self.current_json_path)
|
||||
@@ -352,8 +438,6 @@ class ImageViewer:
|
||||
print(msg)
|
||||
messagebox.showerror("Label Error", msg)
|
||||
return
|
||||
num_added = len(converted_items)
|
||||
|
||||
# Add to accumulator
|
||||
if self.accumulated_results:
|
||||
self.accumulated_results["list"].extend(converted_items)
|
||||
@@ -368,8 +452,15 @@ class ImageViewer:
|
||||
messagebox.showerror("JSON Error", msg)
|
||||
return # Abort advancement
|
||||
|
||||
self.history.append((self.current_pil_image, self.current_json_path,
|
||||
self.current_meta, num_added))
|
||||
self.history.append(
|
||||
(
|
||||
self.current_pil_image,
|
||||
self.current_json_path,
|
||||
self.current_meta,
|
||||
previous_copie_name,
|
||||
previous_results,
|
||||
)
|
||||
)
|
||||
|
||||
# Advance UI
|
||||
self.is_viewing = False
|
||||
|
||||
@@ -109,6 +109,10 @@ class EvaluationWorkspace:
|
||||
def correction_progress_file(self) -> Path:
|
||||
return self.root / "correction_progress.json"
|
||||
|
||||
@property
|
||||
def correction_pending_responses_file(self) -> Path:
|
||||
return self.root / "correction_pending_responses.json"
|
||||
|
||||
@property
|
||||
def batch_jobs_file(self) -> Path:
|
||||
return self.root / "batch_jobs.json"
|
||||
|
||||
Reference in New Issue
Block a user