sebastien/Copies
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

améliorations diverses

Browse Source
This commit is contained in:
sebastien
2026-09-17 22:19:05 +02:00
parent 5b8215e7f5
commit 0a453403cf
11 changed files with 1072 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
copienator/commands/correction.py
+176 -21
View File
@@ -5,6 +5,7 @@ import base64
import concurrent.futures
import json
import os
import signal
import shlex
import shutil
import sys
@@ -45,15 +46,22 @@ COPIES_DIR = Path()
GROUPS_DIR = Path()
output_path = Path()
progress_path = Path()
pending_responses_path = Path()
tasks: list[tuple] = []
tasks_to_process: list[tuple] = []
results: dict = {}
completed_tasks: list = []
errors_summary: list = []
pending_responses: dict[str, str] = {}
overwrite = False
limit = None
client = None
start_time = 0.0
stop_requested = threading.Event()
class CorrectionStopRequested(Exception):
"""Raised in a worker before it starts another Gemini request."""
# --- Thread-safe Logging ---
log_lock = threading.Lock()
@@ -82,6 +90,25 @@ def flush_thread_log(tid=None):
f.write("\n".join(thread_logs[tid]) + "\n\n")
thread_logs[tid].clear()
def report_group_progress(completed: int, total: int) -> None:
"""Emit a stable, human-readable progress line for the GUI and CLI."""
print(
f"[Progression correction] Groupes traités : {completed}/{total}",
flush=True,
)
def request_graceful_stop(_signum=None, _frame=None) -> None:
"""Stop scheduling Gemini calls while allowing in-flight calls to finish."""
if not stop_requested.is_set():
stop_requested.set()
print(
"\n[Interruption] Arrêt des nouveaux appels Gemini demandé. "
"Attente des appels en cours et sauvegarde de leurs résultats…",
flush=True,
)
# --- Lock for thread-safe file writing ---
io_lock = threading.Lock()
pro_lock = threading.Lock()
@@ -140,6 +167,7 @@ def configure_runtime(
api_client=None,
) -> None:
global INPUT_DIR, COPIES_DIR, GROUPS_DIR, output_path, progress_path
global pending_responses_path, pending_responses
global tasks, tasks_to_process, results, completed_tasks, errors_summary
global overwrite, limit, client, start_time
global pro_count, flash_count, pro_quota_exhausted
@@ -149,11 +177,13 @@ def configure_runtime(
GROUPS_DIR = workspace.groups_dir
output_path = workspace.correction_file
progress_path = workspace.correction_progress_file
pending_responses_path = workspace.correction_pending_responses_file
tasks = list(discovered_tasks)
overwrite = bool(args.overwrite)
limit = args.limit
start_time = time.time()
errors_summary = []
pending_responses = {}
completed_tasks = []
results = {label: [] for _file, label in tasks}
thread_logs.clear()
@@ -161,6 +191,7 @@ def configure_runtime(
pro_count = 0
flash_count = 0
pro_quota_exhausted = False
stop_requested.clear()
if not overwrite:
if progress_path.is_file():
@@ -174,6 +205,20 @@ def configure_runtime(
raise TypeError("correction.json must contain a JSON object")
results = loaded_results
# A response saved during a graceful interruption is not a completed
# correction. Resume its auxiliary checks even with --overwrite instead of
# paying for the same primary request again.
if pending_responses_path.is_file():
loaded_pending = read_json(pending_responses_path)
if not isinstance(loaded_pending, dict) or not all(
isinstance(key, str) and isinstance(value, str)
for key, value in loaded_pending.items()
):
raise TypeError(
"correction_pending_responses.json must contain a JSON object"
)
pending_responses = loaded_pending
completed_set = {(str(file_path), label) for file_path, label in completed_tasks}
tasks_to_process = [
task for task in tasks if (str(task[0]), task[1]) not in completed_set
@@ -184,7 +229,11 @@ def configure_runtime(
def reset_workspace(workspace: EvaluationWorkspace) -> None:
"""Apply the explicitly requested correction reset."""
print("--- Running Reset ---")
for path in (workspace.correction_file, workspace.correction_progress_file):
for path in (
workspace.correction_file,
workspace.correction_progress_file,
workspace.correction_pending_responses_file,
):
if path.exists():
path.unlink()
print(f"Deleted: {path}")
@@ -213,6 +262,8 @@ def call_gemini_with_retries(model_id, contents, config,
delays = [60, 300]
for attempt in range(3):
if stop_requested.is_set():
raise CorrectionStopRequested
# Switch to fallback immediately if quota was exhausted by another thread
if model_id == MODEL_ID_pro and pro_quota_exhausted and fallback_model_id:
model_id = fallback_model_id
@@ -228,6 +279,8 @@ def call_gemini_with_retries(model_id, contents, config,
full_response_text += chunk.text
return full_response_text
except Exception as e:
if stop_requested.is_set():
raise CorrectionStopRequested from e
error_msg = str(e).lower()
is_quota_error = "429" in error_msg or "quota" in error_msg or "exhausted" in error_msg
is_minute_limit = "minute" in error_msg or "rpm" in error_msg or "tpm" in error_msg
@@ -239,7 +292,8 @@ def call_gemini_with_retries(model_id, contents, config,
wait_time = float(retry_match.group(1)) + 1.0 if retry_match else delays[attempt]
tprint(f"\tGemini Pro minute limit hit. Waiting {wait_time:.1f}s...")
time.sleep(wait_time)
if stop_requested.wait(wait_time):
raise CorrectionStopRequested
continue # Retry same model
# Immediately fallback to Flash without waiting if it's a Pro quota error
@@ -251,7 +305,8 @@ def call_gemini_with_retries(model_id, contents, config,
if attempt < 2:
tprint(f"\tGemini API failure: {e}. Retrying in {delays[attempt]} seconds...")
time.sleep(delays[attempt])
if stop_requested.wait(delays[attempt]):
raise CorrectionStopRequested
else:
tprint(f"\tGemini API failure: {e}. Maximum retries reached.")
raise
@@ -359,6 +414,8 @@ def handle_label_errors(pid, label, res, pdf_path):
tprint(f"\tHandling additional-answer for {pid} {label}")
try:
add_labels = json.loads(call_gemini_with_retries(MODEL_ID_flash, contents, config))
except CorrectionStopRequested:
raise
except Exception: # noqa: BLE001 - invalid auxiliary model response
add_labels = []
@@ -417,6 +474,9 @@ def process_single_task(task_tuple, precomputed_response=None):
total_height = group_data[-1][2]
use_flash = n >= 4 or total_height <= 500
if precomputed_response is None and stop_requested.is_set():
raise CorrectionStopRequested
# Only apply limits and counts if we are making a live call
if precomputed_response is None:
if not use_flash:
@@ -437,9 +497,11 @@ def process_single_task(task_tuple, precomputed_response=None):
model_to_use = MODEL_ID_flash if use_flash else MODEL_ID_pro
if precomputed_response:
tprint(f"Using batched response for: {label} {group_name}")
tprint(f"Using saved response for: {label} {group_name}")
full_response_text = precomputed_response
else:
if stop_requested.is_set():
raise CorrectionStopRequested
tprint(f"Asking Gemini {'Flash' if use_flash else 'Pro '}: {label} {group_name}")
full_response_text = call_gemini_with_retries(model_to_use, contents, config)
@@ -478,8 +540,28 @@ def process_single_task(task_tuple, precomputed_response=None):
if res["error"] != "":
tprint("\tError :", res["error"], "for Copie", pid, group_name)
if can_spawn_tasks and res.get("error") in ["wrong-label", "additional-answer"]:
new_tasks.extend(handle_label_errors(pid, label, res, pdf_path))
if can_spawn_tasks and res.get("error") in [
"wrong-label",
"additional-answer",
]:
if stop_requested.is_set():
with io_lock:
pending_responses[file_path] = json.dumps(json_data)
atomic_write_json(
pending_responses_path, pending_responses
)
raise CorrectionStopRequested
try:
new_tasks.extend(
handle_label_errors(pid, label, res, pdf_path)
)
except CorrectionStopRequested:
with io_lock:
pending_responses[file_path] = json.dumps(json_data)
atomic_write_json(
pending_responses_path, pending_responses
)
raise
# Si "wrong-label" a déplacé le fichier courant vers _old
if res.get("error", "").startswith("wrg-lbl-moved-to:"):
current_suffix = "_old"
@@ -520,6 +602,8 @@ def process_single_task(task_tuple, precomputed_response=None):
if needs_correction:
tprint(f"\tBox anomalies detected for Copie {pid} {group_name}. \n\tRequesting isolated correction from Gemini Flash...")
try:
if stop_requested.is_set():
raise CorrectionStopRequested
# Pensez à passer pdf_path à la fonction modifiée !
res["feedback"] = correct_boxes_with_gemini(
pid, label, pdf_path, res["feedback"],
@@ -542,7 +626,12 @@ def process_single_task(task_tuple, precomputed_response=None):
# To track progress
completed_tasks.append((file_path, label))
atomic_write_json(progress_path, completed_tasks)
if file_path in pending_responses:
del pending_responses[file_path]
atomic_write_json(pending_responses_path, pending_responses)
except CorrectionStopRequested:
raise
except json.JSONDecodeError:
tprint(f"Error decoding JSON for {file_path}", file=sys.stderr)
with io_lock:
@@ -836,36 +925,82 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
else:
print(f"Warning: Batch results file {batch_results_path} not found.", file=sys.stderr)
report_live_progress = not any(
(args.batch, args.batch_from, args.deal_with_batched, args.refaire)
)
progress_total = len(tasks)
progress_completed = max(0, progress_total - len(tasks_to_process))
if report_live_progress:
report_group_progress(progress_completed, progress_total)
made_progress = True
while tasks_to_process or made_progress:
if tasks_to_process:
print(f"Starting processing on {len(tasks_to_process)} tasks with {NB_THREADS} threads...")
with concurrent.futures.ThreadPoolExecutor(max_workers=NB_THREADS) as executor:
waiting_tasks = list(tasks_to_process)
futures = {}
for task in tasks_to_process:
file_path = task[0]
precomp = batched_responses.get(file_path)
futures[executor.submit(process_single_task, task, precomp)] = task
for future in concurrent.futures.as_completed(futures):
try:
new_generated_tasks = future.result()
if new_generated_tasks:
for new_task in new_generated_tasks:
futures[executor.submit(process_single_task, new_task)] = new_task
except Exception as e: # noqa: BLE001 - future boundary
print(f"Exception during task execution: {e}", file=sys.stderr)
failed_task = futures[future]
with io_lock:
errors_summary.append((str(e), failed_task[0]))
def submit_available_tasks() -> None:
while (
waiting_tasks
and len(futures) < NB_THREADS
and not stop_requested.is_set()
):
task = waiting_tasks.pop(0)
file_path = task[0]
precomp = pending_responses.get(
file_path, batched_responses.get(file_path)
)
futures[
executor.submit(process_single_task, task, precomp)
] = task
submit_available_tasks()
while futures:
completed_futures, _pending = concurrent.futures.wait(
tuple(futures),
return_when=concurrent.futures.FIRST_COMPLETED,
)
for future in completed_futures:
failed_task = futures.pop(future)
task_completed = False
try:
new_generated_tasks = future.result()
task_completed = True
if new_generated_tasks and not stop_requested.is_set():
if report_live_progress:
progress_total += len(new_generated_tasks)
waiting_tasks.extend(new_generated_tasks)
except CorrectionStopRequested:
pass
except Exception as e: # noqa: BLE001 - future boundary
print(
f"Exception during task execution: {e}",
file=sys.stderr,
)
with io_lock:
errors_summary.append((str(e), failed_task[0]))
if report_live_progress and task_completed:
progress_completed += 1
report_group_progress(
progress_completed, progress_total
)
submit_available_tasks()
tasks_to_process = [] # Vider la liste une fois traitée
# Après avoir traité toutes les tâches actuelles (live ou batched),
# on tente de débloquer les mouvements qui étaient en attente
if stop_requested.is_set():
break
delayed_tasks = resolve_delayed_moves()
if delayed_tasks:
print(f"Resolved {len(delayed_tasks)} delayed moves! Running executor for new tasks...")
if report_live_progress:
progress_total += len(delayed_tasks)
report_group_progress(progress_completed, progress_total)
tasks_to_process.extend(delayed_tasks)
made_progress = True
else:
@@ -907,6 +1042,12 @@ def run_configured(args: argparse.Namespace) -> ExitCode:
print(err, file=sys.stderr)
escaped_path = shlex.quote(str(file))
print(f"Run : python -m copienator correct {escaped_path}")
if stop_requested.is_set():
print(
"[Interruption] Appels en cours terminés ; résultats disponibles sauvegardés.",
flush=True,
)
return ExitCode.INTERRUPTED
return ExitCode.PARTIAL if errors_summary else ExitCode.SUCCESS
@@ -931,9 +1072,17 @@ def run(
configure_runtime(workspace, discovered, args, api_client=api_client)
if not discovered and not args.refaire:
return ExitCode.PARTIAL
previous_handlers = {}
for signal_name in ("SIGINT", "SIGBREAK"):
interrupt_signal = getattr(signal, signal_name, None)
if interrupt_signal is not None:
previous_handlers[interrupt_signal] = signal.getsignal(interrupt_signal)
signal.signal(interrupt_signal, request_graceful_stop)
try:
status = run_configured(args)
finally:
for interrupt_signal, previous_handler in previous_handlers.items():
signal.signal(interrupt_signal, previous_handler)
for thread_id in list(thread_logs):
flush_thread_log(thread_id)
if warnings and status == ExitCode.SUCCESS:
@@ -983,6 +1132,12 @@ def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
def handle(args: argparse.Namespace) -> ExitCode:
if args.refaire and args.overwrite:
raise CliError(
"--overwrite cannot be used with --refaire; --refaire already "
"replaces the corrections selected in refaire.json",
ExitCode.INVALID_ARGUMENTS,
)
workspace, target = workspace_from_target(args)
targets = [target]
for additional in args.additional_targets: