sebastien/Copies
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Standardisation 8

Browse Source
This commit is contained in:
sebastien
2026-08-20 15:18:08 +02:00
parent 18d1e5e2bb
commit b19d3b0db6
9 changed files with 758 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files
correction.py
+336 -208
View File
@@ -1,91 +1,59 @@
import sys
import os
import time
from pathlib import Path
from __future__ import annotations
import argparse
import prompting
import signal
from google import genai
import base64
import shlex
import json
import threading
import concurrent.futures
import json
import os
import shlex
import shutil
import sys
import threading
import time
from collections.abc import Sequence
from pathlib import Path
from copienator import atomic_write_json
if len(sys.argv) < 2:
sys.exit("Usage: python script.py 'InterroTest/Ex 2/Group_1.jpg' OR <InputDir> OR 'file1' 'file2'")
# Parse Arguments
parser = argparse.ArgumentParser()
parser.add_argument("paths", nargs="+", help="List of images or directories")
parser.add_argument("--overwrite", action="store_true",
help="Force redo requests even if output exists")
parser.add_argument("--limit", type=int, help="limit calls to gemini rpo integer")
parser.add_argument("--refaire", action="store_true",
help="Redo specific copies/labels defined in refaire.json")
parser.add_argument("--batch", action="store_true",
help="Generate a JSONL file of requests to send to the Gemini Batch API")
parser.add_argument("--batch-from", type=str, metavar="LABEL",
help="Do live requests before LABEL, and batch requests from LABEL onwards")
parser.add_argument("--deal-with-batched", action="store_true",
help="Process a JSONL file containing completed batch results")
parser.add_argument("--reset", action="store_true",
help="Remove correction.json, revert _old.pdf, delete _new.pdf, then exit")
args, _ = parser.parse_known_args()
tasks = [] # List of tuples: (filepath_str, label_str)
results = {}
for path_str in args.paths:
arg_path = Path(path_str)
if not arg_path.exists():
print(f"Warning: {path_str} not found. Skipping.")
continue
if arg_path.is_file() and arg_path.suffix.lower() == ".jpg":
# Handle individual file
# Note: assumes structure InterroTest/Ex 2/Group_1.jpg
label = arg_path.parent.name
INPUT_DIR = arg_path.parent.parent.parent
COPIES_DIR = INPUT_DIR / "Copies"
GROUPS_DIR = INPUT_DIR / "Par label"
tasks.append((str(arg_path), label))
if label not in results:
results[label] = []
elif arg_path.is_dir():
INPUT_DIR = arg_path
COPIES_DIR = INPUT_DIR / "Copies"
GROUPS_DIR = INPUT_DIR / "Par label"
# Handle directory (original behavior)
for sub in GROUPS_DIR.iterdir():
if sub.is_dir():
label = sub.name
if label not in results:
results[label] = []
for img in sub.glob("*.jpg"):
tasks.append((str(img), label))
from google import genai
import config
import grouping
import prompting
from copienator import (
CliError,
EvaluationWorkspace,
ExitCode,
atomic_write_json,
atomic_write_text,
execute,
read_json,
target_parser,
workspace_from_target,
)
from utils import enonce_total, read_all_labels
NB_THREADS = 12
# PROXY_URL = "http://192.168.241.1:3128"
PROXY_URL = None
if PROXY_URL:
os.environ["http_proxy"] = PROXY_URL
os.environ["https_proxy"] = PROXY_URL
import config
MODEL_ID_pro = config.MODEL_PRO_ID
MODEL_ID_flash = config.MODEL_FLASH_ID
api_key = config.API_KEY
# Runtime globals retained while the processing helpers are migrated incrementally.
INPUT_DIR = Path()
COPIES_DIR = Path()
GROUPS_DIR = Path()
output_path = Path()
progress_path = Path()
tasks: list[tuple] = []
tasks_to_process: list[tuple] = []
results: dict = {}
completed_tasks: list = []
errors_summary: list = []
overwrite = False
limit = None
client = None
start_time = 0.0
# --- Thread-safe Logging ---
log_lock = threading.Lock()
thread_logs = {}
@@ -113,57 +81,6 @@ def flush_thread_log(tid=None):
f.write("\n".join(thread_logs[tid]) + "\n\n")
thread_logs[tid].clear()
def handle_interrupt(sig, frame):
"""Flush all partial/unfinished logs if program is interrupted."""
print("\nInterrupt received. Flushing partial logs...", file=sys.stderr)
for tid in list(thread_logs.keys()):
flush_thread_log(tid)
sys.exit(1)
signal.signal(signal.SIGINT, handle_interrupt)
signal.signal(signal.SIGTERM, handle_interrupt)
# ---------------------------
client = genai.Client(api_key=api_key)
output_path = INPUT_DIR / "correction.json"
progress_path = INPUT_DIR / "correction_progress.json"
if args.reset:
print("--- Running Reset ---")
if output_path.exists():
output_path.unlink()
print(f"Deleted: {output_path}")
if progress_path.exists():
progress_path.unlink()
print(f"Deleted: {progress_path}")
if COPIES_DIR.exists():
for copie_dir in COPIES_DIR.iterdir():
if not copie_dir.is_dir():
continue
# Revert _old.pdf files
for old_pdf in copie_dir.glob("*_old.pdf"):
orig_pdf = copie_dir / old_pdf.name.replace("_old.pdf", ".pdf")
if orig_pdf.exists():
orig_pdf.unlink() # Prevent FileExistsError on Windows
old_pdf.rename(orig_pdf)
print(f"Moved: {copie_dir.name}/{old_pdf.name} -> {orig_pdf.name}")
# Delete _new.pdf files
for new_pdf in copie_dir.glob("*_new.pdf"):
new_pdf.unlink()
print(f"Deleted: {copie_dir.name}/{new_pdf.name}")
sys.exit("Reset almost complete. For each deleted `_new`, you should manually delete the group in `Par label`")
start_time = time.time()
overwrite = args.overwrite
limit = args.limit
completed_tasks = []
errors_summary = []
# --- Lock for thread-safe file writing ---
io_lock = threading.Lock()
pro_lock = threading.Lock()
@@ -171,21 +88,119 @@ pro_count = 0
flash_count = 0
pro_quota_exhausted = False
if overwrite:
if output_path.exists():
output_path.unlink()
if progress_path.exists():
progress_path.unlink()
else:
if progress_path.exists():
with open(progress_path, "r", encoding="utf-8") as f:
completed_tasks = json.load(f)
if output_path.exists():
with open(output_path, "r", encoding="utf-8") as f:
results = json.load(f)
completed_set = set((str(f), l) for f, l in completed_tasks)
tasks_to_process = [t for t in tasks if (str(t[0]), t[1]) not in completed_set]
def discover_tasks(
workspace: EvaluationWorkspace,
targets: list[Path],
) -> tuple[list[tuple[str, str]], list[str]]:
workspace.require_directories("Copies", "Par label")
discovered: list[tuple[str, str]] = []
warnings: list[str] = []
for target in targets:
if target.is_file():
if target.suffix.casefold() != ".jpg":
raise CliError(
f"Correction target is not a group JPG: {target}",
ExitCode.INVALID_ARGUMENTS,
)
try:
target.relative_to(workspace.groups_dir)
except ValueError as exc:
raise CliError(
f"Group image is not inside {workspace.groups_dir}: {target}",
ExitCode.INVALID_ARGUMENTS,
) from exc
discovered.append((str(target), target.parent.name))
continue
group_directories = sorted(
(path for path in workspace.groups_dir.iterdir() if path.is_dir()),
key=lambda path: path.name.casefold(),
)
for group_directory in group_directories:
images = sorted(
group_directory.glob("*.jpg"), key=lambda path: path.name.casefold()
)
discovered.extend(
(str(image), group_directory.name) for image in images
)
if not group_directories:
warnings.append(f"No label groups found in {workspace.groups_dir}")
return list(dict.fromkeys(discovered)), warnings
def configure_runtime(
workspace: EvaluationWorkspace,
discovered_tasks: list[tuple[str, str]],
args: argparse.Namespace,
*,
api_client=None,
) -> None:
global INPUT_DIR, COPIES_DIR, GROUPS_DIR, output_path, progress_path
global tasks, tasks_to_process, results, completed_tasks, errors_summary
global overwrite, limit, client, start_time
global pro_count, flash_count, pro_quota_exhausted
INPUT_DIR = workspace.root
COPIES_DIR = workspace.copies_dir
GROUPS_DIR = workspace.groups_dir
output_path = workspace.correction_file
progress_path = workspace.correction_progress_file
tasks = list(discovered_tasks)
overwrite = bool(args.overwrite)
limit = args.limit
start_time = time.time()
errors_summary = []
completed_tasks = []
results = {label: [] for _file, label in tasks}
thread_logs.clear()
pro_count = 0
flash_count = 0
pro_quota_exhausted = False
if not overwrite:
if progress_path.is_file():
loaded_progress = read_json(progress_path)
if not isinstance(loaded_progress, list):
raise TypeError("correction_progress.json must contain a JSON array")
completed_tasks = loaded_progress
if output_path.is_file():
loaded_results = read_json(output_path)
if not isinstance(loaded_results, dict):
raise TypeError("correction.json must contain a JSON object")
results = loaded_results
completed_set = {(str(file_path), label) for file_path, label in completed_tasks}
tasks_to_process = [
task for task in tasks if (str(task[0]), task[1]) not in completed_set
]
client = api_client
def reset_workspace(workspace: EvaluationWorkspace) -> None:
"""Apply the explicitly requested correction reset."""
print("--- Running Reset ---")
for path in (workspace.correction_file, workspace.correction_progress_file):
if path.exists():
path.unlink()
print(f"Deleted: {path}")
if workspace.copies_dir.is_dir():
for copy_directory in workspace.copies_dir.iterdir():
if not copy_directory.is_dir():
continue
for old_pdf in copy_directory.glob("*_old.pdf"):
original = old_pdf.with_name(old_pdf.name.replace("_old.pdf", ".pdf"))
if original.exists():
original.unlink()
old_pdf.replace(original)
print(f"Moved: {copy_directory.name}/{old_pdf.name} -> {original.name}")
for new_pdf in copy_directory.glob("*_new.pdf"):
new_pdf.unlink()
print(f"Deleted: {copy_directory.name}/{new_pdf.name}")
print(
"Reset almost complete. Manually remove groups associated with deleted "
"_new PDFs from 'Par label'."
)
def call_gemini_with_retries(model_id, contents, config,
fallback_model_id=MODEL_ID_flash):
@@ -267,9 +282,6 @@ def correct_boxes_with_gemini(pid, label, pdf_path, original_feedbacks,
return global_feedbacks + corrected_feedbacks
import shutil
import grouping
def get_next_group_idx(label):
"""Finds the next available Group index for a given label."""
target_folder = GROUPS_DIR / label
@@ -278,8 +290,6 @@ def get_next_group_idx(label):
if not existing: return 0
return max([int(f.stem.split("_")[1]) for f in existing])
from utils import read_all_labels, enonce_total
def handle_label_errors(pid, label, res, pdf_path):
"""Handles Gemini labeling errors, moves/copies files, and returns new tasks."""
new_tasks = []
@@ -332,7 +342,7 @@ def handle_label_errors(pid, label, res, pdf_path):
tprint(f"\tHandling additional-answer for {pid} {label}")
try:
add_labels = json.loads(call_gemini_with_retries(MODEL_ID_flash, contents, config))
except Exception:
except Exception: # noqa: BLE001 - invalid auxiliary model response
add_labels = []
keep_error = False
@@ -374,7 +384,7 @@ def handle_label_errors(pid, label, res, pdf_path):
def process_single_task(task_tuple, precomputed_response=None):
try:
global pro_count, flash_count, pro_quota_exhausted
global pro_count, flash_count
file_path = task_tuple[0]
label = task_tuple[1]
can_spawn_tasks = task_tuple[2] if len(task_tuple) > 2 else True
@@ -383,8 +393,7 @@ def process_single_task(task_tuple, precomputed_response=None):
json_path = group_name + '.json'
new_tasks = []
with open(json_path, 'r') as f:
group_data = json.load(f)
group_data = read_json(json_path)
n = len(group_data)
d_data = {l[0]: (l[1], l[2], l[3]) for l in group_data}
@@ -466,7 +475,7 @@ def process_single_task(task_tuple, precomputed_response=None):
for (i,f) in enumerate(res["feedback"]):
b = f.get("box_2d")
if b:
ymin, xmin, ymax, xmax = b
ymin, _xmin, ymax, xmax = b
ymin = ymin * total_height // 1000
ymax = ymax * total_height // 1000
@@ -493,7 +502,7 @@ def process_single_task(task_tuple, precomputed_response=None):
res["feedback"] = correct_boxes_with_gemini(
pid, label, pdf_path, res["feedback"],
yming, ymaxg, width_r, total_height)
except Exception as e:
except Exception as e: # noqa: BLE001 - correction fallback
tprint(f"\tCorrection failed for Copie {pid}, {group_name} : {e}\n\tRemoving the boxes")
# Fallback if the second request fails entirely
for (i, f) in enumerate(res["feedback"]):
@@ -516,7 +525,7 @@ def process_single_task(task_tuple, precomputed_response=None):
tprint(f"Error decoding JSON for {file_path}", file=sys.stderr)
with io_lock:
errors_summary.append(("Error decoding JSON response", file_path))
except Exception as e:
except Exception as e: # noqa: BLE001 - per-task processing boundary
error_msg = f"Exception processing {file_path}: {e}"
print(error_msg, file=sys.stderr)
with io_lock:
@@ -602,19 +611,20 @@ def resolve_delayed_moves():
return new_tasks
if __name__ == "__main__":
def run_configured(args: argparse.Namespace) -> ExitCode:
global client, tasks_to_process
if client is None:
client = genai.Client(api_key=api_key)
if args.refaire:
refaire_path = INPUT_DIR / "refaire.json"
overwritten_path = INPUT_DIR / "overwritten_correction.json"
if refaire_path.exists():
with open(refaire_path, "r", encoding="utf-8") as f:
refaire_list = json.load(f)
refaire_list = read_json(refaire_path)
overwritten_data = []
if overwritten_path.exists():
with open(overwritten_path, "r", encoding="utf-8") as f:
overwritten_data = json.load(f)
overwritten_data = read_json(overwritten_path)
dirty_results = False
@@ -659,10 +669,12 @@ if __name__ == "__main__":
pdf_path = copie_dir / f"{label}.pdf"
is_new = False
if not pdf_path.exists():
if (copie_dir / f"{label}_new.pdf").exists():
pdf_path = copie_dir / f"{label}_new.pdf"
is_new = True
if (
not pdf_path.exists()
and (copie_dir / f"{label}_new.pdf").exists()
):
pdf_path = copie_dir / f"{label}_new.pdf"
is_new = True
# elif (copie_dir / f"{label}_old.pdf").exists():
# pdf_path = copie_dir / f"{label}_old.pdf"
@@ -690,7 +702,11 @@ if __name__ == "__main__":
input(f"About to batch from: {args.batch_from}. Press Enter to confirm...")
break
if args.batch_from not in all_labels:
sys.exit(f"Error: Label '{args.batch_from}' not found. Available labels: {all_labels}")
raise CliError(
f"Label '{args.batch_from}' not found. Available labels: "
f"{all_labels}",
ExitCode.INVALID_ARGUMENTS,
)
target_idx = all_labels.index(args.batch_from)
live_tasks = []
@@ -714,58 +730,66 @@ if __name__ == "__main__":
count_flash = 0
count_pro = 0
with open(batch_flash_file, "w", encoding="utf-8") as f_flash, \
open(batch_pro_file, "w", encoding="utf-8") as f_pro:
for task in batch_tasks:
file_path, label = task[0], task[1]
group_name = os.path.splitext(file_path)[0]
json_path = group_name + '.json'
with open(json_path, 'r') as jf:
group_data = json.load(jf)
use_flash = len(group_data) >= 4 or group_data[-1][2] <= 500
image_data = Path(file_path).read_bytes()
b64_img = base64.b64encode(image_data).decode("utf-8")
# Format payload matching Gemini Batch API file requirements
req = {
"key": file_path, # The ID returned in the output file
"request": {
"contents": [{
flash_lines = []
pro_lines = []
for task in batch_tasks:
file_path, label = task[0], task[1]
json_path = Path(file_path).with_suffix(".json")
group_data = read_json(json_path)
use_flash = len(group_data) >= 4 or group_data[-1][2] <= 500
b64_img = base64.b64encode(Path(file_path).read_bytes()).decode(
"utf-8"
)
request = {
"key": file_path,
"request": {
"contents": [
{
"role": "user",
"parts": [
{"inlineData": {"mimeType": "image/jpeg", "data": b64_img}},
{"text": prompting.make_prompt(INPUT_DIR,label)}
]
}],
"generation_config": {
"temperature": 1.0,
"topP": 0.95,
"maxOutputTokens": 65535,
"responseMimeType": "application/json",
"responseSchema": prompting.UNROLLED_SCHEMA
{
"inlineData": {
"mimeType": "image/jpeg",
"data": b64_img,
}
},
{"text": prompting.make_prompt(INPUT_DIR, label)},
],
}
}
}
],
"generation_config": {
"temperature": 1.0,
"topP": 0.95,
"maxOutputTokens": 65535,
"responseMimeType": "application/json",
"responseSchema": prompting.UNROLLED_SCHEMA,
},
},
}
line = json.dumps(request)
if use_flash:
flash_lines.append(line)
count_flash += 1
else:
pro_lines.append(line)
count_pro += 1
atomic_write_text(
batch_flash_file,
"\n".join(flash_lines) + ("\n" if flash_lines else ""),
)
atomic_write_text(
batch_pro_file,
"\n".join(pro_lines) + ("\n" if pro_lines else ""),
)
if use_flash:
f_flash.write(json.dumps(req) + "\n")
count_flash += 1
else:
f_pro.write(json.dumps(req) + "\n")
count_pro += 1
print(f"Batch generation complete.")
print("Batch generation complete.")
print(f" - {count_flash} requests saved to {batch_flash_file} (for {MODEL_ID_flash})")
print(f" - {count_pro} requests saved to {batch_pro_file} (for {MODEL_ID_pro})")
print("Upload these files via the File API and create two separate batch jobs.")
# If there's no live tasks to do, and we aren't doing a batched ingestion, exit right away
if not tasks_to_process and not args.deal_with_batched:
sys.exit(0)
return ExitCode.SUCCESS
batched_responses = {}
if args.deal_with_batched:
@@ -807,8 +831,11 @@ if __name__ == "__main__":
if new_generated_tasks:
for new_task in new_generated_tasks:
futures[executor.submit(process_single_task, new_task)] = new_task
except Exception as e:
except Exception as e: # noqa: BLE001 - future boundary
print(f"Exception during task execution: {e}", file=sys.stderr)
failed_task = futures[future]
with io_lock:
errors_summary.append((str(e), failed_task[0]))
tasks_to_process = [] # Vider la liste une fois traitée
@@ -840,9 +867,12 @@ if __name__ == "__main__":
if unresolved_delayed:
manual_path = INPUT_DIR / "manual_resolutions.txt"
with open(manual_path, "w", encoding="utf-8") as f:
f.write("### Use -> x>, -x, ss, sx, xx, xs\n")
f.write("\n".join(unresolved_delayed) + "\n")
atomic_write_text(
manual_path,
"### Use -> x>, -x, ss, sx, xx, xs\n"
+ "\n".join(unresolved_delayed)
+ "\n",
)
print(f"\n[!] Unresolved delayed tasks found! Wrote to {manual_path}.")
print(" Please edit it manually, then run `python resolve_manual.py <InputDir>`")
@@ -855,3 +885,101 @@ if __name__ == "__main__":
print(err, file=sys.stderr)
escaped_path = shlex.quote(str(file))
print(f"Run : python correction.py {escaped_path}")
return ExitCode.PARTIAL if errors_summary else ExitCode.SUCCESS
def run(
workspace: EvaluationWorkspace,
targets: list[Path],
args: argparse.Namespace,
*,
api_client=None,
) -> ExitCode:
if args.reset:
workspace.require_directories("Copies")
reset_workspace(workspace)
return ExitCode.SUCCESS
workspace.require_directories("Copies", "Par label")
workspace.require_files("labels")
if args.refaire:
workspace.require_files("refaire.json")
discovered, warnings = discover_tasks(workspace, targets)
for warning in warnings:
print(f"Warning: {warning}")
configure_runtime(workspace, discovered, args, api_client=api_client)
if not discovered and not args.refaire:
return ExitCode.PARTIAL
try:
status = run_configured(args)
finally:
for thread_id in list(thread_logs):
flush_thread_log(thread_id)
if warnings and status == ExitCode.SUCCESS:
return ExitCode.PARTIAL
return status
def build_parser() -> argparse.ArgumentParser:
parser = target_parser("Correct grouped answers with Gemini")
parser.add_argument(
"additional_targets",
nargs="*",
type=Path,
help="Additional group JPG files from the same evaluation",
)
parser.add_argument("--overwrite", action="store_true", help="Redo requests")
parser.add_argument("--limit", type=int, help="Maximum Gemini Pro calls")
parser.add_argument(
"--refaire",
action="store_true",
help="Redo copies and labels listed in refaire.json",
)
parser.add_argument(
"--batch",
action="store_true",
help="Generate Gemini batch request JSONL files",
)
parser.add_argument(
"--batch-from",
metavar="LABEL",
help="Process earlier labels live and batch from LABEL onward",
)
parser.add_argument(
"--deal-with-batched",
action="store_true",
help="Consume batched_correction_result.jsonl",
)
parser.add_argument(
"--reset",
action="store_true",
help="Delete correction state, restore _old PDFs, and delete _new PDFs",
)
return parser
def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
def handle(args: argparse.Namespace) -> ExitCode:
workspace, target = workspace_from_target(args)
targets = [target]
for additional in args.additional_targets:
resolved = additional.expanduser().resolve()
if not resolved.exists():
raise CliError(
f"Target does not exist: {resolved}",
ExitCode.INVALID_WORKSPACE,
)
if EvaluationWorkspace.discover(resolved).root != workspace.root:
raise CliError(
"All targets must belong to the same evaluation",
ExitCode.INVALID_ARGUMENTS,
)
targets.append(resolved)
return run(workspace, targets, args)
return execute(parser, argv, handle)
if __name__ == "__main__":
raise SystemExit(main())