sebastien/Copies
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Using folders "Copies" and "Par label", hopefully

Browse Source
This commit is contained in:
sebastien
2026-05-17 11:25:36 +02:00
parent 7e7045293a
commit c6c1a052e1
12 changed files with 142 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
correction.py
+44 -34
View File
@@ -38,14 +38,20 @@ for path_str in args.paths:
# Handle individual file
# Note: assumes structure InterroTest/Ex 2/Group_1.jpg to get parents[1]
label = arg_path.parent.name
INPUT_DIR = arg_path.parent.parent.parent
COPIES_DIR = INPUT_DIR / "Copies"
GROUPS_DIR = INPUT_DIR / "Par label"
tasks.append((str(arg_path), label))
if label not in results:
results[label] = []
elif arg_path.is_dir():
INPUT_DIR = arg_path
COPIES_DIR = INPUT_DIR / "Copies"
GROUPS_DIR = INPUT_DIR / "Par label"
# Handle directory (original behavior)
for sub in arg_path.iterdir():
if sub.is_dir() and sub.name.startswith("Ex"):
for sub in GROUPS_DIR.iterdir():
if sub.is_dir():
label = sub.name
if label not in results:
results[label] = []
@@ -145,7 +151,7 @@ do not score or give feedback to any other question."""
def make_prompt(full_label):
def read_longest_prefix_file(subdir):
dir_path = Path(INPUT_DIR) / subdir
dir_path = INPUT_DIR / subdir
matches = [f for f in dir_path.iterdir()
if f.is_file()
and full_label.startswith(f.name)
@@ -167,7 +173,6 @@ from google.genai import types
import base64
import shlex
import json
from pathlib import Path
import os
import threading
import concurrent.futures
@@ -210,7 +215,7 @@ def flush_thread_log(tid=None):
tid = tid or threading.current_thread().name
with log_lock:
if thread_logs.get(tid):
with open(Path(INPUT_DIR) / "correction_log", "a", encoding="utf-8") as f:
with open(INPUT_DIR / "correction_log", "a", encoding="utf-8") as f:
f.write(f"--- Task Log [{tid}] ---\n")
f.write("\n".join(thread_logs[tid]) + "\n\n")
thread_logs[tid].clear()
@@ -311,8 +316,8 @@ def generate_request(file, full_label):
return (contents, generate_content_config)
client = genai.Client(api_key=api_key)
output_path = Path(INPUT_DIR) / "correction.json"
progress_path = Path(INPUT_DIR) / "correction_progress.json"
output_path = INPUT_DIR / "correction.json"
progress_path = INPUT_DIR / "correction_progress.json"
start_time = time.time()
overwrite = args.overwrite
limit = args.limit
@@ -407,9 +412,9 @@ def get_single_image_bytes(pdf_path):
return img_byte_arr.getvalue()
def correct_boxes_with_gemini(pid, label, original_feedbacks,
root_dir, yming, ymaxg, width_r, total_height):
yming, ymaxg, width_r, total_height):
"""Requests corrected bounding boxes from Gemini Flash on the single image."""
pdf_path = Path(root_dir) / f"Copie{pid}" / f"{label}.pdf"
pdf_path = COPIES_DIR / f"Copie{pid}" / f"{label}.pdf"
img_bytes = get_single_image_bytes(pdf_path)
localized_feedbacks = [f for f in original_feedbacks if f["box_2d"]]
@@ -473,9 +478,9 @@ it goes wrong, and the feedback is what went wrong.
import shutil
import grouping
def get_next_group_idx(root_dir, label):
def get_next_group_idx(label):
"""Finds the next available Group index for a given label."""
target_folder = Path(root_dir) / label
target_folder = GROUPS_DIR / label
target_folder.mkdir(exist_ok=True)
existing = list(target_folder.glob("Group_*.jpg"))
if not existing: return 0
@@ -489,7 +494,7 @@ def handle_label_errors(pid, label, res, pdf_path):
error_type = res.get("error")
all_labels = read_all_labels(INPUT_DIR)
labels_txt = (Path(INPUT_DIR) / "labels").read_text(encoding="utf-8", errors="replace")
labels_txt = (INPUT_DIR / "labels").read_text(encoding="utf-8", errors="replace")
enonce = enonce_total(INPUT_DIR)
if error_type == "wrong-label":
@@ -523,7 +528,7 @@ Here is a list of all possible labels. You need to answer with one of these :
if new_label == label:
res["error"] = ""
return []
new_pdf_path = Path(INPUT_DIR) / f"Copie{pid}" / f"{new_label}.pdf"
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}.pdf"
if new_pdf_path.exists():
tprint(f"\t\tCopie{pid} tried to move wrong {label} to {new_label}, but it already exists.")
res["error"] = f"wrg-lbl:{new_label}?exists"
@@ -533,12 +538,12 @@ Here is a list of all possible labels. You need to answer with one of these :
shutil.move(str(pdf_path), str(new_pdf_path))
# Since we moved the file, this Copie/label should not be taken
# into account in the future, I think
idx = get_next_group_idx(INPUT_DIR, new_label)
idx = get_next_group_idx(new_label)
height = grouping.get_pdf_height(str(new_pdf_path))
grouping.create_jpg(new_label, idx, [(pid, str(new_pdf_path), height)],
INPUT_DIR)
GROUPS_DIR)
tprint(f"\t\tMaking {new_label} group {idx+1}")
new_tasks.append((str(Path(INPUT_DIR) / new_label / f"Group_{idx+1}.jpg"),
new_tasks.append((str(GROUPS_DIR / new_label / f"Group_{idx+1}.jpg"),
new_label, False))
elif error_type == "additional-answer":
@@ -580,15 +585,15 @@ Here is a list of all possible labels. You need to answer with a list one of the
error += f"{add_label}??"
keep_error = True
continue
new_pdf_path = Path(INPUT_DIR) / f"Copie{pid}" / f"{add_label}.pdf"
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{add_label}.pdf"
if not new_pdf_path.exists():
shutil.copy(str(pdf_path), str(new_pdf_path))
tprint(f"\t\tCopying Copie{pid} : {label} -> {add_label}")
idx = get_next_group_idx(INPUT_DIR, add_label)
idx = get_next_group_idx(add_label)
tprint(f"\t\tMaking {add_label} group {idx+1}")
height = grouping.get_pdf_height(str(new_pdf_path))
grouping.create_jpg(add_label, idx, [(pid, str(new_pdf_path), height)], INPUT_DIR)
new_tasks.append((str(Path(INPUT_DIR) / add_label / f"Group_{idx+1}.jpg"),
grouping.create_jpg(add_label, idx, [(pid, str(new_pdf_path), height)], GROUPS_DIR)
new_tasks.append((str(GROUPS_DIR / add_label / f"Group_{idx+1}.jpg"),
add_label, False))
error += f"(->){add_label}"
keep_error = True
@@ -657,7 +662,7 @@ def process_single_task(task_tuple, precomputed_response=None):
res = p["result"]
yming, ymaxg, width_r = d_data[pid]
pdf_path = Path(INPUT_DIR) / f"Copie{pid}" / f"{label}.pdf"
pdf_path = COPIES_DIR / f"Copie{pid}" / f"{label}.pdf"
if (not can_spawn_tasks) and res["error"] == "additional-answer":
tprint("\tSwallowing an additional-answer from a subsequent task.")
res["error"]= ""
@@ -680,17 +685,22 @@ def process_single_task(task_tuple, precomputed_response=None):
pid, label, group_name)
continue
if (ymin < yming - 50 or
ymax > ymaxg + 50 or
xmax / 1000 > width_r):
if (ymin < yming - 50 or ymax > ymaxg + 50 or xmax / 1000 > width_r):
needs_correction.append(i)
break
if ymin < yming - 5:
ymin = yming - 5
b[0] = ymin * 1000 // total_height
if ymax > ymaxg + 5:
ymax = ymaxg + 5
b[2] = ymax * 1000 // total_height
if needs_correction:
tprint(f"\tBox anomalies detected for Copie {pid} {group_name}. \n\tRequesting isolated correction from Gemini Flash...")
try:
res["feedback"] = correct_boxes_with_gemini(
pid, label, res["feedback"], INPUT_DIR,
pid, label, res["feedback"],
yming, ymaxg, width_r, total_height)
except Exception as e:
tprint(f"\tCorrection failed for Copie {pid}, {group_name} : {e}\n\tRemoving the boxes")
@@ -726,8 +736,8 @@ def process_single_task(task_tuple, precomputed_response=None):
if __name__ == "__main__":
if args.refaire:
refaire_path = Path(INPUT_DIR) / "refaire.json"
overwritten_path = Path(INPUT_DIR) / "overwritten_correction.json"
refaire_path = INPUT_DIR / "refaire.json"
overwritten_path = INPUT_DIR / "overwritten_correction.json"
if refaire_path.exists():
with open(refaire_path, "r", encoding="utf-8") as f:
@@ -742,7 +752,7 @@ if __name__ == "__main__":
for copie_name, labels in refaire_list:
pid = copie_name.replace("Copie", "")
copie_dir = Path(INPUT_DIR) / copie_name
copie_dir = COPIES_DIR / copie_name
# If list is empty, redo all labels available for this Copie
if not labels:
@@ -772,10 +782,10 @@ if __name__ == "__main__":
# 2. Make new group and add to tasks
pdf_path = copie_dir / f"{label}.pdf"
if pdf_path.exists():
idx = get_next_group_idx(INPUT_DIR, label)
idx = get_next_group_idx(label)
height = grouping.get_pdf_height(str(pdf_path))
grouping.create_jpg(label, idx, [(pid, str(pdf_path), height)], INPUT_DIR)
new_group_path = str(Path(INPUT_DIR) / label / f"Group_{idx+1}.jpg")
grouping.create_jpg(label, idx, [(pid, str(pdf_path), height)], GROUPS_DIR)
new_group_path = str(GROUPS_DIR / label / f"Group_{idx+1}.jpg")
tasks_to_process.append((new_group_path, label))
if dirty_results:
@@ -813,8 +823,8 @@ if __name__ == "__main__":
tasks_to_process = [] # Run nothing live if just `--batch`
if batch_tasks:
batch_flash_file = Path(INPUT_DIR) / "batch_requests_flash.jsonl"
batch_pro_file = Path(INPUT_DIR) / "batch_requests_pro.jsonl"
batch_flash_file = INPUT_DIR / "batch_requests_flash.jsonl"
batch_pro_file = INPUT_DIR / "batch_requests_pro.jsonl"
count_flash = 0
count_pro = 0
@@ -873,7 +883,7 @@ if __name__ == "__main__":
batched_responses = {}
if args.deal_with_batched:
batch_results_path = Path(INPUT_DIR) / "batched_correction_result.jsonl"
batch_results_path = INPUT_DIR / "batched_correction_result.jsonl"
if batch_results_path.exists():
print(f"Loading batch results from {batch_results_path}...")
with open(batch_results_path, "r", encoding="utf-8") as f: