DS09 : Conflict resolution, and other
This commit is contained in:
+116
-43
@@ -29,6 +29,8 @@ parser.add_argument("--batch-from", type=str, metavar="LABEL",
|
||||
help="Do live requests before LABEL, and batch requests from LABEL onwards")
|
||||
parser.add_argument("--deal-with-batched", action="store_true",
|
||||
help="Process a JSONL file containing completed batch results")
|
||||
parser.add_argument("--reset", action="store_true",
|
||||
help="Remove correction.json, revert _old.pdf, delete _new.pdf, then exit")
|
||||
args, _ = parser.parse_known_args()
|
||||
|
||||
tasks = [] # List of tuples: (filepath_str, label_str)
|
||||
@@ -122,6 +124,36 @@ signal.signal(signal.SIGTERM, handle_interrupt)
|
||||
client = genai.Client(api_key=api_key)
|
||||
output_path = INPUT_DIR / "correction.json"
|
||||
progress_path = INPUT_DIR / "correction_progress.json"
|
||||
|
||||
if args.reset:
|
||||
print("--- Running Reset ---")
|
||||
if output_path.exists():
|
||||
output_path.unlink()
|
||||
print(f"Deleted: {output_path}")
|
||||
if progress_path.exists():
|
||||
progress_path.unlink()
|
||||
print(f"Deleted: {progress_path}")
|
||||
|
||||
if COPIES_DIR.exists():
|
||||
for copie_dir in COPIES_DIR.iterdir():
|
||||
if not copie_dir.is_dir():
|
||||
continue
|
||||
|
||||
# Revert _old.pdf files
|
||||
for old_pdf in copie_dir.glob("*_old.pdf"):
|
||||
orig_pdf = copie_dir / old_pdf.name.replace("_old.pdf", ".pdf")
|
||||
if orig_pdf.exists():
|
||||
orig_pdf.unlink() # Prevent FileExistsError on Windows
|
||||
old_pdf.rename(orig_pdf)
|
||||
print(f"Moved: {copie_dir.name}/{old_pdf.name} -> {orig_pdf.name}")
|
||||
|
||||
# Delete _new.pdf files
|
||||
for new_pdf in copie_dir.glob("*_new.pdf"):
|
||||
new_pdf.unlink()
|
||||
print(f"Deleted: {copie_dir.name}/{new_pdf.name}")
|
||||
|
||||
sys.exit("Reset almost complete. For each deleted `_new`, you should manually delete the group in `Par label`")
|
||||
|
||||
start_time = time.time()
|
||||
overwrite = args.overwrite
|
||||
limit = args.limit
|
||||
@@ -269,9 +301,11 @@ def handle_label_errors(pid, label, res, pdf_path):
|
||||
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}_new.pdf"
|
||||
|
||||
if base_new_pdf_path.exists() or new_pdf_path.exists():
|
||||
tprint(f"\t\tCopie{pid} tried to move wrong {label} to {new_label}, but it already exists. Delaying.")
|
||||
tprint(f"""\t\tCopie{pid} tried to move wrong {label} to {new_label},
|
||||
but it already exists. Delaying.""")
|
||||
# res["error"] = f"wrg-lbl:{new_label}?exists"
|
||||
res["error"] = f"wrg-lbl:{new_label}?delayed"
|
||||
res["error"] = f"wrg-lbl:{new_label}?"
|
||||
res.setdefault("delayed", []).append(["wrong-label", new_label])
|
||||
else:
|
||||
res["error"] = f"wrg-lbl-moved-to:{new_label}"
|
||||
tprint(f"\t\tCopie{pid} : moving wrong {label} to {new_label}.")
|
||||
@@ -324,8 +358,8 @@ def handle_label_errors(pid, label, res, pdf_path):
|
||||
keep_error = True
|
||||
else:
|
||||
keep_error = True
|
||||
# error += f"(xx){add_label}"
|
||||
error += f"(delayed){add_label}"
|
||||
error += f"(->){add_label}?"
|
||||
res.setdefault("delayed", []).append(["add-label", add_label])
|
||||
tprint(f"\t\tAlready present (not copied) Copie{pid} : {label} -> {add_label}. Delaying.")
|
||||
if not keep_error:
|
||||
res["error"] = ""
|
||||
@@ -496,8 +530,9 @@ def resolve_delayed_moves():
|
||||
for label, batches in results.items():
|
||||
for batch in batches:
|
||||
for p in batch:
|
||||
err = p.get("result", {}).get("error", "")
|
||||
if not err or ("?delayed" not in err and "(delayed)" not in err):
|
||||
res = p.get("result", {})
|
||||
delayed_list = res.get("delayed", [])
|
||||
if not delayed_list:
|
||||
continue
|
||||
|
||||
pid = p["id"]
|
||||
@@ -509,55 +544,58 @@ def resolve_delayed_moves():
|
||||
elif pdf_path.with_name(f"{label}_old.pdf").exists():
|
||||
pdf_path = pdf_path.with_name(f"{label}_old.pdf")
|
||||
|
||||
# 1. Résolution de wrong-label
|
||||
if err.startswith("wrg-lbl:") and "?delayed" in err:
|
||||
new_label = err.split(":")[1].split("?")[0]
|
||||
base_new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}.pdf"
|
||||
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}_new.pdf"
|
||||
remaining_delayed = []
|
||||
resolved_any = False
|
||||
|
||||
# Si la place s'est libérée (l'ancien a été bougé vers _old)
|
||||
if not base_new_pdf_path.exists() and not new_pdf_path.exists():
|
||||
tprint(f"Resolving delayed move: Copie{pid} {label} -> {new_label}")
|
||||
p["result"]["error"] = f"wrg-lbl-moved-to:{new_label}"
|
||||
p["result"]["suffixe"] = "_old" # Très important pour l'ignorer ensuite
|
||||
for delay_type, target_label in delayed_list:
|
||||
# 1. Résolution de wrong-label
|
||||
if delay_type == "wrong-label":
|
||||
base_new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}.pdf"
|
||||
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}_new.pdf"
|
||||
|
||||
shutil.copy(str(pdf_path), str(new_pdf_path))
|
||||
old_pdf_path = pdf_path.with_name(f"{label}_old.pdf")
|
||||
if pdf_path != old_pdf_path:
|
||||
shutil.move(str(pdf_path), str(old_pdf_path))
|
||||
if not base_new_pdf_path.exists() and not new_pdf_path.exists():
|
||||
tprint(f"Resolving delayed move: Copie{pid} {label} -> {target_label}")
|
||||
res["error"] = f"wrg-lbl-moved-to:{target_label}"
|
||||
res["suffix"] = "_old" # Fixed typo: was suffixe
|
||||
resolved_any = True
|
||||
|
||||
idx = get_next_group_idx(new_label)
|
||||
height = grouping.get_pdf_height(str(new_pdf_path))
|
||||
grouping.create_jpg(new_label, idx, [(pid, str(new_pdf_path), height)], GROUPS_DIR)
|
||||
new_tasks.append((str(GROUPS_DIR / new_label / f"Group_{idx+1}.jpg"), new_label, False))
|
||||
shutil.copy(str(pdf_path), str(new_pdf_path))
|
||||
old_pdf_path = pdf_path.with_name(f"{label}_old.pdf")
|
||||
if pdf_path != old_pdf_path:
|
||||
shutil.move(str(pdf_path), str(old_pdf_path))
|
||||
|
||||
# 2. Résolution de additional-answer
|
||||
elif err.startswith("al:") and "(delayed)" in err:
|
||||
import re
|
||||
delayed_matches = re.findall(r'\(delayed\)([^?()]+)', err)
|
||||
new_err = err
|
||||
resolved_any = False
|
||||
idx = get_next_group_idx(target_label)
|
||||
height = grouping.get_pdf_height(str(new_pdf_path))
|
||||
grouping.create_jpg(target_label, idx, [(pid, str(new_pdf_path), height)], GROUPS_DIR)
|
||||
new_tasks.append((str(GROUPS_DIR / target_label / f"Group_{idx+1}.jpg"), target_label, False))
|
||||
else:
|
||||
remaining_delayed.append([delay_type, target_label])
|
||||
|
||||
for add_label in delayed_matches:
|
||||
base_add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{add_label}.pdf"
|
||||
add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{add_label}_new.pdf"
|
||||
# 2. Résolution de additional-answer
|
||||
elif delay_type == "add-label":
|
||||
base_add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}.pdf"
|
||||
add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}_new.pdf"
|
||||
|
||||
if not base_add_pdf_path.exists() and not add_pdf_path.exists():
|
||||
tprint(f"Resolving delayed additional-answer: Copie{pid} {label} -> {add_label}")
|
||||
new_err = new_err.replace(f"(delayed){add_label}", f"(->){add_label}")
|
||||
tprint(f"Resolving delayed additional-answer: Copie{pid} {label} -> {target_label}")
|
||||
res["error"] = res["error"].replace(f"(xx){target_label}", f"(->){target_label}")
|
||||
resolved_any = True
|
||||
|
||||
shutil.copy(str(pdf_path), str(add_pdf_path))
|
||||
idx = get_next_group_idx(add_label)
|
||||
idx = get_next_group_idx(target_label)
|
||||
height = grouping.get_pdf_height(str(add_pdf_path))
|
||||
grouping.create_jpg(add_label, idx, [(pid, str(add_pdf_path), height)], GROUPS_DIR)
|
||||
new_tasks.append((str(GROUPS_DIR / add_label / f"Group_{idx+1}.jpg"), add_label, False))
|
||||
grouping.create_jpg(target_label, idx, [(pid, str(add_pdf_path), height)], GROUPS_DIR)
|
||||
new_tasks.append((str(GROUPS_DIR / target_label / f"Group_{idx+1}.jpg"), target_label, False))
|
||||
else:
|
||||
remaining_delayed.append([delay_type, target_label])
|
||||
|
||||
if resolved_any:
|
||||
p["result"]["error"] = new_err
|
||||
if resolved_any:
|
||||
if remaining_delayed:
|
||||
res["delayed"] = remaining_delayed
|
||||
else:
|
||||
del res["delayed"]
|
||||
|
||||
if new_tasks:
|
||||
# Sauvegarder les modifications d'erreurs (les tags delayed enlevés)
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
@@ -585,7 +623,15 @@ if __name__ == "__main__":
|
||||
|
||||
# If list is empty, redo all labels available for this Copie
|
||||
if not labels:
|
||||
labels = [p.stem for p in copie_dir.glob("*.pdf")]
|
||||
labels_set = set()
|
||||
for p in copie_dir.glob("*.pdf"):
|
||||
if p.name.endswith("_old.pdf"):
|
||||
continue # Strictly ignore old files
|
||||
elif p.name.endswith("_new.pdf"):
|
||||
labels_set.add(p.stem[:-4]) # Strip '_new' to get base label
|
||||
else:
|
||||
labels_set.add(p.stem)
|
||||
labels = list(labels_set)
|
||||
|
||||
for label in labels:
|
||||
# 1. Extract and backup old corrections
|
||||
@@ -610,9 +656,12 @@ if __name__ == "__main__":
|
||||
|
||||
# 2. Make new group and add to tasks
|
||||
pdf_path = copie_dir / f"{label}.pdf"
|
||||
is_new = False
|
||||
|
||||
if not pdf_path.exists():
|
||||
if (copie_dir / f"{label}_new.pdf").exists():
|
||||
pdf_path = copie_dir / f"{label}_new.pdf"
|
||||
is_new = True
|
||||
# elif (copie_dir / f"{label}_old.pdf").exists():
|
||||
# pdf_path = copie_dir / f"{label}_old.pdf"
|
||||
|
||||
@@ -621,7 +670,7 @@ if __name__ == "__main__":
|
||||
height = grouping.get_pdf_height(str(pdf_path))
|
||||
grouping.create_jpg(label, idx, [(pid, str(pdf_path), height)], GROUPS_DIR)
|
||||
new_group_path = str(GROUPS_DIR / label / f"Group_{idx+1}.jpg")
|
||||
tasks_to_process.append((new_group_path, label))
|
||||
tasks_to_process.append((new_group_path, label, not is_new))
|
||||
|
||||
if dirty_results:
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
@@ -774,6 +823,30 @@ if __name__ == "__main__":
|
||||
else:
|
||||
made_progress = False
|
||||
|
||||
# Check for remaining unresolved delayed tasks
|
||||
unresolved_delayed = []
|
||||
with io_lock:
|
||||
for label, batches in results.items():
|
||||
for batch in batches:
|
||||
for p in batch:
|
||||
res = p.get("result", {})
|
||||
delayed = res.get("delayed", [])
|
||||
pid = p["id"]
|
||||
|
||||
for delay_type, target_label in delayed:
|
||||
if delay_type == "wrong-label":
|
||||
unresolved_delayed.append(f"Copie{pid} {label} x> {target_label}|")
|
||||
elif delay_type == "add-label":
|
||||
unresolved_delayed.append(f"Copie{pid} {label} -> {target_label}|")
|
||||
|
||||
if unresolved_delayed:
|
||||
manual_path = INPUT_DIR / "manual_resolutions.txt"
|
||||
with open(manual_path, "w", encoding="utf-8") as f:
|
||||
f.write("### Use -> x>, -x, ss, sx, xx, xs\n")
|
||||
f.write("\n".join(unresolved_delayed) + "\n")
|
||||
print(f"\n[!] Unresolved delayed tasks found! Wrote to {manual_path}.")
|
||||
print(" Please edit it manually, then run `python resolve_manual.py <InputDir>`")
|
||||
|
||||
end_time = time.time()
|
||||
print("Time elapsed : ", end_time - start_time)
|
||||
print("Requests to pro / flash : ", pro_count, flash_count)
|
||||
|
||||
Reference in New Issue
Block a user