Miscs
This commit is contained in:
+111
-22
@@ -186,7 +186,7 @@ def call_gemini_with_retries(model_id, contents, config,
|
||||
tprint(f"\tGemini Pro minute limit hit. Waiting {wait_time:.1f}s...")
|
||||
time.sleep(wait_time)
|
||||
continue # Retry same model
|
||||
|
||||
|
||||
# Immediately fallback to Flash without waiting if it's a Pro quota error
|
||||
if is_quota_error and model_id == MODEL_ID_pro and fallback_model_id:
|
||||
tprint(f"\tGemini Pro quota hit ({e}). \n\n\tFalling back to Flash permanently...")
|
||||
@@ -269,8 +269,9 @@ def handle_label_errors(pid, label, res, pdf_path):
|
||||
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}_new.pdf"
|
||||
|
||||
if base_new_pdf_path.exists() or new_pdf_path.exists():
|
||||
tprint(f"\t\tCopie{pid} tried to move wrong {label} to {new_label}, but it already exists.")
|
||||
res["error"] = f"wrg-lbl:{new_label}?exists"
|
||||
tprint(f"\t\tCopie{pid} tried to move wrong {label} to {new_label}, but it already exists. Delaying.")
|
||||
# res["error"] = f"wrg-lbl:{new_label}?exists"
|
||||
res["error"] = f"wrg-lbl:{new_label}?delayed"
|
||||
else:
|
||||
res["error"] = f"wrg-lbl-moved-to:{new_label}"
|
||||
tprint(f"\t\tCopie{pid} : moving wrong {label} to {new_label}.")
|
||||
@@ -323,8 +324,9 @@ def handle_label_errors(pid, label, res, pdf_path):
|
||||
keep_error = True
|
||||
else:
|
||||
keep_error = True
|
||||
error += f"(xx){add_label}"
|
||||
tprint(f"\t\tAlready present (not copied) Copie{pid} : {label} -> {add_label}")
|
||||
# error += f"(xx){add_label}"
|
||||
error += f"(delayed){add_label}"
|
||||
tprint(f"\t\tAlready present (not copied) Copie{pid} : {label} -> {add_label}. Delaying.")
|
||||
if not keep_error:
|
||||
res["error"] = ""
|
||||
else:
|
||||
@@ -487,6 +489,80 @@ def process_single_task(task_tuple, precomputed_response=None):
|
||||
finally:
|
||||
flush_thread_log()
|
||||
|
||||
def resolve_delayed_moves():
|
||||
"""Scans the current results to find delayed moves and executes them if space was freed."""
|
||||
new_tasks = []
|
||||
with io_lock:
|
||||
for label, batches in results.items():
|
||||
for batch in batches:
|
||||
for p in batch:
|
||||
err = p.get("result", {}).get("error", "")
|
||||
if not err or ("?delayed" not in err and "(delayed)" not in err):
|
||||
continue
|
||||
|
||||
pid = p["id"]
|
||||
pdf_path = COPIES_DIR / f"Copie{pid}" / f"{label}.pdf"
|
||||
|
||||
if not pdf_path.exists():
|
||||
if pdf_path.with_name(f"{label}_new.pdf").exists():
|
||||
pdf_path = pdf_path.with_name(f"{label}_new.pdf")
|
||||
elif pdf_path.with_name(f"{label}_old.pdf").exists():
|
||||
pdf_path = pdf_path.with_name(f"{label}_old.pdf")
|
||||
|
||||
# 1. Résolution de wrong-label
|
||||
if err.startswith("wrg-lbl:") and "?delayed" in err:
|
||||
new_label = err.split(":")[1].split("?")[0]
|
||||
base_new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}.pdf"
|
||||
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}_new.pdf"
|
||||
|
||||
# Si la place s'est libérée (l'ancien a été bougé vers _old)
|
||||
if not base_new_pdf_path.exists() and not new_pdf_path.exists():
|
||||
tprint(f"Resolving delayed move: Copie{pid} {label} -> {new_label}")
|
||||
p["result"]["error"] = f"wrg-lbl-moved-to:{new_label}"
|
||||
p["result"]["suffixe"] = "_old" # Très important pour l'ignorer ensuite
|
||||
|
||||
shutil.copy(str(pdf_path), str(new_pdf_path))
|
||||
old_pdf_path = pdf_path.with_name(f"{label}_old.pdf")
|
||||
if pdf_path != old_pdf_path:
|
||||
shutil.move(str(pdf_path), str(old_pdf_path))
|
||||
|
||||
idx = get_next_group_idx(new_label)
|
||||
height = grouping.get_pdf_height(str(new_pdf_path))
|
||||
grouping.create_jpg(new_label, idx, [(pid, str(new_pdf_path), height)], GROUPS_DIR)
|
||||
new_tasks.append((str(GROUPS_DIR / new_label / f"Group_{idx+1}.jpg"), new_label, False))
|
||||
|
||||
# 2. Résolution de additional-answer
|
||||
elif err.startswith("al:") and "(delayed)" in err:
|
||||
import re
|
||||
delayed_matches = re.findall(r'\(delayed\)([^?()]+)', err)
|
||||
new_err = err
|
||||
resolved_any = False
|
||||
|
||||
for add_label in delayed_matches:
|
||||
base_add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{add_label}.pdf"
|
||||
add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{add_label}_new.pdf"
|
||||
|
||||
if not base_add_pdf_path.exists() and not add_pdf_path.exists():
|
||||
tprint(f"Resolving delayed additional-answer: Copie{pid} {label} -> {add_label}")
|
||||
new_err = new_err.replace(f"(delayed){add_label}", f"(->){add_label}")
|
||||
resolved_any = True
|
||||
|
||||
shutil.copy(str(pdf_path), str(add_pdf_path))
|
||||
idx = get_next_group_idx(add_label)
|
||||
height = grouping.get_pdf_height(str(add_pdf_path))
|
||||
grouping.create_jpg(add_label, idx, [(pid, str(add_pdf_path), height)], GROUPS_DIR)
|
||||
new_tasks.append((str(GROUPS_DIR / add_label / f"Group_{idx+1}.jpg"), add_label, False))
|
||||
|
||||
if resolved_any:
|
||||
p["result"]["error"] = new_err
|
||||
|
||||
if new_tasks:
|
||||
# Sauvegarder les modifications d'erreurs (les tags delayed enlevés)
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
return new_tasks
|
||||
|
||||
if __name__ == "__main__":
|
||||
if args.refaire:
|
||||
refaire_path = INPUT_DIR / "refaire.json"
|
||||
@@ -666,24 +742,37 @@ if __name__ == "__main__":
|
||||
else:
|
||||
print(f"Warning: Batch results file {batch_results_path} not found.", file=sys.stderr)
|
||||
|
||||
print(f"Starting processing on {len(tasks_to_process)} tasks with {NB_THREADS} threads...")
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=NB_THREADS) as executor:
|
||||
futures = {}
|
||||
for task in tasks_to_process:
|
||||
file_path = task[0]
|
||||
precomp = batched_responses.get(file_path)
|
||||
futures[executor.submit(process_single_task, task, precomp)] = task
|
||||
made_progress = True
|
||||
while tasks_to_process or made_progress:
|
||||
if tasks_to_process:
|
||||
print(f"Starting processing on {len(tasks_to_process)} tasks with {NB_THREADS} threads...")
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=NB_THREADS) as executor:
|
||||
futures = {}
|
||||
for task in tasks_to_process:
|
||||
file_path = task[0]
|
||||
precomp = batched_responses.get(file_path)
|
||||
futures[executor.submit(process_single_task, task, precomp)] = task
|
||||
|
||||
# Process tasks as they complete, allowing dynamic task addition
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
new_generated_tasks = future.result()
|
||||
if new_generated_tasks:
|
||||
for new_task in new_generated_tasks:
|
||||
# New tasks from wrong-label/additional-answer will fallback to live API
|
||||
futures[executor.submit(process_single_task, new_task)] = new_task
|
||||
except Exception as e:
|
||||
print(f"Exception during task execution: {e}", file=sys.stderr)
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
new_generated_tasks = future.result()
|
||||
if new_generated_tasks:
|
||||
for new_task in new_generated_tasks:
|
||||
futures[executor.submit(process_single_task, new_task)] = new_task
|
||||
except Exception as e:
|
||||
print(f"Exception during task execution: {e}", file=sys.stderr)
|
||||
|
||||
tasks_to_process = [] # Vider la liste une fois traitée
|
||||
|
||||
# Après avoir traité toutes les tâches actuelles (live ou batched),
|
||||
# on tente de débloquer les mouvements qui étaient en attente
|
||||
delayed_tasks = resolve_delayed_moves()
|
||||
if delayed_tasks:
|
||||
print(f"Resolved {len(delayed_tasks)} delayed moves! Running executor for new tasks...")
|
||||
tasks_to_process.extend(delayed_tasks)
|
||||
made_progress = True
|
||||
else:
|
||||
made_progress = False
|
||||
|
||||
end_time = time.time()
|
||||
print("Time elapsed : ", end_time - start_time)
|
||||
|
||||
Reference in New Issue
Block a user