DS09 : Conflict resolution, and other
This commit is contained in:
+29
-5
@@ -1,7 +1,7 @@
|
|||||||
#+title: Script
|
#+title: Script
|
||||||
#+author: Sébastien Miquel
|
#+author: Sébastien Miquel
|
||||||
#+date: 14-03-2026
|
#+date: 14-03-2026
|
||||||
# Time-stamp: <06-06-26 16:23>
|
# Time-stamp: <29-06-26 13:05>
|
||||||
#+OPTIONS:
|
#+OPTIONS:
|
||||||
|
|
||||||
* Méta
|
* Méta
|
||||||
@@ -93,7 +93,7 @@ export GEMINI_API_KEY=…
|
|||||||
+ `correction.tex`.
|
+ `correction.tex`.
|
||||||
- `python gemini_for_enonce.py Interro`
|
- `python gemini_for_enonce.py Interro`
|
||||||
Se charge de créer des dossiers `Text` et `Sol` avec
|
Se charge de créer des dossiers `Text` et `Sol` avec
|
||||||
+ Le fichier `Text` contient
|
+ Le fichier `Text` contient
|
||||||
|
|
||||||
** Prétraitement des copies
|
** Prétraitement des copies
|
||||||
|
|
||||||
@@ -195,6 +195,19 @@ Set proxy with ~export HTTPS_PROXY="http://10.0.0.1:3128"~
|
|||||||
- Essaye de corriger des erreurs d'encodage/d'accents dans
|
- Essaye de corriger des erreurs d'encodage/d'accents dans
|
||||||
=correction.json=.
|
=correction.json=.
|
||||||
- aussi échappe les `_` en dehors du mode math, pour LaTeX.
|
- aussi échappe les `_` en dehors du mode math, pour LaTeX.
|
||||||
|
4. Résolution manuel de conflits
|
||||||
|
|
||||||
|
Edit `manual_resolutions.txt`. Use :
|
||||||
|
+ `->` or `x>` : Here set a pipe `|` before or after the new_label name
|
||||||
|
+ `-x` : replace the goal
|
||||||
|
+ `ss` : do nothing
|
||||||
|
+ `sx` : stay, and remove goal.
|
||||||
|
+ `xx` : move to goal.
|
||||||
|
+ `xs` : remove old, keep goal.
|
||||||
|
|
||||||
|
Then call `python resolve_manual.py Interro`
|
||||||
|
5. Call `python correction.py Interro --refaire`.
|
||||||
|
|
||||||
|
|
||||||
** Génération des copies annotées
|
** Génération des copies annotées
|
||||||
|
|
||||||
@@ -282,8 +295,8 @@ groupée into refaire !!
|
|||||||
+ =python plotting.py InterroTest/Copie01.pdf=
|
+ =python plotting.py InterroTest/Copie01.pdf=
|
||||||
+ =python splitting_int.py InterroTest/Copie20.pdf=
|
+ =python splitting_int.py InterroTest/Copie20.pdf=
|
||||||
2. Créer =refaire.json=, avec un contenu comme
|
2. Créer =refaire.json=, avec un contenu comme
|
||||||
: [["Copie01", []],
|
: [["Copie02", []],
|
||||||
: ["Copie01", ["Ex 1 : 1)"]]]
|
: ["Copie01", ["Ex 1 : 1)"]]]
|
||||||
3. Appeler =correction= avec --refaire. Il doit créer des groupes
|
3. Appeler =correction= avec --refaire. Il doit créer des groupes
|
||||||
individuels, faire des requêtes, et remplacer les corrections
|
individuels, faire des requêtes, et remplacer les corrections
|
||||||
précédentes (à sauver ailleurs).
|
précédentes (à sauver ailleurs).
|
||||||
@@ -291,7 +304,18 @@ groupée into refaire !!
|
|||||||
Ou non, si tu veux le faire à la main.
|
Ou non, si tu veux le faire à la main.
|
||||||
4. ?? Si je fais refaire, avant d'avoir créer les annotating with
|
4. ?? Si je fais refaire, avant d'avoir créer les annotating with
|
||||||
checks, que se passe-t-il ???
|
checks, que se passe-t-il ???
|
||||||
5. Appeler =annotating_with_checks.py --refaire --overwrite= avec --refaire.
|
5. Appeler =annotating_with_checks.py --refaire --overwrite=
|
||||||
6. =python to_tablette.py --refaire Interro24=
|
6. =python to_tablette.py --refaire Interro24=
|
||||||
6. =python from_tablette.py --refaire Interro24=
|
6. =python from_tablette.py --refaire Interro24=
|
||||||
7. =python reading_grouped_annotations.py --refaire Interro24=
|
7. =python reading_grouped_annotations.py --refaire Interro24=
|
||||||
|
|
||||||
|
** Exemple de replotting, refaire d'une copie
|
||||||
|
|
||||||
|
1. replot it.
|
||||||
|
2. `python splitting_int.py DS09VA/Copies/Copie25.pdf`
|
||||||
|
this will get rid of old/new.
|
||||||
|
!! Attention, et si ça dégage un new : bad bad bad.
|
||||||
|
3. Make `refaire.json`, avec la copie, et les labels à refaire.
|
||||||
|
4. `python correction.py DS09VA --refaire`
|
||||||
|
5. `python annotating_with_checks.py DS09VA --refaire`
|
||||||
|
6. `python from_tablette.py Interro24 --refaire`
|
||||||
|
|||||||
+11
-9
@@ -73,6 +73,8 @@ def make_dictionary(root_dir, refaire=False, refaire_list=[]):
|
|||||||
fb = result_obj.get("feedback", [])
|
fb = result_obj.get("feedback", [])
|
||||||
for i in range(len(fb)):
|
for i in range(len(fb)):
|
||||||
el = fb[i]
|
el = fb[i]
|
||||||
|
if height == None or width == None:
|
||||||
|
print("?? height or width is None, for ", student_id, label)
|
||||||
if "box_2d" in el and el["box_2d"]:
|
if "box_2d" in el and el["box_2d"]:
|
||||||
el["box_2d"][0] = (el["box_2d"][0] * height)//1000
|
el["box_2d"][0] = (el["box_2d"][0] * height)//1000
|
||||||
el["box_2d"][2] = (el["box_2d"][2] * height)//1000
|
el["box_2d"][2] = (el["box_2d"][2] * height)//1000
|
||||||
@@ -102,15 +104,15 @@ def make_dictionary(root_dir, refaire=False, refaire_list=[]):
|
|||||||
else:
|
else:
|
||||||
print("Debug : asked to refaire", sid, lbl, "but pdf absent")
|
print("Debug : asked to refaire", sid, lbl, "but pdf absent")
|
||||||
continue
|
continue
|
||||||
result_data[sid][lbl] = {
|
# result_data[sid][lbl] = {
|
||||||
"pdf_path": pdf_path,
|
# "pdf_path": pdf_path,
|
||||||
"result": {
|
# "result": {
|
||||||
"score": 0.0,
|
# "score": 0.0,
|
||||||
"feedback": [],
|
# "feedback": [],
|
||||||
"error": "non traité"
|
# "error": "non traité"
|
||||||
},
|
# },
|
||||||
"coordinates": (0,0)
|
# "coordinates": (0,0)
|
||||||
}
|
# }
|
||||||
else: # Ce student id n'a jamais été corrigé
|
else: # Ce student id n'a jamais été corrigé
|
||||||
result_data[sid] = {}
|
result_data[sid] = {}
|
||||||
for lbl in labels_to_redo:
|
for lbl in labels_to_redo:
|
||||||
|
|||||||
@@ -143,6 +143,13 @@ def main():
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
base_prefix = os.path.commonprefix(safe_labels).strip()
|
base_prefix = os.path.commonprefix(safe_labels).strip()
|
||||||
|
|
||||||
|
if base_prefix and previous_prefix is not None:
|
||||||
|
if natural_key(base_prefix) < natural_key(previous_prefix):
|
||||||
|
base_prefix_maybe = f"{safe_labels[0]}+"
|
||||||
|
if natural_key(base_prefix_maybe) > natural_key(previous_prefix):
|
||||||
|
base_prefix = base_prefix_maybe
|
||||||
|
|
||||||
if not base_prefix:
|
if not base_prefix:
|
||||||
base_prefix = "Group"
|
base_prefix = "Group"
|
||||||
|
|
||||||
@@ -156,7 +163,7 @@ def main():
|
|||||||
if counter == 2 and previous_prefix and previous_prefix in unique_prefix:
|
if counter == 2 and previous_prefix and previous_prefix in unique_prefix:
|
||||||
unique_prefix = f"{previous_prefix}-{counter}"
|
unique_prefix = f"{previous_prefix}-{counter}"
|
||||||
elif counter == 2:
|
elif counter == 2:
|
||||||
previous_prefx = unique_prefix
|
previous_prefix = unique_prefix
|
||||||
|
|
||||||
used_prefixes.add(unique_prefix)
|
used_prefixes.add(unique_prefix)
|
||||||
|
|
||||||
|
|||||||
+116
-43
@@ -29,6 +29,8 @@ parser.add_argument("--batch-from", type=str, metavar="LABEL",
|
|||||||
help="Do live requests before LABEL, and batch requests from LABEL onwards")
|
help="Do live requests before LABEL, and batch requests from LABEL onwards")
|
||||||
parser.add_argument("--deal-with-batched", action="store_true",
|
parser.add_argument("--deal-with-batched", action="store_true",
|
||||||
help="Process a JSONL file containing completed batch results")
|
help="Process a JSONL file containing completed batch results")
|
||||||
|
parser.add_argument("--reset", action="store_true",
|
||||||
|
help="Remove correction.json, revert _old.pdf, delete _new.pdf, then exit")
|
||||||
args, _ = parser.parse_known_args()
|
args, _ = parser.parse_known_args()
|
||||||
|
|
||||||
tasks = [] # List of tuples: (filepath_str, label_str)
|
tasks = [] # List of tuples: (filepath_str, label_str)
|
||||||
@@ -122,6 +124,36 @@ signal.signal(signal.SIGTERM, handle_interrupt)
|
|||||||
client = genai.Client(api_key=api_key)
|
client = genai.Client(api_key=api_key)
|
||||||
output_path = INPUT_DIR / "correction.json"
|
output_path = INPUT_DIR / "correction.json"
|
||||||
progress_path = INPUT_DIR / "correction_progress.json"
|
progress_path = INPUT_DIR / "correction_progress.json"
|
||||||
|
|
||||||
|
if args.reset:
|
||||||
|
print("--- Running Reset ---")
|
||||||
|
if output_path.exists():
|
||||||
|
output_path.unlink()
|
||||||
|
print(f"Deleted: {output_path}")
|
||||||
|
if progress_path.exists():
|
||||||
|
progress_path.unlink()
|
||||||
|
print(f"Deleted: {progress_path}")
|
||||||
|
|
||||||
|
if COPIES_DIR.exists():
|
||||||
|
for copie_dir in COPIES_DIR.iterdir():
|
||||||
|
if not copie_dir.is_dir():
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Revert _old.pdf files
|
||||||
|
for old_pdf in copie_dir.glob("*_old.pdf"):
|
||||||
|
orig_pdf = copie_dir / old_pdf.name.replace("_old.pdf", ".pdf")
|
||||||
|
if orig_pdf.exists():
|
||||||
|
orig_pdf.unlink() # Prevent FileExistsError on Windows
|
||||||
|
old_pdf.rename(orig_pdf)
|
||||||
|
print(f"Moved: {copie_dir.name}/{old_pdf.name} -> {orig_pdf.name}")
|
||||||
|
|
||||||
|
# Delete _new.pdf files
|
||||||
|
for new_pdf in copie_dir.glob("*_new.pdf"):
|
||||||
|
new_pdf.unlink()
|
||||||
|
print(f"Deleted: {copie_dir.name}/{new_pdf.name}")
|
||||||
|
|
||||||
|
sys.exit("Reset almost complete. For each deleted `_new`, you should manually delete the group in `Par label`")
|
||||||
|
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
overwrite = args.overwrite
|
overwrite = args.overwrite
|
||||||
limit = args.limit
|
limit = args.limit
|
||||||
@@ -269,9 +301,11 @@ def handle_label_errors(pid, label, res, pdf_path):
|
|||||||
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}_new.pdf"
|
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}_new.pdf"
|
||||||
|
|
||||||
if base_new_pdf_path.exists() or new_pdf_path.exists():
|
if base_new_pdf_path.exists() or new_pdf_path.exists():
|
||||||
tprint(f"\t\tCopie{pid} tried to move wrong {label} to {new_label}, but it already exists. Delaying.")
|
tprint(f"""\t\tCopie{pid} tried to move wrong {label} to {new_label},
|
||||||
|
but it already exists. Delaying.""")
|
||||||
# res["error"] = f"wrg-lbl:{new_label}?exists"
|
# res["error"] = f"wrg-lbl:{new_label}?exists"
|
||||||
res["error"] = f"wrg-lbl:{new_label}?delayed"
|
res["error"] = f"wrg-lbl:{new_label}?"
|
||||||
|
res.setdefault("delayed", []).append(["wrong-label", new_label])
|
||||||
else:
|
else:
|
||||||
res["error"] = f"wrg-lbl-moved-to:{new_label}"
|
res["error"] = f"wrg-lbl-moved-to:{new_label}"
|
||||||
tprint(f"\t\tCopie{pid} : moving wrong {label} to {new_label}.")
|
tprint(f"\t\tCopie{pid} : moving wrong {label} to {new_label}.")
|
||||||
@@ -324,8 +358,8 @@ def handle_label_errors(pid, label, res, pdf_path):
|
|||||||
keep_error = True
|
keep_error = True
|
||||||
else:
|
else:
|
||||||
keep_error = True
|
keep_error = True
|
||||||
# error += f"(xx){add_label}"
|
error += f"(->){add_label}?"
|
||||||
error += f"(delayed){add_label}"
|
res.setdefault("delayed", []).append(["add-label", add_label])
|
||||||
tprint(f"\t\tAlready present (not copied) Copie{pid} : {label} -> {add_label}. Delaying.")
|
tprint(f"\t\tAlready present (not copied) Copie{pid} : {label} -> {add_label}. Delaying.")
|
||||||
if not keep_error:
|
if not keep_error:
|
||||||
res["error"] = ""
|
res["error"] = ""
|
||||||
@@ -496,8 +530,9 @@ def resolve_delayed_moves():
|
|||||||
for label, batches in results.items():
|
for label, batches in results.items():
|
||||||
for batch in batches:
|
for batch in batches:
|
||||||
for p in batch:
|
for p in batch:
|
||||||
err = p.get("result", {}).get("error", "")
|
res = p.get("result", {})
|
||||||
if not err or ("?delayed" not in err and "(delayed)" not in err):
|
delayed_list = res.get("delayed", [])
|
||||||
|
if not delayed_list:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
pid = p["id"]
|
pid = p["id"]
|
||||||
@@ -509,55 +544,58 @@ def resolve_delayed_moves():
|
|||||||
elif pdf_path.with_name(f"{label}_old.pdf").exists():
|
elif pdf_path.with_name(f"{label}_old.pdf").exists():
|
||||||
pdf_path = pdf_path.with_name(f"{label}_old.pdf")
|
pdf_path = pdf_path.with_name(f"{label}_old.pdf")
|
||||||
|
|
||||||
# 1. Résolution de wrong-label
|
remaining_delayed = []
|
||||||
if err.startswith("wrg-lbl:") and "?delayed" in err:
|
resolved_any = False
|
||||||
new_label = err.split(":")[1].split("?")[0]
|
|
||||||
base_new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}.pdf"
|
|
||||||
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}_new.pdf"
|
|
||||||
|
|
||||||
# Si la place s'est libérée (l'ancien a été bougé vers _old)
|
for delay_type, target_label in delayed_list:
|
||||||
if not base_new_pdf_path.exists() and not new_pdf_path.exists():
|
# 1. Résolution de wrong-label
|
||||||
tprint(f"Resolving delayed move: Copie{pid} {label} -> {new_label}")
|
if delay_type == "wrong-label":
|
||||||
p["result"]["error"] = f"wrg-lbl-moved-to:{new_label}"
|
base_new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}.pdf"
|
||||||
p["result"]["suffixe"] = "_old" # Très important pour l'ignorer ensuite
|
new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}_new.pdf"
|
||||||
|
|
||||||
shutil.copy(str(pdf_path), str(new_pdf_path))
|
if not base_new_pdf_path.exists() and not new_pdf_path.exists():
|
||||||
old_pdf_path = pdf_path.with_name(f"{label}_old.pdf")
|
tprint(f"Resolving delayed move: Copie{pid} {label} -> {target_label}")
|
||||||
if pdf_path != old_pdf_path:
|
res["error"] = f"wrg-lbl-moved-to:{target_label}"
|
||||||
shutil.move(str(pdf_path), str(old_pdf_path))
|
res["suffix"] = "_old" # Fixed typo: was suffixe
|
||||||
|
resolved_any = True
|
||||||
|
|
||||||
idx = get_next_group_idx(new_label)
|
shutil.copy(str(pdf_path), str(new_pdf_path))
|
||||||
height = grouping.get_pdf_height(str(new_pdf_path))
|
old_pdf_path = pdf_path.with_name(f"{label}_old.pdf")
|
||||||
grouping.create_jpg(new_label, idx, [(pid, str(new_pdf_path), height)], GROUPS_DIR)
|
if pdf_path != old_pdf_path:
|
||||||
new_tasks.append((str(GROUPS_DIR / new_label / f"Group_{idx+1}.jpg"), new_label, False))
|
shutil.move(str(pdf_path), str(old_pdf_path))
|
||||||
|
|
||||||
# 2. Résolution de additional-answer
|
idx = get_next_group_idx(target_label)
|
||||||
elif err.startswith("al:") and "(delayed)" in err:
|
height = grouping.get_pdf_height(str(new_pdf_path))
|
||||||
import re
|
grouping.create_jpg(target_label, idx, [(pid, str(new_pdf_path), height)], GROUPS_DIR)
|
||||||
delayed_matches = re.findall(r'\(delayed\)([^?()]+)', err)
|
new_tasks.append((str(GROUPS_DIR / target_label / f"Group_{idx+1}.jpg"), target_label, False))
|
||||||
new_err = err
|
else:
|
||||||
resolved_any = False
|
remaining_delayed.append([delay_type, target_label])
|
||||||
|
|
||||||
for add_label in delayed_matches:
|
# 2. Résolution de additional-answer
|
||||||
base_add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{add_label}.pdf"
|
elif delay_type == "add-label":
|
||||||
add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{add_label}_new.pdf"
|
base_add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}.pdf"
|
||||||
|
add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}_new.pdf"
|
||||||
|
|
||||||
if not base_add_pdf_path.exists() and not add_pdf_path.exists():
|
if not base_add_pdf_path.exists() and not add_pdf_path.exists():
|
||||||
tprint(f"Resolving delayed additional-answer: Copie{pid} {label} -> {add_label}")
|
tprint(f"Resolving delayed additional-answer: Copie{pid} {label} -> {target_label}")
|
||||||
new_err = new_err.replace(f"(delayed){add_label}", f"(->){add_label}")
|
res["error"] = res["error"].replace(f"(xx){target_label}", f"(->){target_label}")
|
||||||
resolved_any = True
|
resolved_any = True
|
||||||
|
|
||||||
shutil.copy(str(pdf_path), str(add_pdf_path))
|
shutil.copy(str(pdf_path), str(add_pdf_path))
|
||||||
idx = get_next_group_idx(add_label)
|
idx = get_next_group_idx(target_label)
|
||||||
height = grouping.get_pdf_height(str(add_pdf_path))
|
height = grouping.get_pdf_height(str(add_pdf_path))
|
||||||
grouping.create_jpg(add_label, idx, [(pid, str(add_pdf_path), height)], GROUPS_DIR)
|
grouping.create_jpg(target_label, idx, [(pid, str(add_pdf_path), height)], GROUPS_DIR)
|
||||||
new_tasks.append((str(GROUPS_DIR / add_label / f"Group_{idx+1}.jpg"), add_label, False))
|
new_tasks.append((str(GROUPS_DIR / target_label / f"Group_{idx+1}.jpg"), target_label, False))
|
||||||
|
else:
|
||||||
|
remaining_delayed.append([delay_type, target_label])
|
||||||
|
|
||||||
if resolved_any:
|
if resolved_any:
|
||||||
p["result"]["error"] = new_err
|
if remaining_delayed:
|
||||||
|
res["delayed"] = remaining_delayed
|
||||||
|
else:
|
||||||
|
del res["delayed"]
|
||||||
|
|
||||||
if new_tasks:
|
if new_tasks:
|
||||||
# Sauvegarder les modifications d'erreurs (les tags delayed enlevés)
|
|
||||||
with open(output_path, "w", encoding="utf-8") as f:
|
with open(output_path, "w", encoding="utf-8") as f:
|
||||||
json.dump(results, f, indent=2)
|
json.dump(results, f, indent=2)
|
||||||
|
|
||||||
@@ -585,7 +623,15 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
# If list is empty, redo all labels available for this Copie
|
# If list is empty, redo all labels available for this Copie
|
||||||
if not labels:
|
if not labels:
|
||||||
labels = [p.stem for p in copie_dir.glob("*.pdf")]
|
labels_set = set()
|
||||||
|
for p in copie_dir.glob("*.pdf"):
|
||||||
|
if p.name.endswith("_old.pdf"):
|
||||||
|
continue # Strictly ignore old files
|
||||||
|
elif p.name.endswith("_new.pdf"):
|
||||||
|
labels_set.add(p.stem[:-4]) # Strip '_new' to get base label
|
||||||
|
else:
|
||||||
|
labels_set.add(p.stem)
|
||||||
|
labels = list(labels_set)
|
||||||
|
|
||||||
for label in labels:
|
for label in labels:
|
||||||
# 1. Extract and backup old corrections
|
# 1. Extract and backup old corrections
|
||||||
@@ -610,9 +656,12 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
# 2. Make new group and add to tasks
|
# 2. Make new group and add to tasks
|
||||||
pdf_path = copie_dir / f"{label}.pdf"
|
pdf_path = copie_dir / f"{label}.pdf"
|
||||||
|
is_new = False
|
||||||
|
|
||||||
if not pdf_path.exists():
|
if not pdf_path.exists():
|
||||||
if (copie_dir / f"{label}_new.pdf").exists():
|
if (copie_dir / f"{label}_new.pdf").exists():
|
||||||
pdf_path = copie_dir / f"{label}_new.pdf"
|
pdf_path = copie_dir / f"{label}_new.pdf"
|
||||||
|
is_new = True
|
||||||
# elif (copie_dir / f"{label}_old.pdf").exists():
|
# elif (copie_dir / f"{label}_old.pdf").exists():
|
||||||
# pdf_path = copie_dir / f"{label}_old.pdf"
|
# pdf_path = copie_dir / f"{label}_old.pdf"
|
||||||
|
|
||||||
@@ -621,7 +670,7 @@ if __name__ == "__main__":
|
|||||||
height = grouping.get_pdf_height(str(pdf_path))
|
height = grouping.get_pdf_height(str(pdf_path))
|
||||||
grouping.create_jpg(label, idx, [(pid, str(pdf_path), height)], GROUPS_DIR)
|
grouping.create_jpg(label, idx, [(pid, str(pdf_path), height)], GROUPS_DIR)
|
||||||
new_group_path = str(GROUPS_DIR / label / f"Group_{idx+1}.jpg")
|
new_group_path = str(GROUPS_DIR / label / f"Group_{idx+1}.jpg")
|
||||||
tasks_to_process.append((new_group_path, label))
|
tasks_to_process.append((new_group_path, label, not is_new))
|
||||||
|
|
||||||
if dirty_results:
|
if dirty_results:
|
||||||
with open(output_path, "w", encoding="utf-8") as f:
|
with open(output_path, "w", encoding="utf-8") as f:
|
||||||
@@ -774,6 +823,30 @@ if __name__ == "__main__":
|
|||||||
else:
|
else:
|
||||||
made_progress = False
|
made_progress = False
|
||||||
|
|
||||||
|
# Check for remaining unresolved delayed tasks
|
||||||
|
unresolved_delayed = []
|
||||||
|
with io_lock:
|
||||||
|
for label, batches in results.items():
|
||||||
|
for batch in batches:
|
||||||
|
for p in batch:
|
||||||
|
res = p.get("result", {})
|
||||||
|
delayed = res.get("delayed", [])
|
||||||
|
pid = p["id"]
|
||||||
|
|
||||||
|
for delay_type, target_label in delayed:
|
||||||
|
if delay_type == "wrong-label":
|
||||||
|
unresolved_delayed.append(f"Copie{pid} {label} x> {target_label}|")
|
||||||
|
elif delay_type == "add-label":
|
||||||
|
unresolved_delayed.append(f"Copie{pid} {label} -> {target_label}|")
|
||||||
|
|
||||||
|
if unresolved_delayed:
|
||||||
|
manual_path = INPUT_DIR / "manual_resolutions.txt"
|
||||||
|
with open(manual_path, "w", encoding="utf-8") as f:
|
||||||
|
f.write("### Use -> x>, -x, ss, sx, xx, xs\n")
|
||||||
|
f.write("\n".join(unresolved_delayed) + "\n")
|
||||||
|
print(f"\n[!] Unresolved delayed tasks found! Wrote to {manual_path}.")
|
||||||
|
print(" Please edit it manually, then run `python resolve_manual.py <InputDir>`")
|
||||||
|
|
||||||
end_time = time.time()
|
end_time = time.time()
|
||||||
print("Time elapsed : ", end_time - start_time)
|
print("Time elapsed : ", end_time - start_time)
|
||||||
print("Requests to pro / flash : ", pro_count, flash_count)
|
print("Requests to pro / flash : ", pro_count, flash_count)
|
||||||
|
|||||||
+17
-7
@@ -135,17 +135,23 @@ def worker_thread(base_dir, files_to_process, all_labels):
|
|||||||
print(f"Buffering {img_path.name}...")
|
print(f"Buffering {img_path.name}...")
|
||||||
(pil_image, last_label_index) = \
|
(pil_image, last_label_index) = \
|
||||||
prepare_image(str(img_path), bb_list, all_labels, nb_pages, last_label_index)
|
prepare_image(str(img_path), bb_list, all_labels, nb_pages, last_label_index)
|
||||||
|
error_msg = None
|
||||||
metadata = {
|
|
||||||
"copie": copie,
|
|
||||||
"part": copie_part,
|
|
||||||
"schema": json_schema,
|
|
||||||
"name": json_name
|
|
||||||
}
|
|
||||||
|
|
||||||
image_queue.put((pil_image, json_path, metadata))
|
image_queue.put((pil_image, json_path, metadata))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error processing {img_path.name}: {e}")
|
print(f"Error processing {img_path.name}: {e}")
|
||||||
|
pil_image = Image.open(str(img_path))
|
||||||
|
error_msg = str(e)
|
||||||
|
|
||||||
|
metadata = {
|
||||||
|
"copie": copie,
|
||||||
|
"part": copie_part,
|
||||||
|
"schema": json_schema,
|
||||||
|
"name": json_name,
|
||||||
|
"error": error_msg
|
||||||
|
}
|
||||||
|
|
||||||
|
image_queue.put((pil_image, json_path, metadata))
|
||||||
|
|
||||||
# Sentinel to indicate finished
|
# Sentinel to indicate finished
|
||||||
image_queue.put((None, None, None))
|
image_queue.put((None, None, None))
|
||||||
@@ -271,6 +277,10 @@ class ImageViewer:
|
|||||||
self.is_viewing = True
|
self.is_viewing = True
|
||||||
self.root.lift()
|
self.root.lift()
|
||||||
|
|
||||||
|
if metadata.get("error"):
|
||||||
|
msg = f"Error generating boxes for {json_path.name}:\n\n{metadata['error']}\n\nPlease press 'e' to fix the JSON file, then press Enter to retry."
|
||||||
|
messagebox.showerror("Processing Error", msg)
|
||||||
|
|
||||||
def on_enter(self, event):
|
def on_enter(self, event):
|
||||||
if self.is_viewing:
|
if self.is_viewing:
|
||||||
print(f"Committing data for {self.current_json_path.name}...")
|
print(f"Committing data for {self.current_json_path.name}...")
|
||||||
|
|||||||
@@ -0,0 +1,196 @@
|
|||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
from pypdf import PdfWriter
|
||||||
|
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
sys.exit("Usage: python resolve_manual.py <InputDir>")
|
||||||
|
|
||||||
|
input_dir = Path(sys.argv[1])
|
||||||
|
manual_file = input_dir / "manual_resolutions.txt"
|
||||||
|
correction_file = input_dir / "correction.json"
|
||||||
|
refaire_file = input_dir / "refaire.json"
|
||||||
|
copies_dir = input_dir / "Copies"
|
||||||
|
|
||||||
|
if not manual_file.exists():
|
||||||
|
sys.exit(f"No {manual_file.name} found. Nothing to resolve.")
|
||||||
|
|
||||||
|
with open(correction_file, "r", encoding="utf-8") as f:
|
||||||
|
results = json.load(f)
|
||||||
|
|
||||||
|
def set_suffix_and_clean_error(pid, label, suffix, new_lbl_target=None):
|
||||||
|
"""Updates correction.json to set suffixes and clear resolved delayed tags."""
|
||||||
|
if label in results:
|
||||||
|
for batch in results[label]:
|
||||||
|
for item in batch:
|
||||||
|
if item["id"] == pid:
|
||||||
|
if suffix:
|
||||||
|
item["result"]["suffix"] = suffix
|
||||||
|
err = item["result"].get("error", "")
|
||||||
|
if new_lbl_target:
|
||||||
|
if f"wrg-lbl:{new_lbl_target}?delayed" in err:
|
||||||
|
item["result"]["error"] = f"wrg-lbl-moved-to:{new_lbl_target}"
|
||||||
|
if f"(delayed){new_lbl_target}" in err:
|
||||||
|
item["result"]["error"] = err.replace(f"(delayed){new_lbl_target}", f"(->){new_lbl_target}")
|
||||||
|
|
||||||
|
def get_actual_pdf(pid, label):
|
||||||
|
"""Finds the effective PDF considering possible suffixes."""
|
||||||
|
base = copies_dir / f"Copie{pid}" / f"{label}.pdf"
|
||||||
|
if base.exists(): return base
|
||||||
|
if base.with_name(f"{label}_new.pdf").exists(): return base.with_name(f"{label}_new.pdf")
|
||||||
|
if base.with_name(f"{label}_old.pdf").exists(): return base.with_name(f"{label}_old.pdf")
|
||||||
|
return base
|
||||||
|
|
||||||
|
def safe_strip_suffix(stem):
|
||||||
|
if stem.endswith("_new"): return stem[:-4]
|
||||||
|
if stem.endswith("_old"): return stem[:-4]
|
||||||
|
return stem
|
||||||
|
|
||||||
|
instructions = []
|
||||||
|
with open(manual_file, "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("###"): continue
|
||||||
|
|
||||||
|
# Regex to split on the operator properly handles spaces in labels
|
||||||
|
match = re.search(r'\s+(-x|->|x>)\s+', line)
|
||||||
|
if not match:
|
||||||
|
print(f"Skipping malformed line: {line}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
op = match.group(1)
|
||||||
|
left = line[:match.start()].strip()
|
||||||
|
right = line[match.end():].strip()
|
||||||
|
|
||||||
|
m_left = re.match(r'Copie(\d+)\s+(.+)', left)
|
||||||
|
if not m_left:
|
||||||
|
continue
|
||||||
|
|
||||||
|
pid = m_left.group(1)
|
||||||
|
old_label = m_left.group(2).strip()
|
||||||
|
new_part = right
|
||||||
|
|
||||||
|
pipe_first = new_part.startswith("|")
|
||||||
|
# pipe_last = new_part.endswith("|")
|
||||||
|
new_label = new_part.strip("|").strip()
|
||||||
|
|
||||||
|
instructions.append((pid, old_label, op, new_label, pipe_first))
|
||||||
|
|
||||||
|
# State trackers
|
||||||
|
initial_paths = {} # Tracks the source files at script start (enables safe Swapping)
|
||||||
|
current_paths = {} # Tracks the "latest active file" (which may be a temp file for chained merges)
|
||||||
|
files_to_old = set()
|
||||||
|
temp_files = []
|
||||||
|
|
||||||
|
# Pre-load existing paths
|
||||||
|
for pid, old_label, _, new_label, _ in instructions:
|
||||||
|
for lbl in (old_label, new_label):
|
||||||
|
if (pid, lbl) not in initial_paths:
|
||||||
|
p = get_actual_pdf(pid, lbl)
|
||||||
|
initial_paths[(pid, lbl)] = p
|
||||||
|
current_paths[(pid, lbl)] = p
|
||||||
|
|
||||||
|
# Evaluate instructions
|
||||||
|
for pid, old_label, op, new_label, pipe_first in instructions:
|
||||||
|
should_merge = op[1] == ">"
|
||||||
|
src_pdf = initial_paths[(pid, old_label)]
|
||||||
|
dest_pdf = current_paths[(pid, new_label)]
|
||||||
|
|
||||||
|
temp_out = copies_dir / f"Copie{pid}" / f"temp_{len(temp_files)}.pdf"
|
||||||
|
|
||||||
|
if op[0] == "x":
|
||||||
|
files_to_old.add(initial_paths[(pid, old_label)])
|
||||||
|
if op[1] == "x":
|
||||||
|
files_to_old.add(initial_paths[(pid, new_label)])
|
||||||
|
|
||||||
|
if should_merge:
|
||||||
|
if not dest_pdf.exists() or not src_pdf.exists():
|
||||||
|
print("Debug : should_merge but, {} or {} doesn't exist"
|
||||||
|
.format(src_pdf, dest_pdf))
|
||||||
|
input("You should Ctrl-C and fix.")
|
||||||
|
# MERGE
|
||||||
|
merger = PdfWriter()
|
||||||
|
if pipe_first:
|
||||||
|
merger.append(src_pdf)
|
||||||
|
merger.append(dest_pdf)
|
||||||
|
else:
|
||||||
|
merger.append(dest_pdf)
|
||||||
|
merger.append(src_pdf)
|
||||||
|
merger.write(temp_out)
|
||||||
|
merger.close()
|
||||||
|
current_paths[(pid, new_label)] = temp_out
|
||||||
|
temp_files.append(temp_out)
|
||||||
|
# Original destination is now embedded in the merge, back it up
|
||||||
|
files_to_old.add(initial_paths[(pid, new_label)])
|
||||||
|
else:
|
||||||
|
if op[1] != "s" and op[0] != "s": # xx or -x
|
||||||
|
shutil.copy(src_pdf, temp_out)
|
||||||
|
current_paths[(pid, new_label)] = temp_out
|
||||||
|
temp_files.append(temp_out)
|
||||||
|
|
||||||
|
|
||||||
|
# Commit Filesystem Changes
|
||||||
|
|
||||||
|
# 1. Archive deprecated files to _old.pdf
|
||||||
|
for pdf in files_to_old:
|
||||||
|
if pdf.exists():
|
||||||
|
pid_str = pdf.parent.name.replace("Copie", "")
|
||||||
|
label = safe_strip_suffix(pdf.stem)
|
||||||
|
old_name = pdf.with_name(f"{label}_old.pdf")
|
||||||
|
|
||||||
|
if pdf != old_name:
|
||||||
|
if old_name.exists(): old_name.unlink()
|
||||||
|
shutil.move(str(pdf), str(old_name))
|
||||||
|
|
||||||
|
set_suffix_and_clean_error(pid_str, label, "_old")
|
||||||
|
|
||||||
|
# 2. Clear all delayed errors for the instructions
|
||||||
|
for pid, old_label, op, new_label, pf in instructions:
|
||||||
|
set_suffix_and_clean_error(pid, old_label, None, new_label)
|
||||||
|
|
||||||
|
refaire_tasks = []
|
||||||
|
|
||||||
|
# 3. Rename active temp files to _new.pdf and queue for --refaire
|
||||||
|
for (pid, label), current_path in current_paths.items():
|
||||||
|
if "temp_" in current_path.name:
|
||||||
|
final_name = copies_dir / f"Copie{pid}" / f"{label}_new.pdf"
|
||||||
|
if final_name.exists(): final_name.unlink()
|
||||||
|
shutil.move(str(current_path), str(final_name))
|
||||||
|
|
||||||
|
set_suffix_and_clean_error(pid, label, "_new")
|
||||||
|
|
||||||
|
# Push to refaire_tasks uniquely
|
||||||
|
added = False
|
||||||
|
for t in refaire_tasks:
|
||||||
|
if t[0] == f"Copie{pid}":
|
||||||
|
if label not in t[1]:
|
||||||
|
t[1].append(label)
|
||||||
|
added = True
|
||||||
|
break
|
||||||
|
if not added:
|
||||||
|
refaire_tasks.append([f"Copie{pid}", [label]])
|
||||||
|
|
||||||
|
# 4. Clean up any unused temp files (overwritten by chained replacements)
|
||||||
|
used_temps = set(current_paths.values())
|
||||||
|
for temp in temp_files:
|
||||||
|
if temp not in used_temps and temp.exists():
|
||||||
|
temp.unlink()
|
||||||
|
|
||||||
|
# Finalize JSONs
|
||||||
|
with open(correction_file, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(results, f, indent=2)
|
||||||
|
|
||||||
|
if refaire_tasks:
|
||||||
|
with open(refaire_file, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(refaire_tasks, f, indent=2)
|
||||||
|
|
||||||
|
manual_file.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
print("Manual resolutions successfully applied.")
|
||||||
|
if refaire_tasks:
|
||||||
|
print(f"File {refaire_file.name} generated. Run `python correction.py \"{input_dir}\" --refaire` to process updates.")
|
||||||
|
else:
|
||||||
|
print("No new corrections required.")
|
||||||
Reference in New Issue
Block a user