diff --git a/Readme.org b/Readme.org index 56c2c6d..48b84fd 100644 --- a/Readme.org +++ b/Readme.org @@ -1,7 +1,7 @@ #+title: Script #+author: Sébastien Miquel #+date: 14-03-2026 -# Time-stamp: <06-06-26 16:23> +# Time-stamp: <29-06-26 13:05> #+OPTIONS: * Méta @@ -93,7 +93,7 @@ export GEMINI_API_KEY=… + `correction.tex`. - `python gemini_for_enonce.py Interro` Se charge de créer des dossiers `Text` et `Sol` avec - + Le fichier `Text` contient + + Le fichier `Text` contient ** Prétraitement des copies @@ -195,6 +195,19 @@ Set proxy with ~export HTTPS_PROXY="http://10.0.0.1:3128"~ - Essaye de corriger des erreurs d'encodage/d'accents dans =correction.json=. - aussi échappe les `_` en dehors du mode math, pour LaTeX. + 4. Résolution manuel de conflits + + Edit `manual_resolutions.txt`. Use : + + `->` or `x>` : Here set a pipe `|` before or after the new_label name + + `-x` : replace the goal + + `ss` : do nothing + + `sx` : stay, and remove goal. + + `xx` : move to goal. + + `xs` : remove old, keep goal. + + Then call `python resolve_manual.py Interro` + 5. Call `python correction.py Interro --refaire`. + ** Génération des copies annotées @@ -282,8 +295,8 @@ groupée into refaire !! + =python plotting.py InterroTest/Copie01.pdf= + =python splitting_int.py InterroTest/Copie20.pdf= 2. Créer =refaire.json=, avec un contenu comme - : [["Copie01", []], - : ["Copie01", ["Ex 1 : 1)"]]] + : [["Copie02", []], + : ["Copie01", ["Ex 1 : 1)"]]] 3. Appeler =correction= avec --refaire. Il doit créer des groupes individuels, faire des requêtes, et remplacer les corrections précédentes (à sauver ailleurs). @@ -291,7 +304,18 @@ groupée into refaire !! Ou non, si tu veux le faire à la main. 4. ?? Si je fais refaire, avant d'avoir créer les annotating with checks, que se passe-t-il ??? - 5. Appeler =annotating_with_checks.py --refaire --overwrite= avec --refaire. + 5. Appeler =annotating_with_checks.py --refaire --overwrite= 6. =python to_tablette.py --refaire Interro24= 6. =python from_tablette.py --refaire Interro24= 7. =python reading_grouped_annotations.py --refaire Interro24= + +** Exemple de replotting, refaire d'une copie + + 1. replot it. + 2. `python splitting_int.py DS09VA/Copies/Copie25.pdf` + this will get rid of old/new. + !! Attention, et si ça dégage un new : bad bad bad. + 3. Make `refaire.json`, avec la copie, et les labels à refaire. + 4. `python correction.py DS09VA --refaire` + 5. `python annotating_with_checks.py DS09VA --refaire` + 6. `python from_tablette.py Interro24 --refaire` diff --git a/annotating.py b/annotating.py index 7e63162..a583e38 100644 --- a/annotating.py +++ b/annotating.py @@ -73,6 +73,8 @@ def make_dictionary(root_dir, refaire=False, refaire_list=[]): fb = result_obj.get("feedback", []) for i in range(len(fb)): el = fb[i] + if height == None or width == None: + print("?? height or width is None, for ", student_id, label) if "box_2d" in el and el["box_2d"]: el["box_2d"][0] = (el["box_2d"][0] * height)//1000 el["box_2d"][2] = (el["box_2d"][2] * height)//1000 @@ -102,15 +104,15 @@ def make_dictionary(root_dir, refaire=False, refaire_list=[]): else: print("Debug : asked to refaire", sid, lbl, "but pdf absent") continue - result_data[sid][lbl] = { - "pdf_path": pdf_path, - "result": { - "score": 0.0, - "feedback": [], - "error": "non traité" - }, - "coordinates": (0,0) - } + # result_data[sid][lbl] = { + # "pdf_path": pdf_path, + # "result": { + # "score": 0.0, + # "feedback": [], + # "error": "non traité" + # }, + # "coordinates": (0,0) + # } else: # Ce student id n'a jamais été corrigé result_data[sid] = {} for lbl in labels_to_redo: diff --git a/annotating_by_label.py b/annotating_by_label.py index 502d3ff..c26cc7d 100644 --- a/annotating_by_label.py +++ b/annotating_by_label.py @@ -143,6 +143,13 @@ def main(): continue base_prefix = os.path.commonprefix(safe_labels).strip() + + if base_prefix and previous_prefix is not None: + if natural_key(base_prefix) < natural_key(previous_prefix): + base_prefix_maybe = f"{safe_labels[0]}+" + if natural_key(base_prefix_maybe) > natural_key(previous_prefix): + base_prefix = base_prefix_maybe + if not base_prefix: base_prefix = "Group" @@ -156,7 +163,7 @@ def main(): if counter == 2 and previous_prefix and previous_prefix in unique_prefix: unique_prefix = f"{previous_prefix}-{counter}" elif counter == 2: - previous_prefx = unique_prefix + previous_prefix = unique_prefix used_prefixes.add(unique_prefix) diff --git a/correction.py b/correction.py index a602b9b..c630881 100644 --- a/correction.py +++ b/correction.py @@ -29,6 +29,8 @@ parser.add_argument("--batch-from", type=str, metavar="LABEL", help="Do live requests before LABEL, and batch requests from LABEL onwards") parser.add_argument("--deal-with-batched", action="store_true", help="Process a JSONL file containing completed batch results") +parser.add_argument("--reset", action="store_true", + help="Remove correction.json, revert _old.pdf, delete _new.pdf, then exit") args, _ = parser.parse_known_args() tasks = [] # List of tuples: (filepath_str, label_str) @@ -122,6 +124,36 @@ signal.signal(signal.SIGTERM, handle_interrupt) client = genai.Client(api_key=api_key) output_path = INPUT_DIR / "correction.json" progress_path = INPUT_DIR / "correction_progress.json" + +if args.reset: + print("--- Running Reset ---") + if output_path.exists(): + output_path.unlink() + print(f"Deleted: {output_path}") + if progress_path.exists(): + progress_path.unlink() + print(f"Deleted: {progress_path}") + + if COPIES_DIR.exists(): + for copie_dir in COPIES_DIR.iterdir(): + if not copie_dir.is_dir(): + continue + + # Revert _old.pdf files + for old_pdf in copie_dir.glob("*_old.pdf"): + orig_pdf = copie_dir / old_pdf.name.replace("_old.pdf", ".pdf") + if orig_pdf.exists(): + orig_pdf.unlink() # Prevent FileExistsError on Windows + old_pdf.rename(orig_pdf) + print(f"Moved: {copie_dir.name}/{old_pdf.name} -> {orig_pdf.name}") + + # Delete _new.pdf files + for new_pdf in copie_dir.glob("*_new.pdf"): + new_pdf.unlink() + print(f"Deleted: {copie_dir.name}/{new_pdf.name}") + + sys.exit("Reset almost complete. For each deleted `_new`, you should manually delete the group in `Par label`") + start_time = time.time() overwrite = args.overwrite limit = args.limit @@ -269,9 +301,11 @@ def handle_label_errors(pid, label, res, pdf_path): new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}_new.pdf" if base_new_pdf_path.exists() or new_pdf_path.exists(): - tprint(f"\t\tCopie{pid} tried to move wrong {label} to {new_label}, but it already exists. Delaying.") + tprint(f"""\t\tCopie{pid} tried to move wrong {label} to {new_label}, + but it already exists. Delaying.""") # res["error"] = f"wrg-lbl:{new_label}?exists" - res["error"] = f"wrg-lbl:{new_label}?delayed" + res["error"] = f"wrg-lbl:{new_label}?" + res.setdefault("delayed", []).append(["wrong-label", new_label]) else: res["error"] = f"wrg-lbl-moved-to:{new_label}" tprint(f"\t\tCopie{pid} : moving wrong {label} to {new_label}.") @@ -324,8 +358,8 @@ def handle_label_errors(pid, label, res, pdf_path): keep_error = True else: keep_error = True - # error += f"(xx){add_label}" - error += f"(delayed){add_label}" + error += f"(->){add_label}?" + res.setdefault("delayed", []).append(["add-label", add_label]) tprint(f"\t\tAlready present (not copied) Copie{pid} : {label} -> {add_label}. Delaying.") if not keep_error: res["error"] = "" @@ -496,8 +530,9 @@ def resolve_delayed_moves(): for label, batches in results.items(): for batch in batches: for p in batch: - err = p.get("result", {}).get("error", "") - if not err or ("?delayed" not in err and "(delayed)" not in err): + res = p.get("result", {}) + delayed_list = res.get("delayed", []) + if not delayed_list: continue pid = p["id"] @@ -509,55 +544,58 @@ def resolve_delayed_moves(): elif pdf_path.with_name(f"{label}_old.pdf").exists(): pdf_path = pdf_path.with_name(f"{label}_old.pdf") - # 1. Résolution de wrong-label - if err.startswith("wrg-lbl:") and "?delayed" in err: - new_label = err.split(":")[1].split("?")[0] - base_new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}.pdf" - new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{new_label}_new.pdf" + remaining_delayed = [] + resolved_any = False - # Si la place s'est libérée (l'ancien a été bougé vers _old) - if not base_new_pdf_path.exists() and not new_pdf_path.exists(): - tprint(f"Resolving delayed move: Copie{pid} {label} -> {new_label}") - p["result"]["error"] = f"wrg-lbl-moved-to:{new_label}" - p["result"]["suffixe"] = "_old" # Très important pour l'ignorer ensuite + for delay_type, target_label in delayed_list: + # 1. Résolution de wrong-label + if delay_type == "wrong-label": + base_new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}.pdf" + new_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}_new.pdf" - shutil.copy(str(pdf_path), str(new_pdf_path)) - old_pdf_path = pdf_path.with_name(f"{label}_old.pdf") - if pdf_path != old_pdf_path: - shutil.move(str(pdf_path), str(old_pdf_path)) + if not base_new_pdf_path.exists() and not new_pdf_path.exists(): + tprint(f"Resolving delayed move: Copie{pid} {label} -> {target_label}") + res["error"] = f"wrg-lbl-moved-to:{target_label}" + res["suffix"] = "_old" # Fixed typo: was suffixe + resolved_any = True - idx = get_next_group_idx(new_label) - height = grouping.get_pdf_height(str(new_pdf_path)) - grouping.create_jpg(new_label, idx, [(pid, str(new_pdf_path), height)], GROUPS_DIR) - new_tasks.append((str(GROUPS_DIR / new_label / f"Group_{idx+1}.jpg"), new_label, False)) + shutil.copy(str(pdf_path), str(new_pdf_path)) + old_pdf_path = pdf_path.with_name(f"{label}_old.pdf") + if pdf_path != old_pdf_path: + shutil.move(str(pdf_path), str(old_pdf_path)) - # 2. Résolution de additional-answer - elif err.startswith("al:") and "(delayed)" in err: - import re - delayed_matches = re.findall(r'\(delayed\)([^?()]+)', err) - new_err = err - resolved_any = False + idx = get_next_group_idx(target_label) + height = grouping.get_pdf_height(str(new_pdf_path)) + grouping.create_jpg(target_label, idx, [(pid, str(new_pdf_path), height)], GROUPS_DIR) + new_tasks.append((str(GROUPS_DIR / target_label / f"Group_{idx+1}.jpg"), target_label, False)) + else: + remaining_delayed.append([delay_type, target_label]) - for add_label in delayed_matches: - base_add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{add_label}.pdf" - add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{add_label}_new.pdf" + # 2. Résolution de additional-answer + elif delay_type == "add-label": + base_add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}.pdf" + add_pdf_path = COPIES_DIR / f"Copie{pid}" / f"{target_label}_new.pdf" if not base_add_pdf_path.exists() and not add_pdf_path.exists(): - tprint(f"Resolving delayed additional-answer: Copie{pid} {label} -> {add_label}") - new_err = new_err.replace(f"(delayed){add_label}", f"(->){add_label}") + tprint(f"Resolving delayed additional-answer: Copie{pid} {label} -> {target_label}") + res["error"] = res["error"].replace(f"(xx){target_label}", f"(->){target_label}") resolved_any = True shutil.copy(str(pdf_path), str(add_pdf_path)) - idx = get_next_group_idx(add_label) + idx = get_next_group_idx(target_label) height = grouping.get_pdf_height(str(add_pdf_path)) - grouping.create_jpg(add_label, idx, [(pid, str(add_pdf_path), height)], GROUPS_DIR) - new_tasks.append((str(GROUPS_DIR / add_label / f"Group_{idx+1}.jpg"), add_label, False)) + grouping.create_jpg(target_label, idx, [(pid, str(add_pdf_path), height)], GROUPS_DIR) + new_tasks.append((str(GROUPS_DIR / target_label / f"Group_{idx+1}.jpg"), target_label, False)) + else: + remaining_delayed.append([delay_type, target_label]) - if resolved_any: - p["result"]["error"] = new_err + if resolved_any: + if remaining_delayed: + res["delayed"] = remaining_delayed + else: + del res["delayed"] if new_tasks: - # Sauvegarder les modifications d'erreurs (les tags delayed enlevés) with open(output_path, "w", encoding="utf-8") as f: json.dump(results, f, indent=2) @@ -585,7 +623,15 @@ if __name__ == "__main__": # If list is empty, redo all labels available for this Copie if not labels: - labels = [p.stem for p in copie_dir.glob("*.pdf")] + labels_set = set() + for p in copie_dir.glob("*.pdf"): + if p.name.endswith("_old.pdf"): + continue # Strictly ignore old files + elif p.name.endswith("_new.pdf"): + labels_set.add(p.stem[:-4]) # Strip '_new' to get base label + else: + labels_set.add(p.stem) + labels = list(labels_set) for label in labels: # 1. Extract and backup old corrections @@ -610,9 +656,12 @@ if __name__ == "__main__": # 2. Make new group and add to tasks pdf_path = copie_dir / f"{label}.pdf" + is_new = False + if not pdf_path.exists(): if (copie_dir / f"{label}_new.pdf").exists(): pdf_path = copie_dir / f"{label}_new.pdf" + is_new = True # elif (copie_dir / f"{label}_old.pdf").exists(): # pdf_path = copie_dir / f"{label}_old.pdf" @@ -621,7 +670,7 @@ if __name__ == "__main__": height = grouping.get_pdf_height(str(pdf_path)) grouping.create_jpg(label, idx, [(pid, str(pdf_path), height)], GROUPS_DIR) new_group_path = str(GROUPS_DIR / label / f"Group_{idx+1}.jpg") - tasks_to_process.append((new_group_path, label)) + tasks_to_process.append((new_group_path, label, not is_new)) if dirty_results: with open(output_path, "w", encoding="utf-8") as f: @@ -774,6 +823,30 @@ if __name__ == "__main__": else: made_progress = False + # Check for remaining unresolved delayed tasks + unresolved_delayed = [] + with io_lock: + for label, batches in results.items(): + for batch in batches: + for p in batch: + res = p.get("result", {}) + delayed = res.get("delayed", []) + pid = p["id"] + + for delay_type, target_label in delayed: + if delay_type == "wrong-label": + unresolved_delayed.append(f"Copie{pid} {label} x> {target_label}|") + elif delay_type == "add-label": + unresolved_delayed.append(f"Copie{pid} {label} -> {target_label}|") + + if unresolved_delayed: + manual_path = INPUT_DIR / "manual_resolutions.txt" + with open(manual_path, "w", encoding="utf-8") as f: + f.write("### Use -> x>, -x, ss, sx, xx, xs\n") + f.write("\n".join(unresolved_delayed) + "\n") + print(f"\n[!] Unresolved delayed tasks found! Wrote to {manual_path}.") + print(" Please edit it manually, then run `python resolve_manual.py `") + end_time = time.time() print("Time elapsed : ", end_time - start_time) print("Requests to pro / flash : ", pro_count, flash_count) diff --git a/plotting.py b/plotting.py index 5685b06..9f5e9b9 100644 --- a/plotting.py +++ b/plotting.py @@ -135,17 +135,23 @@ def worker_thread(base_dir, files_to_process, all_labels): print(f"Buffering {img_path.name}...") (pil_image, last_label_index) = \ prepare_image(str(img_path), bb_list, all_labels, nb_pages, last_label_index) - - metadata = { - "copie": copie, - "part": copie_part, - "schema": json_schema, - "name": json_name - } + error_msg = None image_queue.put((pil_image, json_path, metadata)) except Exception as e: print(f"Error processing {img_path.name}: {e}") + pil_image = Image.open(str(img_path)) + error_msg = str(e) + + metadata = { + "copie": copie, + "part": copie_part, + "schema": json_schema, + "name": json_name, + "error": error_msg + } + + image_queue.put((pil_image, json_path, metadata)) # Sentinel to indicate finished image_queue.put((None, None, None)) @@ -271,6 +277,10 @@ class ImageViewer: self.is_viewing = True self.root.lift() + if metadata.get("error"): + msg = f"Error generating boxes for {json_path.name}:\n\n{metadata['error']}\n\nPlease press 'e' to fix the JSON file, then press Enter to retry." + messagebox.showerror("Processing Error", msg) + def on_enter(self, event): if self.is_viewing: print(f"Committing data for {self.current_json_path.name}...") diff --git a/resolve_manual.py b/resolve_manual.py new file mode 100644 index 0000000..b4f73b5 --- /dev/null +++ b/resolve_manual.py @@ -0,0 +1,196 @@ +import sys +import os +import json +import re +import shutil +from pathlib import Path +from pypdf import PdfWriter + +if len(sys.argv) < 2: + sys.exit("Usage: python resolve_manual.py ") + +input_dir = Path(sys.argv[1]) +manual_file = input_dir / "manual_resolutions.txt" +correction_file = input_dir / "correction.json" +refaire_file = input_dir / "refaire.json" +copies_dir = input_dir / "Copies" + +if not manual_file.exists(): + sys.exit(f"No {manual_file.name} found. Nothing to resolve.") + +with open(correction_file, "r", encoding="utf-8") as f: + results = json.load(f) + +def set_suffix_and_clean_error(pid, label, suffix, new_lbl_target=None): + """Updates correction.json to set suffixes and clear resolved delayed tags.""" + if label in results: + for batch in results[label]: + for item in batch: + if item["id"] == pid: + if suffix: + item["result"]["suffix"] = suffix + err = item["result"].get("error", "") + if new_lbl_target: + if f"wrg-lbl:{new_lbl_target}?delayed" in err: + item["result"]["error"] = f"wrg-lbl-moved-to:{new_lbl_target}" + if f"(delayed){new_lbl_target}" in err: + item["result"]["error"] = err.replace(f"(delayed){new_lbl_target}", f"(->){new_lbl_target}") + +def get_actual_pdf(pid, label): + """Finds the effective PDF considering possible suffixes.""" + base = copies_dir / f"Copie{pid}" / f"{label}.pdf" + if base.exists(): return base + if base.with_name(f"{label}_new.pdf").exists(): return base.with_name(f"{label}_new.pdf") + if base.with_name(f"{label}_old.pdf").exists(): return base.with_name(f"{label}_old.pdf") + return base + +def safe_strip_suffix(stem): + if stem.endswith("_new"): return stem[:-4] + if stem.endswith("_old"): return stem[:-4] + return stem + +instructions = [] +with open(manual_file, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("###"): continue + + # Regex to split on the operator properly handles spaces in labels + match = re.search(r'\s+(-x|->|x>)\s+', line) + if not match: + print(f"Skipping malformed line: {line}") + continue + + op = match.group(1) + left = line[:match.start()].strip() + right = line[match.end():].strip() + + m_left = re.match(r'Copie(\d+)\s+(.+)', left) + if not m_left: + continue + + pid = m_left.group(1) + old_label = m_left.group(2).strip() + new_part = right + + pipe_first = new_part.startswith("|") + # pipe_last = new_part.endswith("|") + new_label = new_part.strip("|").strip() + + instructions.append((pid, old_label, op, new_label, pipe_first)) + +# State trackers +initial_paths = {} # Tracks the source files at script start (enables safe Swapping) +current_paths = {} # Tracks the "latest active file" (which may be a temp file for chained merges) +files_to_old = set() +temp_files = [] + +# Pre-load existing paths +for pid, old_label, _, new_label, _ in instructions: + for lbl in (old_label, new_label): + if (pid, lbl) not in initial_paths: + p = get_actual_pdf(pid, lbl) + initial_paths[(pid, lbl)] = p + current_paths[(pid, lbl)] = p + +# Evaluate instructions +for pid, old_label, op, new_label, pipe_first in instructions: + should_merge = op[1] == ">" + src_pdf = initial_paths[(pid, old_label)] + dest_pdf = current_paths[(pid, new_label)] + + temp_out = copies_dir / f"Copie{pid}" / f"temp_{len(temp_files)}.pdf" + + if op[0] == "x": + files_to_old.add(initial_paths[(pid, old_label)]) + if op[1] == "x": + files_to_old.add(initial_paths[(pid, new_label)]) + + if should_merge: + if not dest_pdf.exists() or not src_pdf.exists(): + print("Debug : should_merge but, {} or {} doesn't exist" + .format(src_pdf, dest_pdf)) + input("You should Ctrl-C and fix.") + # MERGE + merger = PdfWriter() + if pipe_first: + merger.append(src_pdf) + merger.append(dest_pdf) + else: + merger.append(dest_pdf) + merger.append(src_pdf) + merger.write(temp_out) + merger.close() + current_paths[(pid, new_label)] = temp_out + temp_files.append(temp_out) + # Original destination is now embedded in the merge, back it up + files_to_old.add(initial_paths[(pid, new_label)]) + else: + if op[1] != "s" and op[0] != "s": # xx or -x + shutil.copy(src_pdf, temp_out) + current_paths[(pid, new_label)] = temp_out + temp_files.append(temp_out) + + +# Commit Filesystem Changes + +# 1. Archive deprecated files to _old.pdf +for pdf in files_to_old: + if pdf.exists(): + pid_str = pdf.parent.name.replace("Copie", "") + label = safe_strip_suffix(pdf.stem) + old_name = pdf.with_name(f"{label}_old.pdf") + + if pdf != old_name: + if old_name.exists(): old_name.unlink() + shutil.move(str(pdf), str(old_name)) + + set_suffix_and_clean_error(pid_str, label, "_old") + +# 2. Clear all delayed errors for the instructions +for pid, old_label, op, new_label, pf in instructions: + set_suffix_and_clean_error(pid, old_label, None, new_label) + +refaire_tasks = [] + +# 3. Rename active temp files to _new.pdf and queue for --refaire +for (pid, label), current_path in current_paths.items(): + if "temp_" in current_path.name: + final_name = copies_dir / f"Copie{pid}" / f"{label}_new.pdf" + if final_name.exists(): final_name.unlink() + shutil.move(str(current_path), str(final_name)) + + set_suffix_and_clean_error(pid, label, "_new") + + # Push to refaire_tasks uniquely + added = False + for t in refaire_tasks: + if t[0] == f"Copie{pid}": + if label not in t[1]: + t[1].append(label) + added = True + break + if not added: + refaire_tasks.append([f"Copie{pid}", [label]]) + +# 4. Clean up any unused temp files (overwritten by chained replacements) +used_temps = set(current_paths.values()) +for temp in temp_files: + if temp not in used_temps and temp.exists(): + temp.unlink() + +# Finalize JSONs +with open(correction_file, "w", encoding="utf-8") as f: + json.dump(results, f, indent=2) + +if refaire_tasks: + with open(refaire_file, "w", encoding="utf-8") as f: + json.dump(refaire_tasks, f, indent=2) + +manual_file.unlink(missing_ok=True) + +print("Manual resolutions successfully applied.") +if refaire_tasks: + print(f"File {refaire_file.name} generated. Run `python correction.py \"{input_dir}\" --refaire` to process updates.") +else: + print("No new corrections required.")