Miscs changes (Interro22)

correction.py

@@ -400,14 +400,22 @@ Here is a list of all possible lables. You need to answer with one of these :
             types.Part.from_text(text=prompt) ])]
         config = types.GenerateContentConfig(temperature=0.0)
         new_label = call_gemini_with_retries(MODEL_ID_flash, contents, config).strip().strip('"\'')
-
+        if new_label not in all_labels:
+            print(f"\t\tCopie{pid} returned an incorrect label {new_label} from an initial wrong label {label}. Ignoring")
+            res["error"] = "wrg-lbl:cldtfix"
+            return []
+        if new_label == label:
+            res["error"] =""
+            return []
         new_pdf_path = Path(INPUT_DIR) / f"Copie{pid}" / f"{new_label}.pdf"
         if new_pdf_path.exists():
             print(f"\t\tCopie{pid} tried to move wrong {label} to {new_label}, but it already exists.")
-            res["error"] = f"wrong-label:{new_label}?"
+            res["error"] = f"wrg-lbl:{new_label}?exists"
         else:
             print(f"\t\tCopie{pid} : moving wrong {label} to {new_label}.")
             shutil.move(str(pdf_path), str(new_pdf_path))
+            # Since we moved the file, this Copie/label should not be taken
+            # into account in the future, I think
             idx = get_next_group_idx(INPUT_DIR, new_label)
             height = grouping.get_pdf_height(str(new_pdf_path))
             grouping.create_jpg(new_label, idx, [(pid, str(new_pdf_path), height)], INPUT_DIR)
@@ -444,10 +452,14 @@ Here is a list of all possible labels. You need to answer with a list one of the
             add_labels = []
 
         print(f"\tHandling additional-answer for {pid} {label}")
-        some_present = False
+        keep_error = False
         for add_label in add_labels:
             if add_label == label:
                 continue
+            if add_label not in all_labels:
+                print(f"\t\t Inexistent label from additional-answer processing {pid} {label}. Ignoring")
+                keep_error = True
+                continue
             new_pdf_path = Path(INPUT_DIR) / f"Copie{pid}" / f"{add_label}.pdf"
             if not new_pdf_path.exists():
                 shutil.copy(str(pdf_path), str(new_pdf_path))
@@ -459,11 +471,11 @@ Here is a list of all possible labels. You need to answer with a list one of the
                 new_tasks.append((str(Path(INPUT_DIR) / add_label / f"Group_{idx+1}.jpg"),
                                   add_label, False))
             else:
-                some_present = True
+                keep_error = True
                 print(f"\t\tAlready present (not copied) Copie{pid} : {label} -> {add_label}")
 
 
-        if not some_present:
+        if not keep_error:
             res["error"] = ""
 
     return new_tasks
@@ -578,6 +590,11 @@ def process_single_task(task_tuple):
             with open(output_path, "w", encoding="utf-8") as f:
                 json.dump(results, f, indent=2)
 
+            # To track progress
+            completed_tasks.append((file_path, label))
+            with open(progress_path, "w", encoding="utf-8") as f:
+                json.dump(completed_tasks, f, indent=2)
+
     except json.JSONDecodeError:
         print(f"Error decoding JSON for {file_path}", file=sys.stderr)
     except Exception as e:
@@ -587,28 +604,29 @@ def process_single_task(task_tuple):
             errors_summary.append((error_msg, file_path))
     return new_tasks
 
-print(f"Starting processing on {len(tasks_to_process)} tasks with {NB_THREADS} threads...")
+if __name__ == "__main__":
+    print(f"Starting processing on {len(tasks_to_process)} tasks with {NB_THREADS} threads...")
 
-with concurrent.futures.ThreadPoolExecutor(max_workers=NB_THREADS) as executor:
-    futures = {executor.submit(process_single_task, task): task for task in tasks_to_process}
+    with concurrent.futures.ThreadPoolExecutor(max_workers=NB_THREADS) as executor:
+        futures = {executor.submit(process_single_task, task): task for task in tasks_to_process}
 
-    # Process tasks as they complete, allowing dynamic task addition
-    for future in concurrent.futures.as_completed(futures):
-        try:
-            new_generated_tasks = future.result()
-            if new_generated_tasks:
-                for new_task in new_generated_tasks:
-                    futures[executor.submit(process_single_task, new_task)] = new_task
-        except Exception as e:
-            print(f"Exception during task execution: {e}", file=sys.stderr)
+        # Process tasks as they complete, allowing dynamic task addition
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                new_generated_tasks = future.result()
+                if new_generated_tasks:
+                    for new_task in new_generated_tasks:
+                        futures[executor.submit(process_single_task, new_task)] = new_task
+            except Exception as e:
+                print(f"Exception during task execution: {e}", file=sys.stderr)
 
 
-end_time = time.time()
-print("Time elapsed : ", end_time - start_time)
-print("Requests to pro / flash : ", pro_count, flash_count)
-if errors_summary:
-    print("\n--- Summary of Exceptions ---", file=sys.stderr)
-    for (err, file) in errors_summary:
-        print(err, file=sys.stderr)
-        escaped_path = shlex.quote(str(file))
-        print(f"Run :    python correction.py {escaped_path}")
+    end_time = time.time()
+    print("Time elapsed : ", end_time - start_time)
+    print("Requests to pro / flash : ", pro_count, flash_count)
+    if errors_summary:
+        print("\n--- Summary of Exceptions ---", file=sys.stderr)
+        for (err, file) in errors_summary:
+            print(err, file=sys.stderr)
+            escaped_path = shlex.quote(str(file))
+            print(f"Run :    python correction.py {escaped_path}")