Standardisation 6

This commit is contained in:
2026-08-20 15:01:17 +02:00
parent 644e287586
commit aa40e58dd1
4 changed files with 419 additions and 191 deletions
+12 -1
View File
@@ -171,7 +171,8 @@ scripts migrés vers cette convention sont actuellement :
- =copies_tools.py=, =grouping.py= et =verify_groups.py= ; - =copies_tools.py=, =grouping.py= et =verify_groups.py= ;
- =post-correction.py= et =resolve_manual.py= ; - =post-correction.py= et =resolve_manual.py= ;
- =cutleft.py= et =splitting_int.py= ; - =page_splitter.py=, =cutleft.py=, =plotting.py= et
=splitting_int.py= ;
- =annotating.py=, =annotating_with_checks.py= et - =annotating.py=, =annotating_with_checks.py= et
=annotating_by_label.py= ; =annotating_by_label.py= ;
- =reading_annotations.py= et =reading_grouped_annotations.py= ; - =reading_annotations.py= et =reading_grouped_annotations.py= ;
@@ -252,6 +253,12 @@ Mettre les copies scannées au format pdf dans =Interro=.
+ de déplacer la délimitation à droite/gauche + de déplacer la délimitation à droite/gauche
Fix issues with =python page_splitter.py Interro14/Copies/Copie01.pdf= Fix issues with =python page_splitter.py Interro14/Copies/Copie01.pdf=
Le PDF transformé est construit dans un dossier temporaire. La
copie produite et la sauvegarde dans =Copies Originales= sont
ensuite installées avec rollback : une erreur conserve les deux
versions précédentes. Une relance ciblée lit directement la
sauvegarde originale sans la déplacer au préalable.
4. =python cutleft.py Interro= 4. =python cutleft.py Interro=
Découpe la partie gauche des copies, là où il devrait y avoir les Découpe la partie gauche des copies, là où il devrait y avoir les
@@ -287,6 +294,10 @@ Optional : Set proxy with ~export HTTPS_PROXY="http://10.0.0.1:3128"~
Pour modifier une seule copie : Pour modifier une seule copie :
=python plotting.py Interro/Copies/Copie01.pdf= =python plotting.py Interro/Copies/Copie01.pdf=
Les coordonnées agrégées sont écrites atomiquement dans le JSON de
la copie. Fermer la fenêtre avant la fin d'une copie ne remplace pas
son JSON par un résultat incomplet.
It also generates les =Copie01.json=, à partir des =Copie01_01.json= It also generates les =Copie01.json=, à partir des =Copie01_01.json=
En cas de soucis, (par exemple les pages ne sont pas dans le bon ordre) En cas de soucis, (par exemple les pages ne sont pas dans le bon ordre)
- Réordonner les pages du fichier pdf - Réordonner les pages du fichier pdf
+171 -110
View File
@@ -1,9 +1,15 @@
from __future__ import annotations
import argparse
import glob import glob
import os import os
import re import re
import shutil import shutil
import sys import tempfile
import tkinter as tk import tkinter as tk
import uuid
from collections.abc import Sequence
from pathlib import Path
from tkinter import messagebox from tkinter import messagebox
import fitz # PyMuPDF import fitz # PyMuPDF
@@ -11,16 +17,78 @@ from PIL import Image, ImageDraw, ImageTk
from pypdf import PdfReader, PdfWriter from pypdf import PdfReader, PdfWriter
from config import PAGE_SPLITTER_KB from config import PAGE_SPLITTER_KB
from copienator import (
CliError,
EvaluationWorkspace,
ExitCode,
execute,
target_parser,
workspace_from_target,
)
from platform_utils import launch_pdf_arranger from platform_utils import launch_pdf_arranger
# --- Constants --- # --- Constants ---
# Conversion factor: 1 cm to points (1 inch = 2.54 cm, 72 points = 1 inch) # Conversion factor: 1 cm to points (1 inch = 2.54 cm, 72 points = 1 inch)
CM_TO_POINTS = (1 / 2.54) * 72 CM_TO_POINTS = (1 / 2.54) * 72
def list_pdf_files(directory): def list_pdf_files(directory: str | Path) -> list[Path]:
l = list(reversed(sorted(glob.glob(os.path.join(directory, "*.pdf"))))) paths = sorted(Path(directory).glob("*.pdf"), key=lambda path: path.name.casefold())
return [u for u in l if "enonce" not in u] return [path for path in paths if "enonce" not in path.name.casefold()]
def _temporary_sibling(path: Path, purpose: str) -> Path:
return path.with_name(f".{path.name}.{purpose}.{uuid.uuid4().hex}.tmp")
def commit_processed_pdf(
workspace: EvaluationWorkspace,
original_path: Path,
generated_path: Path,
) -> Path:
"""Commit a processed copy and its original backup with rollback."""
backup_path = workspace.original_copies_dir / original_path.name
output_path = workspace.copies_dir / original_path.name
workspace.original_copies_dir.mkdir(parents=True, exist_ok=True)
workspace.copies_dir.mkdir(parents=True, exist_ok=True)
staged_backup = None
if original_path.resolve() != backup_path.resolve():
staged_backup = _temporary_sibling(backup_path, "new-original")
shutil.copy2(original_path, staged_backup)
saved_backup = _temporary_sibling(backup_path, "old-original")
saved_output = _temporary_sibling(output_path, "old-output")
backup_replaced = False
output_replaced = False
try:
if staged_backup is not None:
if backup_path.exists():
backup_path.replace(saved_backup)
staged_backup.replace(backup_path)
backup_replaced = True
if output_path.exists():
output_path.replace(saved_output)
generated_path.replace(output_path)
output_replaced = True
if original_path.resolve() not in {
backup_path.resolve(),
output_path.resolve(),
}:
original_path.unlink()
except Exception:
if output_replaced and output_path.exists():
output_path.unlink()
if saved_output.exists():
saved_output.replace(output_path)
if backup_replaced and backup_path.exists():
backup_path.unlink()
if saved_backup.exists():
saved_backup.replace(backup_path)
raise
finally:
for temporary in (staged_backup, saved_backup, saved_output):
if temporary is not None and temporary.exists():
temporary.unlink()
return output_path
class PDFPreviewer: class PDFPreviewer:
@@ -30,26 +98,36 @@ class PDFPreviewer:
return False return False
self.pdf_path = self.inputs.pop() self.pdf_path = self.inputs.pop()
self.file_rotation = 0 self.file_rotation = 0
self.base_name = os.path.splitext(os.path.basename(self.pdf_path))[0] self.base_name = self.pdf_path.stem
self.split_dir = f"{self.base_name}_split" self._temporary_directory = tempfile.TemporaryDirectory(
self.reorder_dir = f"{self.base_name}_reorder" prefix=f".{self.base_name}.page-splitter.",
dir=self.workspace.root,
# Create a temporary output file )
self.final_file = f"{self.base_name}_temp.pdf" working_dir = Path(self._temporary_directory.name)
self.split_dir = working_dir / "split"
self.reorder_dir = working_dir / "reorder"
self.final_file = working_dir / f"{self.base_name}.pdf"
self.current_page_index = 0 self.current_page_index = 0
self.page_settings = [] self.page_settings = []
self.processing = False # Flag to prevent multiple finish calls self.processing = False # Flag to prevent multiple finish calls
try: try:
self.doc = fitz.open(self.pdf_path) self.doc = fitz.open(self.pdf_path)
except Exception as e: except (OSError, RuntimeError, ValueError) as e:
self.failed = True
self._temporary_directory.cleanup()
messagebox.showerror("Error", f"Failed to open PDF file: {e}") messagebox.showerror("Error", f"Failed to open PDF file: {e}")
self.master.destroy() self.master.destroy()
return return
self.master.title(f"PDF Splitter - {os.path.basename(self.pdf_path)}") self.master.title(f"PDF Splitter - {self.pdf_path.name}")
return True return True
def __init__(self, master, path): def __init__(
self,
master: tk.Tk,
workspace: EvaluationWorkspace,
inputs: list[Path],
) -> None:
""" """
Initializes the application. Initializes the application.
@@ -57,40 +135,16 @@ class PDFPreviewer:
master (tk.Tk): The root Tkinter window. master (tk.Tk): The root Tkinter window.
pdf_path (str): The path to the input PDF file. pdf_path (str): The path to the input PDF file.
""" """
if not os.path.exists(path): self.workspace = workspace
messagebox.showerror("Error", f"File not found: {path}") self.inputs = inputs
master.destroy()
return
if os.path.isdir(path):
self.inputs = list_pdf_files(path)
else:
# Check for existing original in backup and restore if found
dir_name = os.path.dirname(os.path.abspath(path))
file_name = os.path.basename(path)
if os.path.basename(dir_name) == "Copies":
dir_name = os.path.dirname(dir_name)
path = os.path.join(dir_name, file_name)
backup_path = os.path.join(dir_name, "Copies Originales", file_name)
if os.path.exists(backup_path):
try:
shutil.move(backup_path, path)
print(f"Restored original file from: {backup_path}")
except Exception as e:
messagebox.showerror("Error", f"Failed to restore original file: {e}")
master.destroy()
return
self.inputs = [path]
self.output_dir = None self.output_dir = None
self.master = master self.master = master
self.num = 0 self.num = 0
self.global_rotation = 0 # Rotation appliquée à tous les fichiers self.global_rotation = 0 # Rotation appliquée à tous les fichiers
self.history = [] self.history = []
self.failed = False
if not self.setup_next_file(): if not self.setup_next_file():
print(f"Aucun fichier PDF valide trouvé dans : {path}") print(f"No PDF files found in {workspace.root}")
master.destroy() master.destroy()
return return
@@ -249,7 +303,7 @@ class PDFPreviewer:
# Re-open the file from disk to reset changes (like moved pages) # Re-open the file from disk to reset changes (like moved pages)
try: try:
self.doc = fitz.open(self.pdf_path) self.doc = fitz.open(self.pdf_path)
except Exception as e: except (OSError, RuntimeError, ValueError) as e:
messagebox.showerror("Error", f"Failed to reopen PDF file: {e}") messagebox.showerror("Error", f"Failed to reopen PDF file: {e}")
self.master.destroy() self.master.destroy()
return return
@@ -328,7 +382,8 @@ class PDFPreviewer:
self._initialize_current_page_settings() self._initialize_current_page_settings()
self.load_page() self.load_page()
else: else:
self.finish_and_process() if not self.finish_and_process():
return
self.history.append(self.pdf_path) self.history.append(self.pdf_path)
if self.setup_next_file(): if self.setup_next_file():
self._initialize_current_page_settings() self._initialize_current_page_settings()
@@ -336,59 +391,24 @@ class PDFPreviewer:
else: else:
self.master.destroy() self.master.destroy()
def finish_and_process(self): def finish_and_process(self) -> bool:
"""Starts the PDF splitting process and moves files.""" """Render and transactionally install the processed PDF."""
self.split_pdf()
# print("Debug : ", self.page_settings)
# input("Splitting done. Continue ?")
self.reorder_pdfs()
# input("Reorder done. Continue ?")
self.concate_files()
# Logic to move original to backup and replace with new file
try: try:
abs_path = os.path.abspath(self.pdf_path) self.split_pdf()
dir_name = os.path.dirname(abs_path) self.reorder_pdfs()
file_name = os.path.basename(abs_path) self.concate_files()
commit_processed_pdf(self.workspace, self.pdf_path, self.final_file)
backup_dir = os.path.join(dir_name, "Copies Originales") except Exception as exc: # noqa: BLE001 - interactive boundary
copies_dir = os.path.join(dir_name, "Copies") self.failed = True
os.makedirs(backup_dir, exist_ok=True) self.processing = False
os.makedirs(copies_dir, exist_ok=True) print(f"Failed to process {self.pdf_path}: {exc}")
messagebox.showerror("Error", f"Failed to process PDF: {exc}")
backup_path = os.path.join(backup_dir, file_name) self._temporary_directory.cleanup()
copies_path = os.path.join(copies_dir, file_name) self.master.destroy()
return False
# Remove backup if it already exists (overwrite) else:
if os.path.exists(backup_path): self._temporary_directory.cleanup()
os.remove(backup_path) return True
# Move the original file to "Copies Originales"
shutil.move(self.pdf_path, backup_path)
# Move the temp output file to replace the original
shutil.move(self.final_file, copies_path)
# print(f"Original moved to {backup_path}, new file saved at {self.pdf_path}")
except Exception as e:
messagebox.showerror("Error", f"Failed to move/replace files: {e}")
self.remove_dirs()
def _restore_original(self, path):
"""Restores the original file from the 'Copies Originales' backup."""
dir_name = os.path.dirname(os.path.abspath(path))
file_name = os.path.basename(path)
backup_path = os.path.join(dir_name, "Copies Originales", file_name)
if os.path.exists(backup_path):
try:
# Moving overwrites the generated PDF with the original backup
shutil.move(backup_path, path)
print(f"Restored original file from: {backup_path}")
except Exception as e:
print(f"Failed to restore original file: {e}")
def go_to_previous_file(self, event=None): def go_to_previous_file(self, event=None):
"""Goes back to the beginning of the previously completed file.""" """Goes back to the beginning of the previously completed file."""
@@ -398,14 +418,16 @@ class PDFPreviewer:
# Close the currently open document to avoid lock issues # Close the currently open document to avoid lock issues
if hasattr(self, 'doc'): if hasattr(self, 'doc'):
self.doc.close() self.doc.close()
if hasattr(self, "_temporary_directory"):
self._temporary_directory.cleanup()
# 1. Push current file back onto the stack so it processes next # 1. Push current file back onto the stack so it processes next
self.inputs.append(self.pdf_path) self.inputs.append(self.pdf_path)
# 2. Get the previous file, restore its original state, and push to stack # 2. Reprocess the previous file from its preserved original backup
prev_file = self.history.pop() prev_file = self.history.pop()
self._restore_original(prev_file) backup = self.workspace.original_copies_dir / Path(prev_file).name
self.inputs.append(prev_file) self.inputs.append(backup if backup.is_file() else Path(prev_file))
# 3. Reload environment (setup_next_file will pop prev_file back off the stack) # 3. Reload environment (setup_next_file will pop prev_file back off the stack)
self.setup_next_file() self.setup_next_file()
@@ -426,13 +448,9 @@ class PDFPreviewer:
for pdf in pdf_files: for pdf in pdf_files:
try: try:
os.remove(pdf) os.remove(pdf)
except Exception as e: except OSError as e:
print(f"Error deleting {pdf}: {e}") print(f"Error deleting {pdf}: {e}")
def remove_dirs(self):
shutil.rmtree(self.split_dir)
shutil.rmtree(self.reorder_dir)
def split_pdf(self): def split_pdf(self):
"""Splits each page of the PDF according to the saved settings.""" """Splits each page of the PDF according to the saved settings."""
print("Starting PDF processing...") print("Starting PDF processing...")
@@ -588,13 +606,56 @@ class PDFPreviewer:
print(f"Created merged PDF: {self.final_file}") print(f"Created merged PDF: {self.final_file}")
if __name__ == "__main__": def _selected_inputs(
if len(sys.argv) != 2: workspace: EvaluationWorkspace,
print("Usage: python script_name.py <path_to_pdf_file>") target: Path,
sys.exit(1) ) -> list[Path]:
if target.is_file():
if target.suffix.casefold() != ".pdf":
raise CliError(f"Target is not a PDF: {target}", ExitCode.INVALID_ARGUMENTS)
backup = workspace.original_copies_dir / target.name
return [backup if backup.is_file() else target]
pdf_file_path = sys.argv[1] directory = target
if target == workspace.copies_dir:
candidates = list_pdf_files(workspace.copies_dir)
candidates = [
(
workspace.original_copies_dir / path.name
if (workspace.original_copies_dir / path.name).is_file()
else path
)
for path in candidates
]
else:
candidates = list_pdf_files(directory)
return list(reversed(candidates))
def run(workspace: EvaluationWorkspace, target: Path) -> ExitCode:
inputs = _selected_inputs(workspace, target)
if not inputs:
print(f"No PDF files found in {target}")
return ExitCode.SUCCESS
root = tk.Tk() root = tk.Tk()
app = PDFPreviewer(root, pdf_file_path) application = PDFPreviewer(root, workspace, inputs)
root.mainloop() root.mainloop()
return ExitCode.FAILURE if application.failed else ExitCode.SUCCESS
def build_parser() -> argparse.ArgumentParser:
return target_parser("Interactively split and reorder scanned PDF pages")
def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
def handle(args: argparse.Namespace) -> ExitCode:
workspace, target = workspace_from_target(args)
return run(workspace, target)
return execute(parser, argv, handle)
if __name__ == "__main__":
raise SystemExit(main())
+117 -80
View File
@@ -1,24 +1,29 @@
import json from __future__ import annotations
import argparse
import queue import queue
import sys
import threading import threading
import tkinter as tk import tkinter as tk
from collections.abc import Sequence
from pathlib import Path from pathlib import Path
from tkinter import messagebox from tkinter import messagebox
from PIL import Image, ImageDraw, ImageFont, ImageTk from PIL import Image, ImageDraw, ImageFont, ImageTk
from copienator import (
EvaluationWorkspace,
ExitCode,
atomic_write_json,
execute,
read_json,
target_parser,
workspace_from_target,
)
from platform_utils import open_path from platform_utils import open_path
from utils import natural_key, read_all_labels
print("o to open pdf, O original pdf, e to emacs part, p to go back, i to interro, click for coordinates")
# --- Configuration & Globals --- # --- Configuration & Globals ---
padding = 60 padding = 60
valid_labels_set = None
# Queue payload: (pil_image, json_path, metadata)
# metadata is a dict: {'copie': str, 'part': int, 'schema': dict}
image_queue = queue.Queue(maxsize=5)
try: try:
font = ImageFont.truetype("DejaVuSans.ttf", size=30) font = ImageFont.truetype("DejaVuSans.ttf", size=30)
@@ -55,6 +60,14 @@ def convert_list(l, group_id, json_schema):
ll.append(ee) ll.append(ee)
return ll return ll
def normalized_labels(entries):
return [
str(value["label"]).removeprefix("|").removesuffix("|")
for value in entries
if str(value["label"]) != "_"
]
def prepare_image(image_path: str, bounding_boxes, all_labels, nb_pages, last_label_index): def prepare_image(image_path: str, bounding_boxes, all_labels, nb_pages, last_label_index):
im = Image.open(image_path) im = Image.open(image_path)
im.load() im.load()
@@ -94,7 +107,7 @@ def prepare_image(image_path: str, bounding_boxes, all_labels, nb_pages, last_la
# --- Processing Logic (Worker Thread) --- # --- Processing Logic (Worker Thread) ---
def worker_thread(base_dir, files_to_process, all_labels): def _worker_items(base_dir, files_to_process, all_labels, output_queue):
""" """
Iterates through files, prepares VISUALS only, and puts metadata in queue. Iterates through files, prepares VISUALS only, and puts metadata in queue.
Does NOT write final JSON files anymore. Does NOT write final JSON files anymore.
@@ -111,9 +124,8 @@ def worker_thread(base_dir, files_to_process, all_labels):
json_schema_path = base_dir / 'Cutleft' / f"{copie}_schema.json" json_schema_path = base_dir / 'Cutleft' / f"{copie}_schema.json"
try: try:
with open(json_schema_path, 'r') as f: json_schema = read_json(json_schema_path)
json_schema = json.load(f) except (OSError, TypeError, ValueError):
except:
print("No json_schema : ", json_schema_path) print("No json_schema : ", json_schema_path)
continue continue
@@ -124,11 +136,10 @@ def worker_thread(base_dir, files_to_process, all_labels):
bb_list = [] bb_list = []
json_name = "" json_name = ""
try: try:
with open(json_path, 'r') as f: json_result = read_json(json_path)
json_result = json.load(f)
bb_list = json_result.get("list", []) bb_list = json_result.get("list", [])
json_name = json_result.get("name", "") json_name = json_result.get("name", "")
except Exception as e: except Exception as e: # noqa: BLE001 - malformed user-editable JSON
print(f"Warning: {json_path.name} is malformed! Loading blank. {e}") print(f"Warning: {json_path.name} is malformed! Loading blank. {e}")
# We do NOT skip; we continue so the user can fix it in the GUI # We do NOT skip; we continue so the user can fix it in the GUI
@@ -138,7 +149,7 @@ def worker_thread(base_dir, files_to_process, all_labels):
prepare_image(str(img_path), bb_list, all_labels, nb_pages, last_label_index) prepare_image(str(img_path), bb_list, all_labels, nb_pages, last_label_index)
error_msg = None error_msg = None
except Exception as e: except Exception as e: # noqa: BLE001 - keep the item editable in the GUI
print(f"Error processing {img_path.name}: {e}") print(f"Error processing {img_path.name}: {e}")
pil_image = Image.open(str(img_path)) pil_image = Image.open(str(img_path))
error_msg = str(e) error_msg = str(e)
@@ -151,15 +162,24 @@ def worker_thread(base_dir, files_to_process, all_labels):
"error": error_msg "error": error_msg
} }
image_queue.put((pil_image, json_path, metadata)) output_queue.put((pil_image, json_path, metadata))
# Sentinel to indicate finished def worker_thread(base_dir, files_to_process, all_labels, output_queue):
image_queue.put((None, None, None)) """Prepare queue items and always terminate the GUI stream."""
failure = None
try:
_worker_items(base_dir, files_to_process, all_labels, output_queue)
except Exception as exc: # noqa: BLE001 - worker boundary
failure = str(exc)
print(f"Plotting worker failed: {exc}")
finally:
metadata = {"worker_error": failure} if failure else None
output_queue.put((None, None, metadata))
# --- GUI Logic (Main Thread) --- # --- GUI Logic (Main Thread) ---
class ImageViewer: class ImageViewer:
def __init__(self, root, base_dir): def __init__(self, root, workspace, valid_labels, input_queue):
self.root = root self.root = root
self.root.resizable(False, False) # If you resize, coordinates will be wrong self.root.resizable(False, False) # If you resize, coordinates will be wrong
@@ -171,7 +191,10 @@ class ImageViewer:
root.geometry(f"+{x}+{y}") root.geometry(f"+{x}+{y}")
self.base_dir = base_dir self.workspace = workspace
self.base_dir = workspace.root
self.valid_labels = valid_labels
self.image_queue = input_queue
self.root.title("Bounding Box Viewer") self.root.title("Bounding Box Viewer")
self.label = tk.Label(root, text="Waiting for images...") self.label = tk.Label(root, text="Waiting for images...")
self.label.pack(expand=True, fill="both") self.label.pack(expand=True, fill="both")
@@ -192,6 +215,7 @@ class ImageViewer:
self.history = [] self.history = []
self.forward_stack = [] self.forward_stack = []
self.current_pil_image = None self.current_pil_image = None
self.failed = False
from config import PLOTTING_KB from config import PLOTTING_KB
@@ -202,7 +226,8 @@ class ImageViewer:
self.root.bind(PLOTTING_KB["open pdf"], self.on_open_pdf) self.root.bind(PLOTTING_KB["open pdf"], self.on_open_pdf)
self.root.bind(PLOTTING_KB["open original pdf"], self.on_open_ori_pdf) self.root.bind(PLOTTING_KB["open original pdf"], self.on_open_ori_pdf)
self.root.bind(PLOTTING_KB["open eval"], self.on_open_interro) self.root.bind(PLOTTING_KB["open eval"], self.on_open_interro)
self.root.bind('<Escape>', lambda e: self.root.quit()) self.root.bind('<Escape>', lambda _event: self.close())
self.root.protocol("WM_DELETE_WINDOW", self.close)
self.label.bind('<Button-1>', self.on_click) self.label.bind('<Button-1>', self.on_click)
self.poll_queue() self.poll_queue()
@@ -214,10 +239,15 @@ class ImageViewer:
if self.forward_stack: if self.forward_stack:
pil_image, json_path, metadata = self.forward_stack.pop() pil_image, json_path, metadata = self.forward_stack.pop()
else: else:
pil_image, json_path, metadata = image_queue.get_nowait() pil_image, json_path, metadata = self.image_queue.get_nowait()
# Handle End of Stream # Handle End of Stream
if pil_image is None: if pil_image is None:
if metadata and metadata.get("worker_error"):
self.failed = True
messagebox.showerror(
"Processing Error", metadata["worker_error"]
)
self.save_current_batch() # Save any remaining data self.save_current_batch() # Save any remaining data
print("All images processed.") print("All images processed.")
self.root.quit() self.root.quit()
@@ -241,10 +271,12 @@ class ImageViewer:
if self.active_copie_name and self.accumulated_results: if self.active_copie_name and self.accumulated_results:
main_json_path = self.base_dir / "Copies" / f"{self.active_copie_name}.json" main_json_path = self.base_dir / "Copies" / f"{self.active_copie_name}.json"
print(f"Writing aggregated result to {main_json_path}") print(f"Writing aggregated result to {main_json_path}")
with open(main_json_path, 'w') as f: atomic_write_json(main_json_path, self.accumulated_results)
json.dump(self.accumulated_results, f)
self.accumulated_results = None self.accumulated_results = None
def close(self):
self.root.quit()
def on_previous(self, event): def on_previous(self, event):
if self.is_viewing and self.history: if self.is_viewing and self.history:
@@ -288,8 +320,7 @@ class ImageViewer:
num_added = 0 # ADD THIS LINE num_added = 0 # ADD THIS LINE
try: try:
with open(self.current_json_path, 'r') as f: current_data = read_json(self.current_json_path)
current_data = json.load(f)
# Perform the conversion now, post-edit # Perform the conversion now, post-edit
converted_items = convert_list( converted_items = convert_list(
@@ -298,11 +329,10 @@ class ImageViewer:
self.current_meta["schema"] self.current_meta["schema"]
) )
labels = [v["label"] for v in current_data["list"]] labels = normalized_labels(current_data["list"])
labels = [label for label in labels if label != "_"] false_labels = [
labels = [label[1:] for label in labels if label[0] == "|"] label for label in labels if label not in self.valid_labels
labels = [label[:-1] for label in labels if label[-1] == "|"] ]
false_labels = [label for label in labels if label not in valid_labels_set]
if false_labels: if false_labels:
msg = f"Wrong label in {self.current_json_path.name}: {false_labels}\n\n\tPlease press 'e' to fix it, then press Enter again." msg = f"Wrong label in {self.current_json_path.name}: {false_labels}\n\n\tPlease press 'e' to fix it, then press Enter again."
@@ -318,7 +348,7 @@ class ImageViewer:
if "name" in current_data and current_data["name"] != "Continued": if "name" in current_data and current_data["name"] != "Continued":
self.accumulated_results["name"] = current_data["name"] self.accumulated_results["name"] = current_data["name"]
except Exception as e: except Exception as e: # noqa: BLE001 - interactive validation boundary
# Warn user and STOP (do not advance to next image) # Warn user and STOP (do not advance to next image)
msg = f"Error reading {self.current_json_path.name}:\n\n{e}\n\nPlease press 'e' to fix it, then press Enter again." msg = f"Error reading {self.current_json_path.name}:\n\n{e}\n\nPlease press 'e' to fix it, then press Enter again."
print(msg) print(msg)
@@ -387,51 +417,58 @@ class ImageViewer:
self.root.clipboard_clear() self.root.clipboard_clear()
self.root.clipboard_append(box_str) self.root.clipboard_append(box_str)
from utils import natural_key, read_all_labels def _selected_images(workspace: EvaluationWorkspace, target: Path) -> list[Path]:
workspace.require_directories("Cutleft", "Copies")
if target.is_file():
stem = target.stem
exact = workspace.cutleft_dir / f"{stem}.jpg"
if exact.is_file():
return [exact]
return sorted(
workspace.cutleft_dir.glob(f"{stem}_*.jpg"),
key=natural_key,
)
return sorted(workspace.cutleft_dir.glob("*.jpg"), key=natural_key)
def run(workspace: EvaluationWorkspace, target: Path) -> ExitCode:
workspace.require_files("labels")
all_labels = read_all_labels(workspace.root)
files_to_process = _selected_images(workspace, target)
if not files_to_process:
print(f"No Cutleft images found for {target}")
return ExitCode.PARTIAL
print(
"o to open pdf, O original pdf, e to edit part, p to go back, "
"i to open the statement, click for coordinates"
)
input_queue = queue.Queue(maxsize=5)
worker = threading.Thread(
target=worker_thread,
args=(workspace.root, files_to_process, all_labels, input_queue),
daemon=True,
)
worker.start()
root = tk.Tk()
application = ImageViewer(root, workspace, set(all_labels), input_queue)
root.mainloop()
return ExitCode.PARTIAL if application.failed else ExitCode.SUCCESS
def build_parser() -> argparse.ArgumentParser:
return target_parser("Interactively verify detected label coordinates")
def main(argv: Sequence[str] | None = None) -> int:
parser = build_parser()
def handle(args: argparse.Namespace) -> ExitCode:
workspace, target = workspace_from_target(args)
return run(workspace, target)
return execute(parser, argv, handle)
if __name__ == "__main__": if __name__ == "__main__":
if len(sys.argv) < 2: raise SystemExit(main())
print("Usage: python plotting.py <directory_or_file>")
sys.exit(1)
input_path = Path(sys.argv[1])
files_to_process = []
if input_path.is_file():
# Correctly identify base_dir if we are in 'Copies' or 'Cutleft'
if input_path.parent.name in ["Copies", "Cutleft"]:
base_dir = input_path.parent.parent
else:
base_dir = input_path.parent
stem = input_path.stem
cutleft_dir = base_dir / "Cutleft"
img_path = cutleft_dir / f"{stem}.jpg"
if img_path.exists():
files_to_process = [img_path]
else:
# We're given something like Copie01.pdf, look for its split image parts
files_to_process = sorted(list(cutleft_dir.glob(f"{stem}_*.jpg")), key=natural_key)
else:
base_dir = input_path
cutleft_dir = base_dir / "Cutleft"
if not cutleft_dir.exists():
print(f"Error: {cutleft_dir} does not exist.")
sys.exit(1)
files_to_process = sorted(cutleft_dir.glob("*.jpg"))
try:
all_labels = read_all_labels(base_dir)
except FileNotFoundError:
all_labels = []
valid_labels_set = set(all_labels)
t = threading.Thread(target=worker_thread, args=(base_dir, files_to_process, all_labels))
t.daemon = True
t.start()
root = tk.Tk()
app = ImageViewer(root, base_dir)
root.mainloop()
+119
View File
@@ -306,6 +306,10 @@ class StandardCliTests(unittest.TestCase):
"splitting_int": load_script_module( "splitting_int": load_script_module(
"splitting_int.py", "splitting_int" "splitting_int.py", "splitting_int"
), ),
"page_splitter": load_script_module(
"page_splitter.py", "page_splitter"
),
"plotting": load_script_module("plotting.py", "plotting"),
"copies_tools": load_script_module( "copies_tools": load_script_module(
"copies_tools.py", "copienator_copies_tools_test" "copies_tools.py", "copienator_copies_tools_test"
), ),
@@ -345,6 +349,8 @@ class StandardCliTests(unittest.TestCase):
"reading_grouped_annotations": [missing], "reading_grouped_annotations": [missing],
"cutleft": [missing], "cutleft": [missing],
"splitting_int": [missing], "splitting_int": [missing],
"page_splitter": [missing],
"plotting": [missing],
} }
for name, arguments in invocations.items(): for name, arguments in invocations.items():
with self.subTest(script=name), redirect_stderr(io.StringIO()): with self.subTest(script=name), redirect_stderr(io.StringIO()):
@@ -447,6 +453,16 @@ class StandardCliTests(unittest.TestCase):
"default", "default",
{"target": evaluation}, {"target": evaluation},
), ),
"page_splitter": (
"page_splitter",
"default",
{"target": evaluation},
),
"plotting": (
"plotting",
"default",
{"target": evaluation},
),
} }
for module_name, (step_id, variant_id, values) in cases.items(): for module_name, (step_id, variant_id, values) in cases.items():
step = steps[step_id] step = steps[step_id]
@@ -654,6 +670,109 @@ class StandardCliTests(unittest.TestCase):
self.assertTrue(answer.is_file()) self.assertTrue(answer.is_file())
self.assertEqual(len(PdfReader(answer).pages), 1) self.assertEqual(len(PdfReader(answer).pages), 1)
def test_page_splitter_commits_original_and_generated_copy(self) -> None:
module = self.modules["page_splitter"]
with tempfile.TemporaryDirectory() as directory:
evaluation = Path(directory) / "Exam"
evaluation.mkdir()
original = evaluation / "Copie01.pdf"
generated = evaluation / ".generated.pdf"
original.write_bytes(b"original")
generated.write_bytes(b"processed")
workspace = EvaluationWorkspace(evaluation)
output = module.commit_processed_pdf(workspace, original, generated)
self.assertEqual(output, evaluation / "Copies" / "Copie01.pdf")
self.assertEqual(output.read_bytes(), b"processed")
self.assertEqual(
(evaluation / "Copies Originales" / "Copie01.pdf").read_bytes(),
b"original",
)
self.assertFalse(original.exists())
def test_page_splitter_commit_failure_rolls_back_both_files(self) -> None:
module = self.modules["page_splitter"]
with tempfile.TemporaryDirectory() as directory:
evaluation = Path(directory) / "Exam"
original = evaluation / "Copie01.pdf"
backup = evaluation / "Copies Originales" / "Copie01.pdf"
output = evaluation / "Copies" / "Copie01.pdf"
backup.parent.mkdir(parents=True)
output.parent.mkdir()
original.write_bytes(b"new original")
backup.write_bytes(b"previous original")
output.write_bytes(b"previous output")
missing_generated = evaluation / "missing-generated.pdf"
with self.assertRaises(FileNotFoundError):
module.commit_processed_pdf(
EvaluationWorkspace(evaluation), original, missing_generated
)
self.assertEqual(original.read_bytes(), b"new original")
self.assertEqual(backup.read_bytes(), b"previous original")
self.assertEqual(output.read_bytes(), b"previous output")
def test_page_splitter_reprocesses_from_preserved_original(self) -> None:
module = self.modules["page_splitter"]
with tempfile.TemporaryDirectory() as directory:
evaluation = Path(directory) / "Exam"
generated = evaluation / "Copies" / "Copie01.pdf"
original = evaluation / "Copies Originales" / "Copie01.pdf"
generated.parent.mkdir(parents=True)
original.parent.mkdir()
generated.write_bytes(b"processed")
original.write_bytes(b"original")
workspace = EvaluationWorkspace(evaluation)
self.assertEqual(
module._selected_inputs(workspace, generated),
[original],
)
def test_plotting_batch_save_is_atomic(self) -> None:
module = self.modules["plotting"]
with tempfile.TemporaryDirectory() as directory:
evaluation = Path(directory) / "Exam"
(evaluation / "Copies").mkdir(parents=True)
viewer = module.ImageViewer.__new__(module.ImageViewer)
viewer.base_dir = evaluation
viewer.active_copie_name = "Copie01"
viewer.accumulated_results = {
"name": "Test",
"list": [{"label": "Ex 1"}],
}
viewer.save_current_batch()
self.assertEqual(
read_json(evaluation / "Copies" / "Copie01.json"),
{"name": "Test", "list": [{"label": "Ex 1"}]},
)
self.assertIsNone(viewer.accumulated_results)
def test_plotting_validates_plain_and_directional_labels(self) -> None:
module = self.modules["plotting"]
self.assertEqual(
module.normalized_labels(
[
{"label": "Ex 1"},
{"label": "|Ex 2"},
{"label": "Ex 3|"},
{"label": "_"},
]
),
["Ex 1", "Ex 2", "Ex 3"],
)
def test_plotting_worker_failure_still_terminates_its_queue(self) -> None:
module = self.modules["plotting"]
output_queue = queue.Queue()
with patch.object(
module, "_worker_items", side_effect=RuntimeError("worker failed")
):
module.worker_thread(Path("."), [], [], output_queue)
image, json_path, metadata = output_queue.get_nowait()
self.assertIsNone(image)
self.assertIsNone(json_path)
self.assertEqual(metadata, {"worker_error": "worker failed"})
def test_post_correction_main_cleans_json_atomically(self) -> None: def test_post_correction_main_cleans_json_atomically(self) -> None:
module = self.modules["post_correction"] module = self.modules["post_correction"]
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory: