Standardisation 5
This commit is contained in:
+10
@@ -171,6 +171,7 @@ scripts migrés vers cette convention sont actuellement :
|
|||||||
|
|
||||||
- =copies_tools.py=, =grouping.py= et =verify_groups.py= ;
|
- =copies_tools.py=, =grouping.py= et =verify_groups.py= ;
|
||||||
- =post-correction.py= et =resolve_manual.py= ;
|
- =post-correction.py= et =resolve_manual.py= ;
|
||||||
|
- =cutleft.py= et =splitting_int.py= ;
|
||||||
- =annotating.py=, =annotating_with_checks.py= et
|
- =annotating.py=, =annotating_with_checks.py= et
|
||||||
=annotating_by_label.py= ;
|
=annotating_by_label.py= ;
|
||||||
- =reading_annotations.py= et =reading_grouped_annotations.py= ;
|
- =reading_annotations.py= et =reading_grouped_annotations.py= ;
|
||||||
@@ -260,6 +261,10 @@ Mettre les copies scannées au format pdf dans =Interro=.
|
|||||||
|
|
||||||
Rerun on a single file with =python cutleft.py Interro/Copies/Copie01.pdf=
|
Rerun on a single file with =python cutleft.py Interro/Copies/Copie01.pdf=
|
||||||
|
|
||||||
|
Les images et le fichier =_schema.json= d'une copie sont remplacés
|
||||||
|
ensemble. Fermer l'outil juste après la dernière validation ne peut
|
||||||
|
donc plus interrompre un thread de sauvegarde en arrière-plan.
|
||||||
|
|
||||||
** Labelisation et regroupement
|
** Labelisation et regroupement
|
||||||
|
|
||||||
Optional : Set proxy with ~export HTTPS_PROXY="http://10.0.0.1:3128"~
|
Optional : Set proxy with ~export HTTPS_PROXY="http://10.0.0.1:3128"~
|
||||||
@@ -291,6 +296,11 @@ Optional : Set proxy with ~export HTTPS_PROXY="http://10.0.0.1:3128"~
|
|||||||
|
|
||||||
Découpe les copies suivant les exercices
|
Découpe les copies suivant les exercices
|
||||||
Peut-être appelé avec une seule copie.
|
Peut-être appelé avec une seule copie.
|
||||||
|
|
||||||
|
Les réponses d'une copie sont préparées dans un dossier temporaire,
|
||||||
|
puis remplacent ensemble le dossier précédent. En cas d'erreur,
|
||||||
|
l'ancienne version est conservée. Les réponses devenues obsolètes
|
||||||
|
restent archivées dans le sous-dossier =Missing=.
|
||||||
4. =python grouping.py Interro=
|
4. =python grouping.py Interro=
|
||||||
|
|
||||||
Regroupe les mêmes questions de différentes copies en groupes de
|
Regroupe les mêmes questions de différentes copies en groupes de
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import shutil
|
import shutil
|
||||||
import uuid
|
import uuid
|
||||||
|
from collections.abc import Iterable
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -44,7 +45,11 @@ def staged_directory(destination: str | Path):
|
|||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def staged_files(destination: str | Path):
|
def staged_files(
|
||||||
|
destination: str | Path,
|
||||||
|
*,
|
||||||
|
remove: Iterable[str] = (),
|
||||||
|
):
|
||||||
"""Stage a set of files and merge them into a directory with rollback."""
|
"""Stage a set of files and merge them into a directory with rollback."""
|
||||||
target = Path(destination)
|
target = Path(destination)
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -56,9 +61,15 @@ def staged_files(destination: str | Path):
|
|||||||
try:
|
try:
|
||||||
yield staging
|
yield staging
|
||||||
staged = sorted(path for path in staging.iterdir() if path.is_file())
|
staged = sorted(path for path in staging.iterdir() if path.is_file())
|
||||||
|
staged_names = {path.name for path in staged}
|
||||||
|
removed_names = set(remove) - staged_names
|
||||||
target.mkdir(parents=True, exist_ok=True)
|
target.mkdir(parents=True, exist_ok=True)
|
||||||
backup.mkdir()
|
backup.mkdir()
|
||||||
try:
|
try:
|
||||||
|
for name in sorted(removed_names):
|
||||||
|
destination_path = target / name
|
||||||
|
if destination_path.is_file() or destination_path.is_symlink():
|
||||||
|
destination_path.replace(backup / name)
|
||||||
for source in staged:
|
for source in staged:
|
||||||
destination_path = target / source.name
|
destination_path = target / source.name
|
||||||
if destination_path.exists() or destination_path.is_symlink():
|
if destination_path.exists() or destination_path.is_symlink():
|
||||||
|
|||||||
+236
-259
@@ -1,366 +1,343 @@
|
|||||||
import sys
|
from __future__ import annotations
|
||||||
from functools import lru_cache
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
import json # Added for schema output
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
import tkinter as tk
|
import tkinter as tk
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
from queue import Empty, Queue
|
||||||
from threading import Thread
|
from threading import Thread
|
||||||
from queue import Queue, Empty
|
|
||||||
from pdf2image import convert_from_path
|
from pdf2image import convert_from_path
|
||||||
from PIL import Image, ImageTk
|
from PIL import Image, ImageTk
|
||||||
|
|
||||||
# --- Configuration ---
|
from copienator import (
|
||||||
|
CliError,
|
||||||
|
EvaluationWorkspace,
|
||||||
|
ExitCode,
|
||||||
|
atomic_write_json,
|
||||||
|
execute,
|
||||||
|
target_parser,
|
||||||
|
workspace_from_target,
|
||||||
|
)
|
||||||
|
from copienator.filesystem import staged_files
|
||||||
|
|
||||||
DELIMITER_WIDTH = 5
|
DELIMITER_WIDTH = 5
|
||||||
DELIMITER_COLOR = (0, 0, 0)
|
DELIMITER_COLOR = (0, 0, 0)
|
||||||
OUTPUT_SIZE = (1800, 1000)
|
OUTPUT_SIZE = (1800, 1000)
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description="PDF Cropper")
|
|
||||||
parser.add_argument("path", help="Directory path or PDF file path")
|
|
||||||
parser.add_argument("--fullpage", action="store_true", help="Process all files in full page mode (1 page per output file)")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
path_arg = args.path
|
|
||||||
fullpage_mode = args.fullpage
|
|
||||||
files = []
|
|
||||||
INPUT_DIR = ""
|
|
||||||
COPIES_DIR = ""
|
|
||||||
|
|
||||||
if os.path.isfile(path_arg) and path_arg.lower().endswith('.pdf'):
|
|
||||||
COPIES_DIR = os.path.abspath(os.path.dirname(path_arg))
|
|
||||||
# If the file is inside a "Copies" folder, set INPUT_DIR to the parent
|
|
||||||
if os.path.basename(COPIES_DIR).lower() == 'copies':
|
|
||||||
INPUT_DIR = os.path.dirname(COPIES_DIR)
|
|
||||||
else:
|
|
||||||
INPUT_DIR = COPIES_DIR
|
|
||||||
files = [os.path.basename(path_arg)]
|
|
||||||
elif os.path.isdir(path_arg):
|
|
||||||
# Support passing either the base dir or the Copies dir directly
|
|
||||||
abs_path = os.path.abspath(path_arg)
|
|
||||||
if os.path.basename(abs_path).lower() == 'copies':
|
|
||||||
COPIES_DIR = abs_path
|
|
||||||
INPUT_DIR = os.path.dirname(abs_path)
|
|
||||||
else:
|
|
||||||
INPUT_DIR = abs_path
|
|
||||||
COPIES_DIR = os.path.join(INPUT_DIR, 'Copies')
|
|
||||||
|
|
||||||
if os.path.exists(COPIES_DIR):
|
|
||||||
files = sorted([f for f in os.listdir(COPIES_DIR) if f.lower().endswith('.pdf') and
|
|
||||||
"nonc" not in f.lower()])
|
|
||||||
else:
|
|
||||||
sys.exit(f"Error: Could not find 'Copies' directory inside {INPUT_DIR}")
|
|
||||||
else:
|
|
||||||
sys.exit("Error: Input must be a directory or a PDF file.")
|
|
||||||
|
|
||||||
|
|
||||||
OUTPUT_DIR = os.path.join(INPUT_DIR, 'Cutleft')
|
|
||||||
|
|
||||||
if not os.path.exists(OUTPUT_DIR):
|
|
||||||
os.makedirs(OUTPUT_DIR)
|
|
||||||
|
|
||||||
# --- Processing Logic ---
|
|
||||||
|
|
||||||
def distribute_pages(total_pages, max_per_file=5):
|
|
||||||
"""
|
|
||||||
Calculates how to split pages into chunks <= max_per_file,
|
|
||||||
balancing the number of columns per file.
|
|
||||||
Example: 12 pages, max 5 -> [4, 4, 4]
|
|
||||||
"""
|
|
||||||
if total_pages == 0:
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Calculate minimum number of files needed
|
|
||||||
num_files = (total_pages + max_per_file - 1) // max_per_file
|
|
||||||
|
|
||||||
# Calculate base size and remainder
|
|
||||||
base_count = total_pages // num_files
|
|
||||||
remainder = total_pages % num_files
|
|
||||||
|
|
||||||
distribution = []
|
|
||||||
for i in range(num_files):
|
|
||||||
# Distribute remainder to the first few files
|
|
||||||
count = base_count + (1 if i < remainder else 0)
|
|
||||||
distribution.append(count)
|
|
||||||
|
|
||||||
return distribution
|
|
||||||
|
|
||||||
def stitch_images(image_list):
|
|
||||||
"""Helper to stitch a list of images horizontally with delimiters."""
|
|
||||||
if not image_list:
|
|
||||||
return None
|
|
||||||
|
|
||||||
num_images = len(image_list)
|
|
||||||
total_width = sum(img.width for img in image_list) + (num_images - 1) * DELIMITER_WIDTH
|
|
||||||
max_height = max(img.height for img in image_list)
|
|
||||||
|
|
||||||
combined = Image.new('RGB', (total_width, max_height), color=(255, 255, 255))
|
|
||||||
|
|
||||||
x_offset = 0
|
|
||||||
for idx, img in enumerate(image_list):
|
|
||||||
combined.paste(img, (x_offset, 0))
|
|
||||||
x_offset += img.width
|
|
||||||
if idx < num_images - 1:
|
|
||||||
delimiter = Image.new('RGB', (DELIMITER_WIDTH, max_height), color=DELIMITER_COLOR)
|
|
||||||
combined.paste(delimiter, (x_offset, 0))
|
|
||||||
x_offset += DELIMITER_WIDTH
|
|
||||||
|
|
||||||
return combined
|
|
||||||
|
|
||||||
import threading
|
|
||||||
pdf_cache_lock = threading.Lock()
|
pdf_cache_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def distribute_pages(total_pages: int, max_per_file: int = 5) -> list[int]:
|
||||||
|
"""Distribute pages into balanced chunks no larger than max_per_file."""
|
||||||
|
if total_pages == 0:
|
||||||
|
return []
|
||||||
|
number_of_files = (total_pages + max_per_file - 1) // max_per_file
|
||||||
|
base_count, remainder = divmod(total_pages, number_of_files)
|
||||||
|
return [
|
||||||
|
base_count + (1 if index < remainder else 0)
|
||||||
|
for index in range(number_of_files)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def stitch_images(image_list: list[Image.Image]) -> Image.Image | None:
|
||||||
|
if not image_list:
|
||||||
|
return None
|
||||||
|
total_width = sum(image.width for image in image_list)
|
||||||
|
total_width += (len(image_list) - 1) * DELIMITER_WIDTH
|
||||||
|
max_height = max(image.height for image in image_list)
|
||||||
|
combined = Image.new("RGB", (total_width, max_height), color="white")
|
||||||
|
x_offset = 0
|
||||||
|
for index, image in enumerate(image_list):
|
||||||
|
combined.paste(image, (x_offset, 0))
|
||||||
|
x_offset += image.width
|
||||||
|
if index < len(image_list) - 1:
|
||||||
|
delimiter = Image.new(
|
||||||
|
"RGB", (DELIMITER_WIDTH, max_height), color=DELIMITER_COLOR
|
||||||
|
)
|
||||||
|
combined.paste(delimiter, (x_offset, 0))
|
||||||
|
x_offset += DELIMITER_WIDTH
|
||||||
|
return combined
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(maxsize=3)
|
@lru_cache(maxsize=3)
|
||||||
def _get_pdf_pages_cached(filename):
|
def _get_pdf_pages_cached(pdf_path: Path) -> list[Image.Image]:
|
||||||
pdf_path = os.path.join(COPIES_DIR, filename)
|
|
||||||
return convert_from_path(pdf_path)
|
return convert_from_path(pdf_path)
|
||||||
|
|
||||||
def get_pdf_pages(filename):
|
|
||||||
"""Thread-safe wrapper for the cached PDF conversion."""
|
def get_pdf_pages(pdf_path: Path) -> list[Image.Image]:
|
||||||
|
"""Thread-safe wrapper around the small PDF conversion cache."""
|
||||||
with pdf_cache_lock:
|
with pdf_cache_lock:
|
||||||
return _get_pdf_pages_cached(filename)
|
return _get_pdf_pages_cached(pdf_path)
|
||||||
|
|
||||||
def process_single_pdf(filename, shift_offset=0, max_per_file=5):
|
|
||||||
"""
|
def process_single_pdf(
|
||||||
Converts PDF to stitched images.
|
pdf_path: Path,
|
||||||
Returns a tuple: (preview_image_resized, list_of_split_images, schema_dict)
|
shift_offset: int = 0,
|
||||||
"""
|
max_per_file: int = 5,
|
||||||
|
) -> tuple[Image.Image, list[Image.Image], dict[str, object]] | None:
|
||||||
|
"""Convert one PDF into a preview, full-resolution splits and metadata."""
|
||||||
try:
|
try:
|
||||||
pages = get_pdf_pages(filename)
|
|
||||||
cropped_images = []
|
cropped_images = []
|
||||||
|
for image in get_pdf_pages(pdf_path):
|
||||||
for img in pages:
|
width, height = image.size
|
||||||
width, height = img.size
|
|
||||||
|
|
||||||
if max_per_file == 1:
|
if max_per_file == 1:
|
||||||
# If Single Page mode, take the full width (ignore shift/crop)
|
left, right = 0, width
|
||||||
left = 0
|
|
||||||
right = width
|
|
||||||
else:
|
else:
|
||||||
# Original "Cutleft" logic (approx 1/3 width)
|
left = max(0, 100 + shift_offset)
|
||||||
left = 100 + shift_offset
|
right = min(width, width // 3 + 100 + shift_offset)
|
||||||
right = (width // 3) + 100 + shift_offset
|
|
||||||
|
|
||||||
# Ensure crop box is valid
|
|
||||||
left = max(0, left)
|
|
||||||
right = min(width, right)
|
|
||||||
|
|
||||||
if right > left:
|
if right > left:
|
||||||
crop_box = (left, 0, right, height)
|
cropped_images.append(image.crop((left, 0, right, height)))
|
||||||
cropped = img.crop(crop_box)
|
|
||||||
cropped_images.append(cropped)
|
|
||||||
|
|
||||||
if not cropped_images:
|
if not cropped_images:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 1. Generate Schema / Distribution
|
distribution = distribute_pages(len(cropped_images), max_per_file)
|
||||||
col_distribution = distribute_pages(len(cropped_images), max_per_file=max_per_file)
|
|
||||||
|
|
||||||
# 2. Generate Split Images (Full Resolution)
|
|
||||||
split_images = []
|
split_images = []
|
||||||
current_idx = 0
|
current_index = 0
|
||||||
for count in col_distribution:
|
for count in distribution:
|
||||||
chunk = cropped_images[current_idx : current_idx + count]
|
stitched = stitch_images(cropped_images[current_index : current_index + count])
|
||||||
stitched_chunk = stitch_images(chunk)
|
if stitched is not None:
|
||||||
split_images.append(stitched_chunk)
|
split_images.append(stitched)
|
||||||
current_idx += count
|
current_index += count
|
||||||
|
|
||||||
# 3. Generate Preview (All stitched together, Resized)
|
|
||||||
full_stitch = stitch_images(cropped_images)
|
full_stitch = stitch_images(cropped_images)
|
||||||
preview_resized = full_stitch.resize(OUTPUT_SIZE, Image.BILINEAR)
|
if full_stitch is None:
|
||||||
|
return None
|
||||||
schema = {
|
preview = full_stitch.resize(OUTPUT_SIZE, Image.Resampling.BILINEAR)
|
||||||
"original_filename": filename,
|
schema: dict[str, object] = {
|
||||||
|
"original_filename": pdf_path.name,
|
||||||
"total_pages": len(cropped_images),
|
"total_pages": len(cropped_images),
|
||||||
"number_of_files": len(split_images),
|
"number_of_files": len(split_images),
|
||||||
"columns_per_file": col_distribution
|
"columns_per_file": distribution,
|
||||||
}
|
}
|
||||||
|
return preview, split_images, schema
|
||||||
return (preview_resized, split_images, schema)
|
except Exception as exc: # noqa: BLE001 - interactive item failure
|
||||||
|
print(f"Error processing {pdf_path.name}: {exc}")
|
||||||
except Exception as e:
|
|
||||||
print(f"Error processing {filename}: {e}")
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def save_results(result_tuple, filename):
|
|
||||||
"""
|
|
||||||
Saves the split images and the schema JSON.
|
|
||||||
"""
|
|
||||||
_, splits, schema = result_tuple
|
|
||||||
base_name = os.path.splitext(filename)[0]
|
|
||||||
|
|
||||||
# --- Cleanup: Delete existing files for this PDF ---
|
def _previous_cutleft_outputs(output_dir: Path, base_name: str) -> set[str]:
|
||||||
for f in os.listdir(OUTPUT_DIR):
|
if not output_dir.is_dir():
|
||||||
file_path = os.path.join(OUTPUT_DIR, f)
|
return set()
|
||||||
if f == f"{base_name}_schema.json":
|
result = {f"{base_name}_schema.json"}
|
||||||
os.remove(file_path)
|
for path in output_dir.glob(f"{base_name}_*.jpg"):
|
||||||
elif f.startswith(f"{base_name}_") and f.endswith(".jpg"):
|
suffix = path.stem.removeprefix(f"{base_name}_")
|
||||||
suffix = f[len(base_name)+1:-4]
|
|
||||||
if suffix.isdigit():
|
if suffix.isdigit():
|
||||||
os.remove(file_path)
|
result.add(path.name)
|
||||||
|
return result
|
||||||
# Save Images
|
|
||||||
for i, img in enumerate(splits):
|
|
||||||
suffix = f"_{i+1:02d}"
|
|
||||||
output_filename = f"{base_name}{suffix}.jpg"
|
|
||||||
output_path = os.path.join(OUTPUT_DIR, output_filename)
|
|
||||||
img.save(output_path, "JPEG", quality=95)
|
|
||||||
print(f"Saved: {output_filename}")
|
|
||||||
|
|
||||||
# Save Schema
|
|
||||||
json_filename = f"{base_name}_schema.json"
|
|
||||||
json_path = os.path.join(OUTPUT_DIR, json_filename)
|
|
||||||
with open(json_path, 'w') as f:
|
|
||||||
json.dump(schema, f, indent=4)
|
|
||||||
print(f"Saved schema: {json_filename}")
|
|
||||||
|
|
||||||
|
|
||||||
# --- GUI Application ---
|
def save_results(
|
||||||
|
result: tuple[Image.Image, list[Image.Image], dict[str, object]],
|
||||||
|
pdf_path: Path,
|
||||||
|
output_dir: Path,
|
||||||
|
) -> None:
|
||||||
|
"""Atomically replace every Cutleft output associated with one copy."""
|
||||||
|
_, splits, schema = result
|
||||||
|
base_name = pdf_path.stem
|
||||||
|
previous = _previous_cutleft_outputs(output_dir, base_name)
|
||||||
|
with staged_files(output_dir, remove=previous) as staging:
|
||||||
|
for index, image in enumerate(splits, start=1):
|
||||||
|
filename = f"{base_name}_{index:02d}.jpg"
|
||||||
|
image.save(staging / filename, "JPEG", quality=95)
|
||||||
|
atomic_write_json(staging / f"{base_name}_schema.json", schema)
|
||||||
|
for index in range(1, len(splits) + 1):
|
||||||
|
print(f"Saved: {base_name}_{index:02d}.jpg")
|
||||||
|
print(f"Saved schema: {base_name}_schema.json")
|
||||||
|
|
||||||
|
|
||||||
class ImageReviewer:
|
class ImageReviewer:
|
||||||
def __init__(self, file_list, default_max_per_file=5):
|
def __init__(
|
||||||
self.files = file_list
|
self,
|
||||||
|
files: list[Path],
|
||||||
|
output_dir: Path,
|
||||||
|
default_max_per_file: int = 5,
|
||||||
|
) -> None:
|
||||||
|
self.files = files
|
||||||
|
self.output_dir = output_dir
|
||||||
self.index = 0
|
self.index = 0
|
||||||
self.current_shift = 0
|
self.current_shift = 0
|
||||||
self.default_max_per_file = default_max_per_file
|
self.default_max_per_file = default_max_per_file
|
||||||
self.current_max_per_file = default_max_per_file
|
self.current_max_per_file = default_max_per_file
|
||||||
self.current_preview = None # Only stores the resized preview for GUI
|
self.current_preview: Image.Image | None = None
|
||||||
self.is_processing = False
|
self.is_processing = False
|
||||||
|
self.manual_queue: Queue[
|
||||||
|
tuple[Image.Image, list[Image.Image], dict[str, object]] | None
|
||||||
|
] = Queue()
|
||||||
|
|
||||||
# Queue for manual re-processing results
|
|
||||||
self.manual_queue = Queue()
|
|
||||||
|
|
||||||
# Setup GUI
|
|
||||||
self.root = tk.Tk()
|
self.root = tk.Tk()
|
||||||
self.root.title("PDF Cropper")
|
self.root.title("PDF Cropper")
|
||||||
self.root.geometry("+100+100")
|
self.root.geometry("+100+100")
|
||||||
|
|
||||||
self.label_img = tk.Label(self.root)
|
self.label_img = tk.Label(self.root)
|
||||||
self.label_img.pack()
|
self.label_img.pack()
|
||||||
|
|
||||||
self.label_info = tk.Label(self.root, text="", font=("Arial", 12, "bold"))
|
self.label_info = tk.Label(self.root, text="", font=("Arial", 12, "bold"))
|
||||||
self.label_info.pack(pady=5)
|
self.label_info.pack(pady=5)
|
||||||
|
self.root.bind("<Return>", self.on_next)
|
||||||
|
self.root.bind("n", lambda _event: self.on_shift(50))
|
||||||
|
self.root.bind("N", lambda _event: self.on_shift(100))
|
||||||
|
self.root.bind("t", lambda _event: self.on_shift(-50))
|
||||||
|
self.root.bind("1", lambda _event: self.on_set_max_pages(1))
|
||||||
|
|
||||||
# Bindings
|
Thread(target=self.prefetch_worker, daemon=True).start()
|
||||||
self.root.bind('<Return>', self.on_next)
|
|
||||||
self.root.bind('n', lambda e: self.on_shift(50))
|
|
||||||
self.root.bind('N', lambda e: self.on_shift(100))
|
|
||||||
self.root.bind('t', lambda e: self.on_shift(-50))
|
|
||||||
self.root.bind('1', lambda e: self.on_set_max_pages(1))
|
|
||||||
|
|
||||||
# Start background pre-fetcher
|
|
||||||
self.bg_thread = Thread(target=self.prefetch_worker, daemon=True)
|
|
||||||
self.bg_thread.start()
|
|
||||||
|
|
||||||
# Load first image
|
|
||||||
self.load_current_image()
|
self.load_current_image()
|
||||||
|
|
||||||
self.root.lift()
|
self.root.lift()
|
||||||
self.root.focus_force()
|
self.root.focus_force()
|
||||||
self.root.mainloop()
|
self.root.mainloop()
|
||||||
|
|
||||||
def on_set_max_pages(self, count):
|
def on_set_max_pages(self, count: int) -> None:
|
||||||
if self.is_processing:
|
if self.is_processing:
|
||||||
return
|
return
|
||||||
self.current_max_per_file = count
|
self.current_max_per_file = count
|
||||||
print(f"Setting max pages per file: {count}")
|
print(f"Setting max pages per file: {count}")
|
||||||
self.trigger_processing(self.files[self.index], self.current_shift)
|
self.trigger_processing(self.files[self.index], self.current_shift)
|
||||||
|
|
||||||
def prefetch_worker(self):
|
def prefetch_worker(self) -> None:
|
||||||
"""Background thread to load the NEXT file's PDF pages into RAM."""
|
processed_index = -1
|
||||||
idx_to_process = -1
|
|
||||||
while True:
|
while True:
|
||||||
target = self.index + 1
|
target = self.index + 1
|
||||||
if target < len(self.files) and target != idx_to_process:
|
if target < len(self.files) and target != processed_index:
|
||||||
fname = self.files[target]
|
get_pdf_pages(self.files[target])
|
||||||
get_pdf_pages(fname)
|
processed_index = target
|
||||||
idx_to_process = target
|
|
||||||
time.sleep(0.05)
|
time.sleep(0.05)
|
||||||
|
|
||||||
def load_current_image(self, use_prefetch=False):
|
def load_current_image(self) -> None:
|
||||||
if self.index >= len(self.files):
|
if self.index >= len(self.files):
|
||||||
print("All files processed.")
|
print("All files processed.")
|
||||||
self.root.destroy()
|
self.root.destroy()
|
||||||
return
|
return
|
||||||
|
|
||||||
filename = self.files[self.index]
|
|
||||||
self.is_processing = False
|
self.is_processing = False
|
||||||
self.current_shift = 0
|
self.current_shift = 0
|
||||||
|
self.trigger_processing(self.files[self.index], self.current_shift)
|
||||||
|
|
||||||
self.trigger_processing(filename, self.current_shift)
|
def trigger_processing(self, pdf_path: Path, shift: int) -> None:
|
||||||
|
|
||||||
def trigger_processing(self, filename, shift):
|
|
||||||
"""Starts a thread to process image so GUI doesn't freeze."""
|
|
||||||
self.is_processing = True
|
self.is_processing = True
|
||||||
self.label_info.configure(text=f"Processing {filename} (Shift {shift})... Please wait.", fg="red")
|
self.label_info.configure(
|
||||||
|
text=f"Processing {pdf_path.name} (Shift {shift})... Please wait.",
|
||||||
|
fg="red",
|
||||||
|
)
|
||||||
|
|
||||||
def worker():
|
def worker() -> None:
|
||||||
res = process_single_pdf(filename, shift, self.current_max_per_file)
|
self.manual_queue.put(
|
||||||
self.manual_queue.put(res)
|
process_single_pdf(pdf_path, shift, self.current_max_per_file)
|
||||||
|
)
|
||||||
|
|
||||||
Thread(target=worker, daemon=True).start()
|
Thread(target=worker, daemon=True).start()
|
||||||
self.check_manual_queue(filename)
|
self.check_manual_queue(pdf_path)
|
||||||
|
|
||||||
def check_manual_queue(self, filename):
|
def check_manual_queue(self, pdf_path: Path) -> None:
|
||||||
"""Polls the manual queue for result."""
|
|
||||||
try:
|
try:
|
||||||
result = self.manual_queue.get_nowait()
|
result = self.manual_queue.get_nowait()
|
||||||
if result:
|
if result is None:
|
||||||
self.handle_processing_result(result, filename)
|
print(f"Failed to process {pdf_path.name}, skipping.")
|
||||||
else:
|
|
||||||
print(f"Failed to process {filename}, skipping.")
|
|
||||||
self.index += 1
|
self.index += 1
|
||||||
self.load_current_image(use_prefetch=True)
|
self.load_current_image()
|
||||||
|
else:
|
||||||
|
self.handle_processing_result(result, pdf_path)
|
||||||
self.is_processing = False
|
self.is_processing = False
|
||||||
except Empty:
|
except Empty:
|
||||||
self.root.after(100, lambda: self.check_manual_queue(filename))
|
self.root.after(100, lambda: self.check_manual_queue(pdf_path))
|
||||||
|
|
||||||
def handle_processing_result(self, result, filename):
|
def handle_processing_result(
|
||||||
"""Unpacks result, saves files, and updates display."""
|
self,
|
||||||
preview, splits, schema = result
|
result: tuple[Image.Image, list[Image.Image], dict[str, object]],
|
||||||
self.current_preview = preview
|
pdf_path: Path,
|
||||||
|
) -> None:
|
||||||
|
self.current_preview = result[0]
|
||||||
|
save_results(result, pdf_path, self.output_dir)
|
||||||
|
self.update_display(pdf_path.name, result[2])
|
||||||
|
|
||||||
Thread(target=save_results, args=(result, filename), daemon=True).start()
|
def update_display(self, filename: str, schema: dict[str, object]) -> None:
|
||||||
|
if self.current_preview is None:
|
||||||
self.update_display(filename, schema)
|
return
|
||||||
|
|
||||||
def update_display(self, filename, schema=None):
|
|
||||||
if self.current_preview:
|
|
||||||
tk_image = ImageTk.PhotoImage(self.current_preview)
|
tk_image = ImageTk.PhotoImage(self.current_preview)
|
||||||
self.label_img.configure(image=tk_image)
|
self.label_img.configure(image=tk_image)
|
||||||
self.label_img.image = tk_image
|
self.label_img.image = tk_image
|
||||||
|
|
||||||
schema_info = ""
|
|
||||||
if schema:
|
|
||||||
cols = str(schema['columns_per_file'])
|
|
||||||
schema_info = f"\nFiles: {schema['number_of_files']} | Cols: {cols}"
|
|
||||||
|
|
||||||
self.label_info.configure(
|
self.label_info.configure(
|
||||||
text=f"[{self.index+1}/{len(self.files)}] {filename} | Shift: {self.current_shift}px"
|
text=(
|
||||||
f"{schema_info}\n"
|
f"[{self.index + 1}/{len(self.files)}] {filename} | "
|
||||||
f"Enter: Next | n: +50 | N: +100 | t: -50 | 1: use single column",
|
f"Shift: {self.current_shift}px\nFiles: {schema['number_of_files']} | "
|
||||||
fg="black"
|
f"Cols: {schema['columns_per_file']}\n"
|
||||||
|
"Enter: Next | n: +50 | N: +100 | t: -50 | "
|
||||||
|
"1: use single column"
|
||||||
|
),
|
||||||
|
fg="black",
|
||||||
)
|
)
|
||||||
|
|
||||||
def on_shift(self, amount):
|
def on_shift(self, amount: int) -> None:
|
||||||
if self.is_processing:
|
if self.is_processing:
|
||||||
return
|
return
|
||||||
self.current_shift += amount
|
self.current_shift += amount
|
||||||
print(f"Applying shift: {self.current_shift}")
|
print(f"Applying shift: {self.current_shift}")
|
||||||
self.trigger_processing(self.files[self.index], self.current_shift)
|
self.trigger_processing(self.files[self.index], self.current_shift)
|
||||||
|
|
||||||
def on_next(self, event):
|
def on_next(self, _event: object) -> None:
|
||||||
if self.is_processing:
|
if self.is_processing:
|
||||||
return
|
return
|
||||||
self.index += 1
|
self.index += 1
|
||||||
self.current_shift = 0
|
self.current_shift = 0
|
||||||
self.current_max_per_file = self.default_max_per_file
|
self.current_max_per_file = self.default_max_per_file
|
||||||
self.load_current_image(use_prefetch=True)
|
self.load_current_image()
|
||||||
|
|
||||||
|
|
||||||
# --- Entry Point ---
|
def _selected_files(
|
||||||
if __name__ == "__main__":
|
workspace: EvaluationWorkspace,
|
||||||
|
target: Path,
|
||||||
|
) -> list[Path]:
|
||||||
|
workspace.require_directories("Copies")
|
||||||
|
if target.is_file():
|
||||||
|
if target.suffix.casefold() != ".pdf":
|
||||||
|
raise CliError(f"Target is not a PDF: {target}", ExitCode.INVALID_ARGUMENTS)
|
||||||
|
return [target]
|
||||||
|
return sorted(
|
||||||
|
(
|
||||||
|
path
|
||||||
|
for path in workspace.copies_dir.glob("*.pdf")
|
||||||
|
if "nonc" not in path.name.casefold()
|
||||||
|
),
|
||||||
|
key=lambda path: path.name.casefold(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def run(
|
||||||
|
workspace: EvaluationWorkspace,
|
||||||
|
target: Path,
|
||||||
|
*,
|
||||||
|
fullpage: bool = False,
|
||||||
|
) -> ExitCode:
|
||||||
|
files = _selected_files(workspace, target)
|
||||||
if not files:
|
if not files:
|
||||||
print("No PDF files found.")
|
print("No PDF files found.")
|
||||||
else:
|
return ExitCode.SUCCESS
|
||||||
app = ImageReviewer(files, default_max_per_file=1 if fullpage_mode else 5)
|
workspace.cutleft_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
_get_pdf_pages_cached.cache_clear()
|
||||||
|
ImageReviewer(
|
||||||
|
files,
|
||||||
|
workspace.cutleft_dir,
|
||||||
|
default_max_per_file=1 if fullpage else 5,
|
||||||
|
)
|
||||||
|
return ExitCode.SUCCESS
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = target_parser("Interactively crop the label margin from PDF copies")
|
||||||
|
parser.add_argument(
|
||||||
|
"--fullpage",
|
||||||
|
action="store_true",
|
||||||
|
help="Use each complete page instead of cropping the label margin",
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Sequence[str] | None = None) -> int:
|
||||||
|
parser = build_parser()
|
||||||
|
|
||||||
|
def handle(args: argparse.Namespace) -> ExitCode:
|
||||||
|
workspace, target = workspace_from_target(args)
|
||||||
|
return run(workspace, target, fullpage=args.fullpage)
|
||||||
|
|
||||||
|
return execute(parser, argv, handle)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|||||||
+228
-185
@@ -1,223 +1,266 @@
|
|||||||
import fitz # PyMuPDF
|
from __future__ import annotations
|
||||||
from pypdf import PdfWriter
|
|
||||||
from pypdf import PdfReader
|
import argparse
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import json
|
|
||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
import tempfile
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from utils import read_all_labels
|
import fitz
|
||||||
|
from pypdf import PdfReader, PdfWriter
|
||||||
|
|
||||||
carreau = 1000 // 38
|
import utils
|
||||||
|
from copienator import (
|
||||||
|
CliError,
|
||||||
|
EvaluationWorkspace,
|
||||||
|
ExitCode,
|
||||||
|
execute,
|
||||||
|
read_json,
|
||||||
|
target_parser,
|
||||||
|
workspace_from_target,
|
||||||
|
)
|
||||||
|
from copienator.filesystem import staged_directory
|
||||||
|
|
||||||
|
SQUARE = 1000 // 38
|
||||||
|
Coordinate = tuple[str, int, int, int, int, int]
|
||||||
|
ParsedCoordinate = tuple[str, str, int, int, int, int, int]
|
||||||
|
|
||||||
|
|
||||||
def decode_json(pdf_file):
|
def decode_json(pdf_file: str | Path) -> tuple[str, list[Coordinate]]:
|
||||||
file_path = Path(pdf_file)
|
"""Read verified label coordinates associated with one copy PDF."""
|
||||||
with open(file_path.with_suffix(".json"), "r") as f:
|
pdf_path = Path(pdf_file)
|
||||||
json_result = json.load(f)
|
loaded = read_json(pdf_path.with_suffix(".json"))
|
||||||
|
if not isinstance(loaded, dict):
|
||||||
nb_pages = len(PdfReader(file_path).pages)
|
raise TypeError(f"Expected a JSON object for {pdf_path}")
|
||||||
|
boxes = loaded.get("list")
|
||||||
bb_list = json_result["list"]
|
if not isinstance(boxes, list):
|
||||||
name = json_result["name"]
|
raise TypeError(f"Expected a list of labels for {pdf_path}")
|
||||||
column_width = 1000 // nb_pages
|
page_count = len(PdfReader(pdf_path).pages)
|
||||||
|
if page_count == 0:
|
||||||
def page_number(b):
|
raise ValueError(f"PDF contains no pages: {pdf_path}")
|
||||||
return ((b[1] + b[3]) // 2) // column_width
|
column_width = 1000 // page_count
|
||||||
|
result: list[Coordinate] = []
|
||||||
result = []
|
for entry in boxes:
|
||||||
for d in bb_list:
|
if not isinstance(entry, dict):
|
||||||
(b, label) = d["box_2d"], d["label"]
|
raise TypeError(f"Malformed label entry for {pdf_path}: {entry!r}")
|
||||||
pn = page_number(b)
|
box = entry["box_2d"]
|
||||||
result.append((label, pn, b[0] - carreau, b[2]-carreau, b[1], b[3]))
|
label = str(entry["label"])
|
||||||
result.sort(key=lambda x: (x[1], x[2]))
|
page_number = ((box[1] + box[3]) // 2) // column_width
|
||||||
return (name, result)
|
result.append(
|
||||||
|
(label, page_number, box[0] - SQUARE, box[2] - SQUARE, box[1], box[3])
|
||||||
|
)
|
||||||
|
result.sort(key=lambda item: (item[1], item[2]))
|
||||||
|
return str(loaded.get("name", "")), result
|
||||||
|
|
||||||
|
|
||||||
def split_an_interro(base_dir, input_pdf, coords_list):
|
def _parse_coordinates(coords_list: list[Coordinate]) -> list[ParsedCoordinate]:
|
||||||
doc = fitz.open(input_pdf)
|
parsed: list[ParsedCoordinate] = []
|
||||||
|
for label, page, y0, y1, x0, x1 in coords_list:
|
||||||
output_dir = base_dir / "Copies" / input_pdf.stem
|
|
||||||
generated_files = set()
|
|
||||||
parts_by_label = defaultdict(list)
|
|
||||||
|
|
||||||
# 1. Parse labels to strip '|' and determine type: L (Left), R (Right), N (Normal)
|
|
||||||
parsed_coords = []
|
|
||||||
for item in coords_list:
|
|
||||||
label, pn, y0, y1, x0, x1 = item
|
|
||||||
if label.startswith("|"):
|
if label.startswith("|"):
|
||||||
c_type, clean_label = "L", label[1:]
|
kind, clean_label = "L", label[1:]
|
||||||
elif label.endswith("|"):
|
elif label.endswith("|"):
|
||||||
c_type, clean_label = "R", label[:-1]
|
kind, clean_label = "R", label[:-1]
|
||||||
else:
|
else:
|
||||||
c_type, clean_label = "N", label
|
kind, clean_label = "N", label
|
||||||
parsed_coords.append((clean_label, c_type, pn, y0, y1, x0, x1))
|
parsed.append((clean_label, kind, page, y0, y1, x0, x1))
|
||||||
|
filtered: list[ParsedCoordinate] = []
|
||||||
|
for item in parsed:
|
||||||
|
if not filtered or item[0] != filtered[-1][0]:
|
||||||
|
filtered.append(item)
|
||||||
|
return filtered
|
||||||
|
|
||||||
# 2. Filter consecutive duplicate labels based on the cleaned name
|
|
||||||
filtered_coords = []
|
|
||||||
if parsed_coords:
|
|
||||||
filtered_coords.append(parsed_coords[0])
|
|
||||||
for item in parsed_coords[1:]:
|
|
||||||
if item[0] != filtered_coords[-1][0]:
|
|
||||||
filtered_coords.append(item)
|
|
||||||
coords_list = filtered_coords
|
|
||||||
|
|
||||||
def scale_coord(y, page):
|
def _save_cropped_page(
|
||||||
"""Scale y from 0–1000 range to PDF points."""
|
document: fitz.Document,
|
||||||
page_height = page.rect.height
|
page_number: int,
|
||||||
return (y / 1000) * page_height
|
x0: float,
|
||||||
|
y0: float,
|
||||||
def save_cropped_page(doc, page_num, x0, y0, x1, y1, out_path):
|
x1: float,
|
||||||
"""Saves a cropped portion of a page as a new PDF."""
|
y1: float,
|
||||||
page = doc[page_num]
|
output_path: Path,
|
||||||
rotated_rect = page.rect * page.transformation_matrix
|
) -> None:
|
||||||
visual_crop_rect = fitz.Rect(rotated_rect.x0 + x0, y0, rotated_rect.x0 + x1, y1)
|
page = document[page_number]
|
||||||
unrotated_clip_rect = visual_crop_rect * page.derotation_matrix
|
rotated_rectangle = page.rect * page.transformation_matrix
|
||||||
|
visual_crop = fitz.Rect(
|
||||||
temp_doc = fitz.open()
|
rotated_rectangle.x0 + x0,
|
||||||
temp_page = temp_doc.new_page(
|
y0,
|
||||||
width=visual_crop_rect.width,
|
rotated_rectangle.x0 + x1,
|
||||||
height=visual_crop_rect.height
|
y1,
|
||||||
)
|
)
|
||||||
temp_page.show_pdf_page(
|
unrotated_clip = visual_crop * page.derotation_matrix
|
||||||
temp_page.rect,
|
cropped = fitz.open()
|
||||||
doc,
|
try:
|
||||||
page_num,
|
target_page = cropped.new_page(width=visual_crop.width, height=visual_crop.height)
|
||||||
|
target_page.show_pdf_page(
|
||||||
|
target_page.rect,
|
||||||
|
document,
|
||||||
|
page_number,
|
||||||
rotate=-page.rotation,
|
rotate=-page.rotation,
|
||||||
clip=unrotated_clip_rect
|
clip=unrotated_clip,
|
||||||
)
|
)
|
||||||
temp_doc.save(out_path)
|
cropped.save(output_path)
|
||||||
temp_doc.close()
|
finally:
|
||||||
|
cropped.close()
|
||||||
|
|
||||||
# Iterate through all labels
|
|
||||||
for idx, (clean_label, c_type, start_page, y_start_raw, y_end_box, x0_raw, x1_raw) in enumerate(coords_list):
|
def _render_split_outputs(
|
||||||
|
input_pdf: Path,
|
||||||
|
coords_list: list[Coordinate],
|
||||||
|
staging: Path,
|
||||||
|
) -> set[str]:
|
||||||
|
"""Render every current answer into an otherwise empty staging directory."""
|
||||||
|
document = fitz.open(input_pdf)
|
||||||
|
try:
|
||||||
|
parsed = _parse_coordinates(coords_list)
|
||||||
|
parts_by_label: defaultdict[str, list[Path]] = defaultdict(list)
|
||||||
|
with tempfile.TemporaryDirectory(prefix="copienator-split-") as temp_directory:
|
||||||
|
temporary = Path(temp_directory)
|
||||||
|
for index, item in enumerate(parsed):
|
||||||
|
clean_label, kind, start_page, y_start, _y_end, x0_raw, _x1_raw = item
|
||||||
if clean_label == "_":
|
if clean_label == "_":
|
||||||
continue
|
continue
|
||||||
|
if not 0 <= start_page < document.page_count:
|
||||||
temp_parts = []
|
raise ValueError(
|
||||||
end_page = doc.page_count - 1
|
f"Invalid page {start_page} for {input_pdf.name}"
|
||||||
end_y_target_raw = 1000
|
)
|
||||||
|
end_page = document.page_count - 1
|
||||||
# RULE 2: Determine stopping label
|
end_y = 1000
|
||||||
for next_item in coords_list[idx + 1:]:
|
for next_item in parsed[index + 1 :]:
|
||||||
n_clean, n_type, n_pn, n_y_start, n_y_end, _, _ = next_item
|
_next_label, next_kind, next_page, next_y, *_rest = next_item
|
||||||
|
if (
|
||||||
if c_type == "L":
|
(kind == "L" and next_kind in {"L", "N"})
|
||||||
is_stop = (n_type in ("L", "N"))
|
or (kind == "R" and next_kind in {"R", "N"})
|
||||||
elif c_type == "R":
|
or kind == "N"
|
||||||
is_stop = (n_type in ("R", "N"))
|
):
|
||||||
else:
|
end_page = next_page
|
||||||
is_stop = True # Normal labels stop at anything
|
end_y = min(next_y + int(1.5 * SQUARE), 1000)
|
||||||
|
|
||||||
if is_stop:
|
|
||||||
end_page = n_pn
|
|
||||||
# end_y_target_raw = n_y_start
|
|
||||||
# On avait retiré un carreau précédemment inutilement, on le rajoute, plus un demi carreau
|
|
||||||
end_y_target_raw = min(n_y_start + int(1.5 * carreau), 1000)
|
|
||||||
break
|
break
|
||||||
|
|
||||||
# RULES 3 & 4: Calculate horizontal boundaries (0.0 to 1.0 fraction of local page width)
|
column_width = 1000 / document.page_count
|
||||||
col_w = 1000 / doc.page_count
|
if kind == "L":
|
||||||
if c_type == "L": # |name
|
fraction_x0 = (x0_raw % column_width) / column_width
|
||||||
fraction_x0 = (x0_raw % col_w) / col_w
|
|
||||||
fraction_x1 = 1.0
|
fraction_x1 = 1.0
|
||||||
end_y_target_raw = min(1000, end_y_target_raw + 40)
|
end_y = min(1000, end_y + 40)
|
||||||
elif c_type == "R": # name|
|
elif kind == "R":
|
||||||
fraction_x0 = 0.0
|
fraction_x0 = 0.0
|
||||||
# Find the closest 'L' label in y-distance
|
left_labels = [entry for entry in parsed if entry[1] == "L"]
|
||||||
L_labels = [it for it in parsed_coords if it[1] == "L"]
|
if left_labels:
|
||||||
if L_labels:
|
closest = min(left_labels, key=lambda entry: abs(entry[3] - y_start))
|
||||||
closest_L = min(L_labels, key=lambda it: abs(it[3] - y_start_raw))
|
center = (closest[5] + closest[6]) / 2.0
|
||||||
closest_L_x_center = (closest_L[5] + closest_L[6]) / 2.0
|
fraction_x1 = (center % column_width) / column_width
|
||||||
fraction_x1 = (closest_L_x_center % col_w) / col_w
|
if fraction_x1 <= fraction_x0:
|
||||||
if fraction_x1 <= fraction_x0: fraction_x1 = 1.0 # Fallback
|
fraction_x1 = 1.0
|
||||||
else:
|
else:
|
||||||
fraction_x1 = 1.0
|
fraction_x1 = 1.0
|
||||||
else: # Normal
|
else:
|
||||||
fraction_x0 = 0.0
|
fraction_x0, fraction_x1 = 0.0, 1.0
|
||||||
fraction_x1 = 1.0
|
|
||||||
|
|
||||||
current_p = start_page
|
for page_number in range(start_page, end_page + 1):
|
||||||
while current_p <= end_page:
|
page = document[page_number]
|
||||||
page = doc[current_p]
|
y0 = (y_start / 1000) * page.rect.height if page_number == start_page else 0
|
||||||
|
y1 = (end_y / 1000) * page.rect.height if page_number == end_page else page.rect.height
|
||||||
|
if y1 <= y0 + 1:
|
||||||
|
continue
|
||||||
|
part_path = temporary / f"part-{index}-{page_number}.pdf"
|
||||||
|
_save_cropped_page(
|
||||||
|
document,
|
||||||
|
page_number,
|
||||||
|
fraction_x0 * page.rect.width,
|
||||||
|
y0,
|
||||||
|
fraction_x1 * page.rect.width,
|
||||||
|
y1,
|
||||||
|
part_path,
|
||||||
|
)
|
||||||
|
parts_by_label[clean_label].append(part_path)
|
||||||
|
|
||||||
y0 = scale_coord(y_start_raw, page) if current_p == start_page else 0
|
generated: set[str] = set()
|
||||||
y1 = scale_coord(end_y_target_raw, page) if current_p == end_page else page.rect.height
|
for label, parts in parts_by_label.items():
|
||||||
|
filename = f"{label}.pdf"
|
||||||
if y1 > y0 + 1:
|
|
||||||
# Convert fractions to absolute PDF points
|
|
||||||
x0_pdf = fraction_x0 * page.rect.width
|
|
||||||
x1_pdf = fraction_x1 * page.rect.width
|
|
||||||
|
|
||||||
temp_path = f"_part_{idx}_{current_p}.pdf"
|
|
||||||
save_cropped_page(doc, current_p, x0_pdf, y0, x1_pdf, y1, temp_path)
|
|
||||||
temp_parts.append(temp_path)
|
|
||||||
|
|
||||||
current_p += 1
|
|
||||||
|
|
||||||
parts_by_label[clean_label].extend(temp_parts)
|
|
||||||
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
# Process aggregated parts by label
|
|
||||||
for title, parts in parts_by_label.items():
|
|
||||||
merger = PdfWriter()
|
merger = PdfWriter()
|
||||||
|
try:
|
||||||
for part in parts:
|
for part in parts:
|
||||||
if os.path.exists(part):
|
|
||||||
merger.append(part)
|
merger.append(part)
|
||||||
|
merger.write(staging / filename)
|
||||||
filename = f"{title}.pdf"
|
finally:
|
||||||
merger.write(output_dir / filename)
|
|
||||||
merger.close()
|
merger.close()
|
||||||
generated_files.add(filename)
|
generated.add(filename)
|
||||||
|
return generated
|
||||||
|
finally:
|
||||||
|
document.close()
|
||||||
|
|
||||||
# Cleanup
|
|
||||||
for part in parts:
|
|
||||||
if os.path.exists(part):
|
|
||||||
os.remove(part)
|
|
||||||
|
|
||||||
doc.close()
|
def _preserve_previous_outputs(
|
||||||
|
output_dir: Path,
|
||||||
# Move files not generated in this run to 'Missing' folder
|
staging: Path,
|
||||||
if output_dir.exists():
|
generated_files: set[str],
|
||||||
missing_dir = output_dir / "Missing"
|
) -> None:
|
||||||
for item in output_dir.iterdir():
|
if not output_dir.is_dir():
|
||||||
if item.is_file() and item.name not in generated_files:
|
return
|
||||||
|
for directory in (path for path in output_dir.iterdir() if path.is_dir()):
|
||||||
|
shutil.copytree(directory, staging / directory.name, dirs_exist_ok=True)
|
||||||
|
missing_dir = staging / "Missing"
|
||||||
|
for item in (path for path in output_dir.iterdir() if path.is_file()):
|
||||||
|
if item.name in generated_files:
|
||||||
|
continue
|
||||||
print(f"ALERT: File '{item.name}' not generated. Moving to {missing_dir}")
|
print(f"ALERT: File '{item.name}' not generated. Moving to {missing_dir}")
|
||||||
missing_dir.mkdir(exist_ok=True)
|
missing_dir.mkdir(exist_ok=True)
|
||||||
item.rename(missing_dir / item.name)
|
shutil.copy2(item, missing_dir / item.name)
|
||||||
|
|
||||||
|
|
||||||
|
def split_an_interro(
|
||||||
|
workspace: EvaluationWorkspace,
|
||||||
|
input_pdf: Path,
|
||||||
|
coords_list: list[Coordinate],
|
||||||
|
) -> None:
|
||||||
|
"""Regenerate one copy's answers and preserve obsolete ones under Missing."""
|
||||||
|
output_dir = workspace.copies_dir / input_pdf.stem
|
||||||
|
with staged_directory(output_dir) as staging:
|
||||||
|
generated = _render_split_outputs(input_pdf, coords_list, staging)
|
||||||
|
_preserve_previous_outputs(output_dir, staging, generated)
|
||||||
|
|
||||||
|
|
||||||
|
def _selected_pdfs(workspace: EvaluationWorkspace, target: Path) -> list[Path]:
|
||||||
|
workspace.require_directories("Copies")
|
||||||
|
if target.is_file():
|
||||||
|
if target.suffix.casefold() != ".pdf":
|
||||||
|
raise CliError(f"Target is not a PDF: {target}", ExitCode.INVALID_ARGUMENTS)
|
||||||
|
return [target]
|
||||||
|
return sorted(workspace.copies_dir.glob("*.pdf"), key=lambda path: path.name.casefold())
|
||||||
|
|
||||||
|
|
||||||
|
def run(workspace: EvaluationWorkspace, target: Path) -> ExitCode:
|
||||||
|
workspace.require_files("labels")
|
||||||
|
utils.read_all_labels(workspace.root)
|
||||||
|
pdf_files = _selected_pdfs(workspace, target)
|
||||||
|
status = ExitCode.SUCCESS
|
||||||
|
for pdf_path in pdf_files:
|
||||||
|
json_path = pdf_path.with_suffix(".json")
|
||||||
|
if not json_path.is_file():
|
||||||
|
print(f"Warning: No JSON found for {pdf_path.name}")
|
||||||
|
status = ExitCode.PARTIAL
|
||||||
|
continue
|
||||||
|
name, coordinates = decode_json(pdf_path)
|
||||||
|
print(f"Decoded name: {name}")
|
||||||
|
split_an_interro(workspace, pdf_path, coordinates)
|
||||||
|
if not pdf_files:
|
||||||
|
print("No PDF copies found.")
|
||||||
|
return status
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
return target_parser("Split verified PDF copies into answers by label")
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Sequence[str] | None = None) -> int:
|
||||||
|
parser = build_parser()
|
||||||
|
|
||||||
|
def handle(args: argparse.Namespace) -> ExitCode:
|
||||||
|
workspace, target = workspace_from_target(args)
|
||||||
|
return run(workspace, target)
|
||||||
|
|
||||||
|
return execute(parser, argv, handle)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
if len(sys.argv) < 2:
|
raise SystemExit(main())
|
||||||
print("Usage: python script.py <directory or pdf_file>")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
input_arg = Path(sys.argv[1])
|
|
||||||
|
|
||||||
if input_arg.is_file():
|
|
||||||
base_dir = input_arg.parent
|
|
||||||
if base_dir.name == "Copies":
|
|
||||||
base_dir = base_dir.parent
|
|
||||||
pdf_files = [input_arg]
|
|
||||||
elif input_arg.is_dir():
|
|
||||||
base_dir = input_arg
|
|
||||||
copies_dir = base_dir / "Copies"
|
|
||||||
pdf_files = sorted(copies_dir.glob("*.pdf"))
|
|
||||||
else:
|
|
||||||
print(f"Error: {input_arg} is not a valid file or directory.")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
read_all_labels(base_dir)
|
|
||||||
|
|
||||||
for pdf_path in pdf_files:
|
|
||||||
json_path = pdf_path.with_suffix(".json")
|
|
||||||
# print("Debug :", json_path)
|
|
||||||
if json_path.exists():
|
|
||||||
(name, coords) = decode_json(pdf_path)
|
|
||||||
print("Decoded name : ", name)
|
|
||||||
split_an_interro(base_dir, pdf_path, coords)
|
|
||||||
else:
|
|
||||||
print(f"Warning: No JSON found for {pdf_path.name}")
|
|
||||||
|
|||||||
@@ -302,6 +302,10 @@ class StandardCliTests(unittest.TestCase):
|
|||||||
"reading_grouped_annotations": load_script_module(
|
"reading_grouped_annotations": load_script_module(
|
||||||
"reading_grouped_annotations.py", "reading_grouped_annotations"
|
"reading_grouped_annotations.py", "reading_grouped_annotations"
|
||||||
),
|
),
|
||||||
|
"cutleft": load_script_module("cutleft.py", "cutleft"),
|
||||||
|
"splitting_int": load_script_module(
|
||||||
|
"splitting_int.py", "splitting_int"
|
||||||
|
),
|
||||||
"copies_tools": load_script_module(
|
"copies_tools": load_script_module(
|
||||||
"copies_tools.py", "copienator_copies_tools_test"
|
"copies_tools.py", "copienator_copies_tools_test"
|
||||||
),
|
),
|
||||||
@@ -339,6 +343,8 @@ class StandardCliTests(unittest.TestCase):
|
|||||||
"annotating_by_label": [missing],
|
"annotating_by_label": [missing],
|
||||||
"reading_annotations": [missing],
|
"reading_annotations": [missing],
|
||||||
"reading_grouped_annotations": [missing],
|
"reading_grouped_annotations": [missing],
|
||||||
|
"cutleft": [missing],
|
||||||
|
"splitting_int": [missing],
|
||||||
}
|
}
|
||||||
for name, arguments in invocations.items():
|
for name, arguments in invocations.items():
|
||||||
with self.subTest(script=name), redirect_stderr(io.StringIO()):
|
with self.subTest(script=name), redirect_stderr(io.StringIO()):
|
||||||
@@ -431,6 +437,16 @@ class StandardCliTests(unittest.TestCase):
|
|||||||
"grouped",
|
"grouped",
|
||||||
{"target": evaluation, "update_score": True, "refaire": True},
|
{"target": evaluation, "update_score": True, "refaire": True},
|
||||||
),
|
),
|
||||||
|
"cutleft": (
|
||||||
|
"cutleft",
|
||||||
|
"default",
|
||||||
|
{"target": evaluation, "fullpage": True},
|
||||||
|
),
|
||||||
|
"splitting_int": (
|
||||||
|
"splitting",
|
||||||
|
"default",
|
||||||
|
{"target": evaluation},
|
||||||
|
),
|
||||||
}
|
}
|
||||||
for module_name, (step_id, variant_id, values) in cases.items():
|
for module_name, (step_id, variant_id, values) in cases.items():
|
||||||
step = steps[step_id]
|
step = steps[step_id]
|
||||||
@@ -513,6 +529,131 @@ class StandardCliTests(unittest.TestCase):
|
|||||||
self.assertEqual(module.main([str(evaluation)]), 0)
|
self.assertEqual(module.main([str(evaluation)]), 0)
|
||||||
self.assertTrue((evaluation / "Par label").is_dir())
|
self.assertTrue((evaluation / "Par label").is_dir())
|
||||||
|
|
||||||
|
def test_cutleft_can_target_one_copy_without_rendering_at_import(self) -> None:
|
||||||
|
module = self.modules["cutleft"]
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
evaluation = Path(directory) / "Exam"
|
||||||
|
copy_pdf = evaluation / "Copies" / "Copie01.pdf"
|
||||||
|
copy_pdf.parent.mkdir(parents=True)
|
||||||
|
copy_pdf.write_bytes(b"pdf")
|
||||||
|
with patch.object(module, "ImageReviewer") as reviewer:
|
||||||
|
self.assertEqual(module.main([str(copy_pdf), "--fullpage"]), 0)
|
||||||
|
files, output_dir = reviewer.call_args.args[:2]
|
||||||
|
self.assertEqual(files, [copy_pdf])
|
||||||
|
self.assertEqual(output_dir, evaluation / "Cutleft")
|
||||||
|
self.assertEqual(reviewer.call_args.kwargs["default_max_per_file"], 1)
|
||||||
|
|
||||||
|
def test_cutleft_atomic_save_removes_only_obsolete_copy_outputs(self) -> None:
|
||||||
|
module = self.modules["cutleft"]
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
output_dir = Path(directory) / "Cutleft"
|
||||||
|
output_dir.mkdir()
|
||||||
|
(output_dir / "Copie01_01.jpg").write_bytes(b"old-one")
|
||||||
|
(output_dir / "Copie01_02.jpg").write_bytes(b"old-two")
|
||||||
|
(output_dir / "Copie02_01.jpg").write_bytes(b"other-copy")
|
||||||
|
(output_dir / "Copie01_schema.json").write_text(
|
||||||
|
"{}", encoding="utf-8"
|
||||||
|
)
|
||||||
|
image = Image.new("RGB", (5, 5), "white")
|
||||||
|
result = (
|
||||||
|
image,
|
||||||
|
[image],
|
||||||
|
{
|
||||||
|
"original_filename": "Copie01.pdf",
|
||||||
|
"total_pages": 1,
|
||||||
|
"number_of_files": 1,
|
||||||
|
"columns_per_file": [1],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
module.save_results(result, Path("Copie01.pdf"), output_dir)
|
||||||
|
self.assertTrue((output_dir / "Copie01_01.jpg").is_file())
|
||||||
|
self.assertFalse((output_dir / "Copie01_02.jpg").exists())
|
||||||
|
self.assertEqual(
|
||||||
|
(output_dir / "Copie02_01.jpg").read_bytes(), b"other-copy"
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
read_json(output_dir / "Copie01_schema.json")["total_pages"], 1
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_splitting_missing_json_is_partial(self) -> None:
|
||||||
|
module = self.modules["splitting_int"]
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
evaluation = Path(directory) / "Exam"
|
||||||
|
copy_pdf = evaluation / "Copies" / "Copie01.pdf"
|
||||||
|
copy_pdf.parent.mkdir(parents=True)
|
||||||
|
copy_pdf.write_bytes(b"pdf")
|
||||||
|
(evaluation / "labels").write_text("Ex 1\n", encoding="utf-8")
|
||||||
|
self.assertEqual(module.main([str(evaluation)]), 4)
|
||||||
|
|
||||||
|
def test_splitting_failure_preserves_previous_copy_outputs(self) -> None:
|
||||||
|
module = self.modules["splitting_int"]
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
evaluation = Path(directory) / "Exam"
|
||||||
|
copy_pdf = evaluation / "Copies" / "Copie01.pdf"
|
||||||
|
output_dir = evaluation / "Copies" / "Copie01"
|
||||||
|
output_dir.mkdir(parents=True)
|
||||||
|
copy_pdf.write_bytes(b"pdf")
|
||||||
|
(output_dir / "sentinel.pdf").write_bytes(b"old")
|
||||||
|
workspace = EvaluationWorkspace(evaluation)
|
||||||
|
with patch.object(
|
||||||
|
module,
|
||||||
|
"_render_split_outputs",
|
||||||
|
side_effect=RuntimeError("render failed"),
|
||||||
|
), self.assertRaises(RuntimeError):
|
||||||
|
module.split_an_interro(workspace, copy_pdf, [])
|
||||||
|
self.assertEqual((output_dir / "sentinel.pdf").read_bytes(), b"old")
|
||||||
|
|
||||||
|
def test_splitting_moves_obsolete_outputs_to_missing_on_commit(self) -> None:
|
||||||
|
module = self.modules["splitting_int"]
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
evaluation = Path(directory) / "Exam"
|
||||||
|
copy_pdf = evaluation / "Copies" / "Copie01.pdf"
|
||||||
|
output_dir = evaluation / "Copies" / "Copie01"
|
||||||
|
missing_dir = output_dir / "Missing"
|
||||||
|
missing_dir.mkdir(parents=True)
|
||||||
|
copy_pdf.write_bytes(b"pdf")
|
||||||
|
(output_dir / "Old.pdf").write_bytes(b"obsolete")
|
||||||
|
(missing_dir / "Earlier.pdf").write_bytes(b"earlier")
|
||||||
|
|
||||||
|
def render(_pdf, _coordinates, staging):
|
||||||
|
(staging / "Ex 1.pdf").write_bytes(b"new")
|
||||||
|
return {"Ex 1.pdf"}
|
||||||
|
|
||||||
|
with patch.object(module, "_render_split_outputs", side_effect=render):
|
||||||
|
module.split_an_interro(EvaluationWorkspace(evaluation), copy_pdf, [])
|
||||||
|
self.assertEqual((output_dir / "Ex 1.pdf").read_bytes(), b"new")
|
||||||
|
self.assertEqual(
|
||||||
|
(output_dir / "Missing" / "Old.pdf").read_bytes(), b"obsolete"
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
(output_dir / "Missing" / "Earlier.pdf").read_bytes(), b"earlier"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_splitting_renders_a_real_one_page_answer(self) -> None:
|
||||||
|
module = self.modules["splitting_int"]
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
evaluation = Path(directory) / "Exam"
|
||||||
|
copy_pdf = evaluation / "Copies" / "Copie01.pdf"
|
||||||
|
copy_pdf.parent.mkdir(parents=True)
|
||||||
|
document = module.fitz.open()
|
||||||
|
page = document.new_page(width=600, height=800)
|
||||||
|
page.insert_text((100, 200), "Student answer")
|
||||||
|
document.save(copy_pdf)
|
||||||
|
document.close()
|
||||||
|
(evaluation / "labels").write_text("Ex 1\n", encoding="utf-8")
|
||||||
|
atomic_write_json(
|
||||||
|
copy_pdf.with_suffix(".json"),
|
||||||
|
{
|
||||||
|
"name": "Copie01",
|
||||||
|
"list": [{"label": "Ex 1", "box_2d": [100, 100, 300, 300]}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(module.main([str(evaluation)]), 0)
|
||||||
|
answer = evaluation / "Copies" / "Copie01" / "Ex 1.pdf"
|
||||||
|
self.assertTrue(answer.is_file())
|
||||||
|
self.assertEqual(len(PdfReader(answer).pages), 1)
|
||||||
|
|
||||||
def test_post_correction_main_cleans_json_atomically(self) -> None:
|
def test_post_correction_main_cleans_json_atomically(self) -> None:
|
||||||
module = self.modules["post_correction"]
|
module = self.modules["post_correction"]
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
|||||||
Reference in New Issue
Block a user