From 78f3e4b2f0e462cfb76a74fe3c3923e1ec68ee36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Miquel?= Date: Tue, 3 Nov 2026 22:08:36 +0100 Subject: [PATCH] Automatic page splitting --- copienator/auto_page_split.py | 377 ++++++++++++++++++ .../commands/automatic_page_splitter.py | 183 +++++++++ copienator/commands/page_splitter.py | 123 ++++-- copienator/copy_errors.py | 6 +- copienator/fast_page_split.py | 276 +++++++++++++ copienator/ink_detection.py | 197 ++++++++- copienator/paper_ruling.py | 15 +- copienator_gui/workflow.py | 14 +- docs/auto_page_split.md | 241 +++++++++++ tests/test_auto_page_split.py | 167 ++++++++ tests/test_automatic_page_splitter.py | 204 ++++++++++ tests/test_fast_page_split.py | 88 ++++ tests/test_ink_detection.py | 108 ++++- 13 files changed, 1959 insertions(+), 40 deletions(-) create mode 100644 copienator/auto_page_split.py create mode 100644 copienator/commands/automatic_page_splitter.py create mode 100644 copienator/fast_page_split.py create mode 100644 docs/auto_page_split.md create mode 100644 tests/test_auto_page_split.py create mode 100644 tests/test_automatic_page_splitter.py create mode 100644 tests/test_fast_page_split.py diff --git a/copienator/auto_page_split.py b/copienator/auto_page_split.py new file mode 100644 index 0000000..0e18317 --- /dev/null +++ b/copienator/auto_page_split.py @@ -0,0 +1,377 @@ +"""Experimental, non-destructive spread splitting with an HTML review gallery. + +python -m copienator.auto_page_split INPUT OUTPUT [--workers 4] +No booklet reordering, secondary cuts, or GUI changes. Unknown formats and +uncertain content are retained. OUTPUT must not already exist. +""" +from __future__ import annotations + +import argparse +from concurrent.futures import ProcessPoolExecutor +import hashlib +import html +import json +import multiprocessing +from pathlib import Path +from urllib.parse import quote + +import cv2 +import numpy as np +import pymupdf +from PIL import Image, ImageDraw + +from copienator.ink_detection import detect_bounds, detect_page_content + + +def classify_format(width: float, height: float) -> str: + """Dimensions are displayed PDF points; tolerate 5% scan size error.""" + w, h = width * 25.4 / 72, height * 25.4 / 72 + if abs(min(w, h) / 210 - 1) < .05 and abs(max(w, h) / 297 - 1) < .05: + return 'a4' + if abs(w / 420 - 1) < .05 and abs(h / 297 - 1) < .05: + return 'a3' + return 'unknown' + + +def analyse_ink(rgb: np.ndarray, dpi: float, *, spread: bool = False) -> tuple[np.ndarray, dict]: + """Use normal rognage on logical pages; colour cleanup only for overflow. + + Blankness uses the shared crop detector's content-analysis entry point. + No independent weak-pixel veto or secondary artifact detector is applied. + """ + if not spread: + return detect_page_content(rgb,dpi) + masks = {} + result = detect_bounds(rgb, dpi, content_masks=masks, + colour_cleanup=True, edge_mark_cleanup=False) + ink = masks['foreground'] + # For ownership, follow connected faint pen strokes beyond rognage's local + # 2 mm recovery radius. Do not restore page-wide connected ruling networks. + candidate = ink | masks['weak_candidates'] + n,labels,stats,_ = cv2.connectedComponentsWithStats(candidate.astype(np.uint8),8) + seeded = np.zeros(n,bool) + seeded[np.unique(labels[ink])] = True + seeded[0] = False + giant = (stats[:,2] > .65*ink.shape[1]) & (stats[:,3] > .4*ink.shape[0]) + seeded[giant] = False + ink = ink | seeded[labels] + return ink, result + + +def _analyse_page_full(rgb: np.ndarray, dpi: float, kind: str) -> dict: + """Keep spread geometry analysis separate from logical-page blankness.""" + h, w = rgb.shape[:2] + boundaries = [0, w//2, w] if kind == 'a3' else [0, w] + content = np.zeros((h,w), bool) + evidence = [] + for x0,x1 in zip(boundaries,boundaries[1:]): + foreground, result = analyse_ink(rgb[:,x0:x1],dpi) + content[:,x0:x1] = foreground + evidence.append(result) + ink = analyse_ink(rgb,dpi,spread=True)[0] if kind == 'a3' else content + plan = plan_masks(ink,dpi,kind,content_ink=content,rgb=rgb) + for output,result in zip(plan['outputs'],evidence): + output['crop_analysis'] = result + return plan + + +def analyse_page(rgb: np.ndarray, dpi: float, kind: str) -> dict: + """Analyse a raster with the standard coarse-to-fine splitter.""" + from copienator.fast_page_split import analyse_page_fast + return analyse_page_fast(rgb, dpi, kind) + + +def _fold_regions(ink: np.ndarray, credible: np.ndarray, dpi: float, + rgb: np.ndarray | None = None) -> tuple[list[dict], np.ndarray]: + """Grow local word groups towards page anchors, stopping at supported text. + + Broader gaps are allowed for ownership evidence without merging both pages + into one giant line or changing output bounds to include the anchor word. + """ + import heapq + px = dpi/25.4 + h,w = ink.shape + middle = w//2 + ignored = [] + if rgb is not None: + # Remove weak thin ruling *before* word grouping: otherwise a nearby + # dark word can lend its colour/anchor evidence to a pale grid fragment. + count,raw_labels,raw_stats,_ = cv2.connectedComponentsWithStats(ink.astype(np.uint8),8) + ink = ink.copy() + for i,(x,y,bw,bh,area) in enumerate(raw_stats[1:],1): + if not (x < middle+2*px and x+bw > middle-2*px and bh4*bh): + continue + region = raw_labels[y:y+bh,x:x+bw]==i + values = rgb[y:y+bh,x:x+bw][region].astype(np.int16) + if np.median(np.ptp(values,axis=1))<=80 and np.median(values.min(axis=1))>=110: + ink[y:y+bh,x:x+bw][region] = False + ignored.append(dict(owner='artifact',box=[int(x),int(y),int(x+bw),int(y+bh)], + component_id=0,anchor_boxes=[None,None],anchor_distance_mm=[None,None],reason='weak-thin-ruling')) + kernel = np.ones((max(1,round(.5*px)),max(1,round(3*px))),np.uint8) + joined = cv2.dilate(ink.astype(np.uint8),kernel) + n,labels,stats,_ = cv2.connectedComponentsWithStats(joined,8) + boxes = np.zeros((n,4),np.int32) + areas = np.bincount(labels[ink],minlength=n) + pen = np.zeros(n,bool) + for i in range(1,n): + x,y,bw,bh,_ = stats[i] + yy,xx = np.nonzero((labels[y:y+bh,x:x+bw]==i) & ink[y:y+bh,x:x+bw]) + boxes[i] = [x+xx.min(),y+yy.min(),x+xx.max()+1,y+yy.max()+1] + if rgb is not None and len(xx)>=.4*px*px: + values=rgb[y+yy,x+xx].astype(np.int16) + # Between the crop detector's weak (60) and strong (100) colour + # thresholds: pale blue ruling must not become an ownership anchor. + pen[i] = np.median(np.ptp(values,axis=1))>80 or np.median(values.min(axis=1))<110 + support = credible & ink + # The blank classifier deliberately ignores short isolated pen marks. + # A two-dimensional letter/terminal stroke can still link to its page here. + support |= pen[labels] & ink + left_limit,right_limit = max(0,round(middle-12*px)),min(w,round(middle+12*px)) + left_counts = np.bincount(labels[:,:left_limit][support[:,:left_limit]],minlength=n) + right_counts = np.bincount(labels[:,right_limit:][support[:,right_limit:]],minlength=n) + anchors = np.c_[left_counts>=max(2,.2*px*px),right_counts>=max(2,.2*px*px)] + backed = np.bincount(labels[support],minlength=n)>=max(2,.12*px*px) + backed[0] = False + near = max(1,round(2*px)) + roots = np.unique(labels[:,max(0,middle-near):middle+near]) + regions = ignored + for root in roots: + if root == 0:continue + x0,y0,x1,y1 = boxes[root] + if x1 < middle-near or x0 > middle+near:continue + thin_unanchored = (not anchors[root].any() and + ((not pen[root] and y1-y04*(y1-y0)) or areas[root]<.4*px*px)) + if thin_unanchored: + regions.append(dict(owner='artifact',box=boxes[root].tolist(),component_id=int(root),anchor_boxes=[None,None],anchor_distance_mm=[None,None])) + continue + distances = np.full(n,np.inf) + distances[root]=0 + queue=[(0.,int(root))] + reached=[np.inf,np.inf] + evidence=[None,None] + # Only supported writing may carry a connection onwards. Otherwise + # rows of grid fragments could form a bridge all the way into an answer. + allowed = backed.copy();allowed[root]=True + while queue: + cost,node=heapq.heappop(queue) + if cost != distances[node] or cost>45*px:continue + if anchors[node].any(): + for side in (0,1): + if anchors[node,side] and cost dict: + """Produce rectangles in image coordinates and explicit review reasons.""" + h, w = ink.shape + px = dpi / 25.4 + middle = w // 2 + reasons = [] + crossings = [] + left_end, right_start = middle, middle + owned = (ink if content_ink is None else content_ink).copy() + left_owned, right_owned = owned[:, :middle].copy(), owned[:, middle:].copy() + if kind == 'a3': + pad = round(2*px) + crossings,group_labels = _fold_regions(ink,owned,dpi,rgb) + for crossing in crossings: + owner = crossing['owner'] + x0,y0,x1,y1 = crossing['box'] + if owner == 'artifact': + continue + if owner == 'left': + left_end = max(left_end,min(w,x1+pad)) + if x1>middle: + right_owned[y0:y1,max(x0,middle)-middle:x1-middle] &= ~(ink[y0:y1,max(x0,middle):x1] & (group_labels[y0:y1,max(x0,middle):x1]==crossing['component_id'])) + elif owner == 'right': + right_start = min(right_start,max(0,x0-pad)) + if x0 None: + im = Image.fromarray(rgb) + im.thumbnail((1200, 1000)) + sx, sy = im.width/plan['size_px'][0], im.height/plan['size_px'][1] + overlay = Image.new('RGBA', im.size) + draw = ImageDraw.Draw(overlay) + for output in plan['outputs']: + x0,y0,x1,y1 = output['rect_px'] + box = (x0*sx,y0*sy,x1*sx-1,y1*sy-1) + colour = '#187941' if output['side'] == 'left' else '#a832a6' + draw.rectangle(box, outline=colour, width=3, + fill=(255,30,30,65) if not output['keep'] else None) + for crossing in plan['crossings']: + x0,y0,x1,y1 = crossing['box'] + draw.rectangle((x0*sx,y0*sy,x1*sx,y1*sy), outline='#999999' if crossing['owner']=='artifact' else '#e88b00', width=2) + Image.alpha_composite(im.convert('RGBA'), overlay).convert('RGB').save(path, quality=90) + + +def process_pdf(job: tuple[str, str, int]) -> list[dict]: + source_s, destination_s, dpi = job + source, destination = Path(source_s), Path(destination_s) + cv2.setNumThreads(1) + digest = hashlib.sha256(source.read_bytes()).hexdigest() + records = [] + with pymupdf.open(source) as doc, pymupdf.open() as result: + for i, page in enumerate(doc): + rotation = page.rotation + kind = classify_format(page.rect.width, page.rect.height) + from copienator.fast_page_split import analyse_pdf_page_fast + plan,rgb = analyse_pdf_page_fast(page,dpi,kind) + pixel_width,pixel_height = plan['size_px'] + plan.update(file=source.name, source_page=i+1, source_sha256=digest, + source_rotation=rotation, source_cropbox=list(page.cropbox)) + preview = f'{source.stem}-{i+1:03}.jpg' + plan['preview'] = f'review/{preview}' + save_preview(rgb, plan, destination/'review'/preview) + # Normalize through a full-page PDF form: remove_rotation() loses + # non-default CropBoxes on quarter-turn pages in some MuPDF builds. + w,h = page.rect.width,page.rect.height + normalized = pymupdf.open() + flat = normalized.new_page(width=w,height=h) + page.set_rotation(0) + if page.get_contents(): + flat.show_pdf_page(flat.rect,doc,i,rotate=-rotation) + for output in plan['outputs']: + x0,y0,x1,y1 = output['rect_px'] + clip = pymupdf.Rect(x0/pixel_width*w,y0/pixel_height*h, + x1/pixel_width*w,y1/pixel_height*h) + output['crop_points'] = list(clip) + if output['keep']: + target = result.new_page(width=clip.width,height=clip.height) + # Truly empty PDF pages have no content stream to show. + if flat.get_contents(): + target.show_pdf_page(target.rect,normalized,0,clip=clip) + output['output_page'] = len(result) + records.append(plan) + normalized.close() + # An entirely blank copy has no output PDF; its report is still kept. + if len(result): + result.save(destination/'pdfs'/source.name, garbage=3, deflate=True) + if hashlib.sha256(source.read_bytes()).hexdigest() != digest: + raise RuntimeError(f'Source changed during analysis: {source}') + output_path = destination/'pdfs'/source.name + expected = sum(o['keep'] for r in records for o in r['outputs']) + if expected: + with pymupdf.open(output_path) as check: + if len(check) != expected or any(p.rect.is_empty for p in check): + raise RuntimeError(f'Invalid output: {output_path}') + return records + + +def run(source: Path, destination: Path, *, dpi: int = 200, workers: int = 1) -> dict: + if dpi < 100 or workers < 1: + raise ValueError('Use dpi >= 100 and workers >= 1') + sources = sorted(source.glob('*.pdf')) if source.is_dir() else [source] + if not sources or any(not p.is_file() or p.suffix.lower() != '.pdf' for p in sources): + raise ValueError('No input PDFs found') + # Refuse reuse: this makes accidental source replacement impossible. + destination.mkdir(parents=True, exist_ok=False) + (destination/'review').mkdir() + (destination/'pdfs').mkdir() + jobs = [(str(p),str(destination),dpi) for p in sources] + records = [] + if workers == 1: + batches = map(process_pdf, jobs) + for batch in batches: + records.extend(batch) + print(f'{batch[0]["file"]}: {len(batch)} scanned pages', flush=True) + else: + with ProcessPoolExecutor(max_workers=workers, + mp_context=multiprocessing.get_context('spawn')) as pool: + for batch in pool.map(process_pdf,jobs): + records.extend(batch) + print(f'{batch[0]["file"]}: {len(batch)} scanned pages', flush=True) + return write_report(records,destination,len(sources)) + + +def write_report(records: list[dict], destination: Path, file_count: int) -> dict: + """Rebuild the gallery from saved decisions without reanalysing scans.""" + summary = dict(files=file_count, source_pages=len(records), + a3=sum(r['format']=='a3' for r in records), a4=sum(r['format']=='a4' for r in records), + output_pages=sum(o['keep'] for r in records for o in r['outputs']), + blank_removed=sum(not o['keep'] for r in records for o in r['outputs']), + review_pages=sum(bool(r['review_reasons']) for r in records), + extended_spreads=sum(any(r['extensions_mm']) for r in records)) + (destination/'report.json').write_text(json.dumps(dict(summary=summary,pages=records),indent=2)+'\n') + cards = [] + for row in records: + name = html.escape(row['file']) + states = ', '.join(f'{o["side"]}: {o["state"]}' for o in row['outputs']) + reasons = ', '.join(row['review_reasons']) or 'No review flag' + links = ' · '.join(f'{o["side"]} PDF' + for o in row['outputs'] if o['keep']) + cards.append(f'
' + f'

{name} / {row["source_page"]}

{row["format"]} | {states}

' + f'

Extensions L/R: {row["extensions_mm"]} mm | {reasons}

{links}

' + f'
') + (destination/'index.html').write_text('' + 'Automatic page splitting - review' + '

Automatic page splitting - experimental

Source order; no booklet reordering. ' + 'Green: left output. Purple: right/whole output. Orange: centre writing. Red shading: removed blank.

' + f'
{html.escape(json.dumps(summary,indent=2))}
'+''.join(cards)+'
') + return summary + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('input',type=Path) + parser.add_argument('output',type=Path) + parser.add_argument('--dpi',type=int,default=200) + parser.add_argument('--workers',type=int,default=1) + args = parser.parse_args(argv) + print(json.dumps(run(args.input,args.output,dpi=args.dpi,workers=args.workers),indent=2)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/copienator/commands/automatic_page_splitter.py b/copienator/commands/automatic_page_splitter.py new file mode 100644 index 0000000..5f883c2 --- /dev/null +++ b/copienator/commands/automatic_page_splitter.py @@ -0,0 +1,183 @@ +"""Publish automatic splits into an evaluation, preserving sources and review data.""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import tempfile +from pathlib import Path + +from copienator import CliError, ExitCode +from copienator.auto_page_split import run as analyse_batch +from copienator.copy_errors import clear_copy_error, copy_errors, mark_copy_error +from copienator.filesystem import publish_file_sets +from copienator.orientation import report_path + + +def decision_path(workspace, stem): + return workspace.metadata_dir / "page_splitting" / f"{stem}.json" + + +def load_decisions(workspace, source): + """Only reuse decisions for the exact preserved input and published output.""" + try: + data = json.loads(decision_path(workspace, source.stem).read_text()) + if data["source_sha256"] != hashlib.sha256(source.read_bytes()).hexdigest(): + return None + output = workspace.copies_dir / source.name + digest = ( + hashlib.sha256(output.read_bytes()).hexdigest() if output.exists() else None + ) + if digest != data["output_sha256"]: + return None + pages = data["pages"] + if not isinstance(pages, list) or not all( + isinstance(row, dict) + and type(row.get("source_page")) is int + and isinstance(row.get("outputs"), list) + and all( + isinstance(o, dict) + and o.get("side") in {"left", "right", "whole"} + and type(o.get("keep")) is bool + and isinstance(o.get("crop_points"), list) + and len(o["crop_points"]) == 4 + for o in row["outputs"] + ) + for row in pages + ): + return None + return pages + except (OSError, ValueError, KeyError, TypeError, AttributeError): + return None + + +def run(workspace, inputs: list[Path], *, workers=4): + if workers < 1: + raise CliError( + "Le nombre de traitements parallèles doit être positif.", + ExitCode.INVALID_ARGUMENTS, + ) + if not inputs: + print("Aucune copie PDF à traiter.", flush=True) + return ExitCode.SUCCESS + if len({p.name for p in inputs}) != len(inputs): + raise CliError( + "Plusieurs copies portent le même nom.", ExitCode.INVALID_ARGUMENTS + ) + copy_errors(workspace) # Validate existing review metadata before publishing. + workspace.runs_dir.mkdir(parents=True, exist_ok=True) + run_dir = Path(tempfile.mkdtemp(prefix="page-split-", dir=workspace.runs_dir)) + snapshots = run_dir / "inputs" + snapshots.mkdir() + for source in inputs: + shutil.copy2(source, snapshots / source.name) + result = run_dir / "result" + print(f"Découpage automatique de {len(inputs)} copie(s).", flush=True) + summary = analyse_batch(snapshots, result, workers=workers) + records = json.loads((result / "report.json").read_text())["pages"] + stages = { + name: run_dir / name for name in ("copies", "originals", "origins", "decisions") + } + for stage in stages.values(): + stage.mkdir() + for source in inputs: + rows = [r for r in records if r["file"] == source.name] + expected = rows[0]["source_sha256"] + if hashlib.sha256(source.read_bytes()).hexdigest() != expected: + raise CliError( + f"{source.name} a changé pendant l’analyse. Aucun PDF remplacé." + ) + shutil.copy2(snapshots / source.name, stages["originals"] / source.name) + for directory, label in ( + (workspace.copies_dir, "previous_copies"), + (workspace.original_copies_dir, "previous_originals"), + ): + prior = directory / source.name + if prior.exists(): + backup = run_dir / label + backup.mkdir(exist_ok=True) + shutil.copy2(prior, backup / source.name) + output = result / "pdfs" / source.name + digest = ( + hashlib.sha256(output.read_bytes()).hexdigest() if output.exists() else None + ) + if digest: + shutil.copy2(output, stages["copies"] / source.name) + origins = [ + {"origin": r["source_page"], "crop_points": o["crop_points"]} + for r in rows + for o in r["outputs"] + if o["keep"] + ] + if digest: + (stages["origins"] / (source.stem + ".json")).write_text( + json.dumps({"version": 1, "sha256": digest, "pages": origins}), + encoding="utf-8", + ) + (stages["decisions"] / (source.stem + ".json")).write_text( + json.dumps( + { + "version": 1, + "source_sha256": expected, + "output_sha256": digest, + "pages": rows, + } + ), + encoding="utf-8", + ) + names = [p.name for p in inputs] + for source in inputs: + if source.read_bytes() != (snapshots / source.name).read_bytes(): + raise CliError( + f"{source.name} a changé pendant l’analyse. Aucun PDF remplacé." + ) + publish_file_sets( + { + workspace.copies_dir: (stages["copies"], names), + workspace.original_copies_dir: (stages["originals"], []), + workspace.metadata_dir / "page_origins": ( + stages["origins"], + [p.stem + ".json" for p in inputs], + ), + workspace.metadata_dir / "page_splitting": (stages["decisions"], []), + } + ) + for source in inputs: + if ( + source.parent.resolve() + not in { + workspace.copies_dir.resolve(), + workspace.original_copies_dir.resolve(), + } + and source.read_bytes() == (snapshots / source.name).read_bytes() + ): + # Root scans become preserved originals, as in the manual workflow. + source.unlink() + rows = [r for r in records if r["file"] == source.name] + output = workspace.copies_dir / source.name + clear_copy_error(workspace, output, source="orientation") + report_path(workspace, source.stem).unlink(missing_ok=True) + reasons = {reason for r in rows for reason in r["review_reasons"]} + if reasons or not output.exists(): + mark_copy_error( + workspace, + output, + "Découpage automatique à vérifier manuellement." + if reasons + else "Toutes les pages semblent vides : vérifier l’original.", + source="page_splitter", + ) + else: + clear_copy_error(workspace, output, source="page_splitter") + print( + f"Terminé : {summary['output_pages']} pages conservées, {summary['blank_removed']} pages vides retirées. " + "Ordre des pages inchangé.", + flush=True, + ) + print( + f"Rapport et sauvegardes : {run_dir}\n" + "Choisissez Manuellement pour vérifier ou modifier les propositions.", + flush=True, + ) + return ExitCode.SUCCESS diff --git a/copienator/commands/page_splitter.py b/copienator/commands/page_splitter.py index 328216b..fa5be47 100644 --- a/copienator/commands/page_splitter.py +++ b/copienator/commands/page_splitter.py @@ -27,7 +27,7 @@ from copienator import ( ) from copienator.platform import launch_pdf_arranger from copienator.copy_errors import marked_copy_paths, clear_copy_error -from copienator.orientation import save_provenance, report_path +from copienator.orientation import save_provenance, report_path, provenance_path # --- Constants --- # Conversion factor: 1 cm to points (1 inch = 2.54 cm, 72 points = 1 inch) @@ -49,7 +49,7 @@ def _temporary_sibling(path: Path, purpose: str) -> Path: def commit_processed_pdf( workspace: EvaluationWorkspace, original_path: Path, - generated_path: Path, + generated_path: Path | None, ) -> Path: """Commit a processed copy and its original backup with rollback.""" backup_path = workspace.original_copies_dir / original_path.name @@ -73,8 +73,9 @@ def commit_processed_pdf( backup_replaced = True if output_path.exists(): output_path.replace(saved_output) - generated_path.replace(output_path) - output_replaced = True + if generated_path is not None: + generated_path.replace(output_path) + output_replaced = True if original_path.resolve() not in { backup_path.resolve(), output_path.resolve(), @@ -120,6 +121,10 @@ class PDFPreviewer: try: self.doc = pymupdf.open(self.pdf_path) self.original_page_ids = {page.xref: index + 1 for index, page in enumerate(self.doc)} + from copienator.commands.automatic_page_splitter import load_decisions + saved = load_decisions(self.workspace, self.pdf_path) + self.automatic_plans = {r['source_page']: r for r in saved} if saved else {} + self.automatic_review = bool(self.automatic_plans) except (OSError, RuntimeError, ValueError) as e: self.failed = True self._temporary_directory.cleanup() @@ -176,6 +181,12 @@ class PDFPreviewer: f"{fmt('add_secondary_splitter')}: add secondary line, {fmt('delete_secondary_splitter')}: delete newest (arrows move newest; main line red, active secondary orange)\n" f"'{fmt('reverse_pages')}': reverse page order and restart from the new first page\n" ) + if self.automatic_review: + instructions += ( + "\nPropositions automatiques : la touche suivante valide le choix affiché. " + "Croix rouge : page retirée.\n" + "Déplacer une ligne reprend le découpage manuel pour cette page." + ) self.info_label = tk.Label(master, text=instructions, justify=tk.LEFT) self.info_label.pack(pady=5, side=tk.TOP) @@ -255,6 +266,25 @@ class PDFPreviewer: self.current_line_x = page.rect.width / 2 self.current_rotation = 0 self.secondary_splitters = [] + origin = getattr(self, 'original_page_ids', {}).get(page.xref, self.current_page_index+1) + plan = getattr(self, 'automatic_plans', {}).get(origin) + self.current_automatic_outputs = plan['outputs'] if plan else None + + def _automatic_outputs(self): + """Express saved asymmetric crops in the current preview orientation.""" + outputs = getattr(self, 'current_automatic_outputs', None) + if outputs is None: + return None + flipped = (self.current_rotation+self.file_rotation+self.global_rotation) % 360 + if not flipped: + return outputs + width, height = self.doc[self.current_page_index].rect.width, self.doc[self.current_page_index].rect.height + result = [] + for output in outputs: + x0,y0,x1,y1 = output['crop_points'] + result.append(dict(output, side={'left':'right','right':'left'}.get(output['side'], 'whole'), + crop_points=[width-x1,height-y1,width-x0,height-y0])) + return result def load_page(self): """Loads and displays the current page on the canvas, scaled to fit.""" @@ -296,9 +326,20 @@ class PDFPreviewer: # --- Draw Line and Rotate --- draw = ImageDraw.Draw(img) # The line position is scaled by the same zoom factor - line_x_scaled = self.current_line_x * self.current_zoom - draw.line([(line_x_scaled, 0), (line_x_scaled, pix.height)], - fill="red", width=3) + proposed = self._automatic_outputs() + if proposed is not None: + for output in proposed: + x0,y0,x1,y1 = output['crop_points'] + box = tuple(v*self.current_zoom for v in (x0,y0,x1,y1)) + draw.rectangle(box, outline='green' if output['side']=='left' else 'purple', width=3) + if not output['keep']: + draw.line((box[0],box[1],box[2],box[3]), fill='red', width=3) + draw.line((box[0],box[3],box[2],box[1]), fill='red', width=3) + self.page_label.config(text=self.page_label.cget('text')+' — Proposition automatique (croix rouge : page retirée)') + else: + line_x_scaled = self.current_line_x * self.current_zoom + draw.line([(line_x_scaled, 0), (line_x_scaled, pix.height)], + fill="red", width=3) for index, position in enumerate(self.secondary_splitters): x = position * self.current_zoom active = index == len(self.secondary_splitters) - 1 @@ -353,6 +394,7 @@ class PDFPreviewer: def add_secondary_splitter(self, event=None): """Start a new line in the widest remaining interval of this page.""" width = self.doc.load_page(self.current_page_index).rect.width + self.current_automatic_outputs = None boundaries = sorted({0, width, self.current_line_x, *self.secondary_splitters}) left, right = max(zip(boundaries, boundaries[1:]), key=lambda pair: pair[1] - pair[0]) self.secondary_splitters.append((left + right) / 2) @@ -364,6 +406,7 @@ class PDFPreviewer: self.load_page() def _move_active_line(self, delta): + self.current_automatic_outputs = None width = self.doc.load_page(self.current_page_index).rect.width if self.secondary_splitters: self.secondary_splitters[-1] = min(width, max(0, self.secondary_splitters[-1] + delta)) @@ -415,13 +458,19 @@ class PDFPreviewer: # Reload the canvas to show the new page self.load_page() - def confirm_and_next_page(self, event=None, keep="both"): + def confirm_and_next_page(self, event=None, keep=None): """Saves the settings for the current page and moves to the next.""" + proposed = self._automatic_outputs() + if keep is None: + sides = {o['side'] for o in proposed if o['keep']} if proposed is not None else {'left','right'} + keep = ('as_is' if 'whole' in sides else 'both' if len(sides)==2 + else next(iter(sides)) if sides else 'none') self.page_settings.append({ "line_x": self.current_line_x, "secondary_splitters": list(self.secondary_splitters), "rotation": self.current_rotation, - "keep": keep + "keep": keep, + "automatic_outputs": proposed, }) self.current_page_index += 1 @@ -442,12 +491,18 @@ class PDFPreviewer: def finish_and_process(self) -> bool: """Render and transactionally install the processed PDF.""" try: - self.split_pdf() - self.reorder_pdfs() - self.concate_files() - output = commit_processed_pdf(self.workspace, self.pdf_path, self.final_file) - save_provenance(self.workspace, output, self.output_origins) + if all(s['keep']=='none' for s in self.page_settings): + self.doc.close() + output = commit_processed_pdf(self.workspace, self.pdf_path, None) + provenance_path(self.workspace, output.stem).unlink(missing_ok=True) + else: + self.split_pdf() + self.reorder_pdfs() + self.concate_files() + output = commit_processed_pdf(self.workspace, self.pdf_path, self.final_file) + save_provenance(self.workspace, output, self.output_origins) clear_copy_error(self.workspace, output, source='orientation') + clear_copy_error(self.workspace, output, source='page_splitter') report_path(self.workspace, output.stem).unlink(missing_ok=True) except Exception as exc: # noqa: BLE001 - interactive boundary self.failed = True @@ -514,15 +569,27 @@ class PDFPreviewer: line_x = settings['line_x'] rotation_settings = settings['rotation'] keep = settings['keep'] - # Bake source rotation and the preview's extra rotation into content, - # so all cut coordinates refer to the displayed page. - page.remove_rotation() - page.set_rotation((rotation_settings + self.file_rotation + self.global_rotation) % 360) - page.remove_rotation() + # Normalize the displayed page without losing an offset CropBox. + # This matches the automatic export's coordinate system. + source_rotation = page.rotation width, height = page.rect.width, page.rect.height + normalized = pymupdf.open() + flat = normalized.new_page(width=width, height=height) + page.set_rotation(0) + if page.get_contents(): + flat.show_pdf_page(flat.rect, self.doc, i, rotate=-source_rotation) + flat.set_rotation((rotation_settings+self.file_rotation+self.global_rotation) % 360) + flat.remove_rotation() sides = [] if keep == "as_is": sides.append((0, width, self.split_filename_left(i))) + elif settings.get('automatic_outputs') is not None: + for output in settings['automatic_outputs']: + side = output['side'] + if keep == 'both' or keep == side: + x0,_,x1,_ = output['crop_points'] + filename = self.split_filename_left(i) if side=='left' else self.split_filename_right(i) + sides.append((x0,x1,filename)) else: if keep in {"both", "left"} and line_x > 0: sides.append((0, line_x, self.split_filename_left(i))) @@ -539,10 +606,12 @@ class PDFPreviewer: for x0, x1 in zip(boundaries, boundaries[1:]): clip = pymupdf.Rect(x0, 0, x1, height) target = output.new_page(width=clip.width, height=clip.height) - target.show_pdf_page(target.rect, self.doc, i, clip=clip) + if flat.get_contents(): + target.show_pdf_page(target.rect, normalized, 0, clip=clip) origins.append({'origin': origin, 'crop_points': list(clip)}) output.save(output_path) self.fragment_origins[str(output_path)] = origins + normalized.close() self.doc.close() print(f"\nProcessing complete. Files are in '{self.split_dir}' directory.") @@ -564,7 +633,7 @@ class PDFPreviewer: psk = ps[i]['keep'] # Si c'est une copie double (on s'assure qu'on a bien 2 pages consécutives modifiables) - if psk in ["both", "right", "left", "none"] and i < len(ps)-1 and ps[i+1]['keep'] in ["both", "right", "left", "none"]: + if not getattr(self, 'automatic_review', False) and psk in ["both", "right", "left", "none"] and i < len(ps)-1 and ps[i+1]['keep'] in ["both", "right", "left", "none"]: # 1. Page de garde (Extérieur Droit) if ps[i]['keep'] in ["both", "right"]: @@ -678,11 +747,17 @@ def _selected_inputs( ] else: candidates = list_pdf_files(directory, workspace) + if target == workspace.root and not candidates: + candidates = list_pdf_files(workspace.original_copies_dir, workspace) return list(reversed(candidates)) -def run(workspace: EvaluationWorkspace, target: Path, *, marked: bool = False) -> ExitCode: +def run(workspace: EvaluationWorkspace, target: Path, *, marked: bool = False, + automatic: bool = False, workers: int = 4) -> ExitCode: inputs = list(reversed(marked_copy_paths(workspace, originals=True))) if marked else _selected_inputs(workspace, target) + if automatic: + from copienator.commands.automatic_page_splitter import run as run_automatic + return run_automatic(workspace, list(reversed(inputs)), workers=workers) if not inputs: print(f"No PDF files found in {target}") return ExitCode.SUCCESS @@ -695,6 +770,8 @@ def run(workspace: EvaluationWorkspace, target: Path, *, marked: bool = False) - def build_parser() -> argparse.ArgumentParser: parser = target_parser("Interactively split and reorder scanned PDF pages") parser.add_argument("--marked", action="store_true", help="Process only copies flagged during margin review") + parser.add_argument("--automatic", action="store_true", help="Split automatically and save editable proposals without reordering") + parser.add_argument("--workers", type=int, default=4, help="Copies analysed simultaneously in automatic mode") return parser @@ -703,7 +780,7 @@ def main(argv: Sequence[str] | None = None) -> int: def handle(args: argparse.Namespace) -> ExitCode: workspace, target = workspace_from_target(args) - return run(workspace, target, marked=args.marked) + return run(workspace, target, marked=args.marked, automatic=args.automatic, workers=args.workers) return execute(parser, argv, handle) diff --git a/copienator/copy_errors.py b/copienator/copy_errors.py index 3110e01..5eaa471 100644 --- a/copienator/copy_errors.py +++ b/copienator/copy_errors.py @@ -20,7 +20,7 @@ def _validate(value) -> dict[str, dict[str, str]]: if isinstance(flags, str): flags = {"cutleft": flags} if (not isinstance(flags, dict) or not flags - or any(step not in {"cutleft", "plotting", "orientation"} or not isinstance(reason, str) + or any(step not in {"cutleft", "plotting", "orientation", "page_splitter"} or not isinstance(reason, str) for step, reason in flags.items())): raise CliError("Invalid copy_errors.json: invalid step flags") normalized[name] = dict(flags) @@ -35,7 +35,7 @@ def copy_errors(workspace: EvaluationWorkspace) -> dict[str, str]: def mark_copy_error(workspace: EvaluationWorkspace, pdf: Path, reason: str, *, source: str = "cutleft") -> None: - if source not in {"cutleft", "plotting", "orientation"}: + if source not in {"cutleft", "plotting", "orientation", "page_splitter"}: raise ValueError(f"Unknown copy flag source: {source}") def update(errors): normalized = _validate(errors) @@ -47,7 +47,7 @@ def mark_copy_error(workspace: EvaluationWorkspace, pdf: Path, reason: str, def clear_copy_error(workspace: EvaluationWorkspace, pdf: Path, *, source: str = "cutleft") -> None: - if source not in {"cutleft", "plotting", "orientation"}: + if source not in {"cutleft", "plotting", "orientation", "page_splitter"}: raise ValueError(f"Unknown copy flag source: {source}") if not _path(workspace).exists(): return diff --git a/copienator/fast_page_split.py b/copienator/fast_page_split.py new file mode 100644 index 0000000..1ed71e8 --- /dev/null +++ b/copienator/fast_page_split.py @@ -0,0 +1,276 @@ +"""Standard coarse-to-fine splitting: cheap positive content gate, precise fold. + +The whole-page gate only establishes obvious writing. All uncertain/sparse pages +use the reference blank detector. Full-resolution fold processing is restricted +to a central strip; edge-reaching overflow and ambiguous ownership fall back to +the complete reference analysis. Full-spread analysis remains an internal fallback. +""" + +from __future__ import annotations +import cv2 +import numpy as np +import pymupdf +from copienator.auto_page_split import analyse_ink, _analyse_page_full, plan_masks + + +def quick_page_content(rgb, dpi, target_dpi=75, *, native=True, _fallback=None): + """Positive-only coarse gate. Ambiguous/sparse pages use full detection.""" + h, w = rgb.shape[:2] + small = cv2.resize( + rgb, + (round(w * target_dpi / dpi), round(h * target_dpi / dpi)), + interpolation=cv2.INTER_AREA, + ) + px = target_dpi / 25.4 + lo = small.min(axis=2) + ch = small.max(axis=2) - lo + gray = cv2.cvtColor(small, cv2.COLOR_RGB2GRAY) + strong = (ch > 100) | (gray < 95) + n, lab, stats, _ = cv2.connectedComponentsWithStats(strong.astype("uint8"), 8) + ok = np.zeros(n, bool) + x, y, bw, bh, area = stats[1:].T + # Only compact interior pen components can establish an obvious written page. + ok[1:] = ( + (area >= 0.25 * px * px) + & (bw >= 0.35 * px) + & (bh >= 0.5 * px) + & (bw < 20 * px) + & (bh < 15 * px) + & (x > 0.09 * small.shape[1]) + & (x + bw < 0.91 * small.shape[1]) + ) + selected = ok[lab] + grouped = cv2.dilate( + selected.astype("uint8"), + np.ones((max(1, round(px)), max(1, round(3 * px))), np.uint8), + ) + ng, groups, gs, _ = cv2.connectedComponentsWithStats(grouped, 8) + comp_group = np.zeros(n, np.int32) + ys, xs = np.nonzero(selected) + comp_group[lab[ys, xs]] = groups[ys, xs] + counts = np.bincount(comp_group[ok], minlength=ng) + amounts = np.bincount(groups[selected], minlength=ng) + good = ( + (counts >= 4) + & (amounts >= 3 * px * px) + & (gs[:, 2] >= 8 * px) + & (gs[:, 3] >= 2 * px) + ) + good[0] = False + if good.any(): + # Retain all strong ink for ownership support; the gate alone is interior-only. + return ( + strong + if native + else cv2.resize( + strong.astype("uint8"), (w, h), interpolation=cv2.INTER_NEAREST + ) + > 0 + ), True + return (_fallback() if _fallback is not None else analyse_ink(rgb, dpi)[0]), False + + +def _a4_plan(mask, gate, h, w, dpi, gate_dpi): + count = int(mask.sum()) + return dict( + format="a4", + size_px=[w, h], + outputs=[ + dict( + side="whole", + rect_px=[0, 0, w, h], + state="content" if count else "blank", + keep=bool(count), + ink_pixels=count, + content_dpi=gate_dpi if gate else dpi, + ) + ], + crossings=[], + review_reasons=[], + extensions_mm=[0, 0], + analysis=dict( + mode="fast", + early_written=[gate], + fold_fraction=0, + gate_dpi=gate_dpi, + fallback=None, + ), + ) + + +def analyse_page_fast(rgb, dpi, kind, *, fold_fraction=0.1, gate_dpi=75): + """Fast analysis for an existing raster image.""" + if kind not in {"a3", "a4"}: + return _analyse_page_full(rgb, dpi, kind) + gate_dpi = min(gate_dpi, dpi) + h, w = rgb.shape[:2] + if kind == "a4": + mask, gate = quick_page_content(rgb, dpi, gate_dpi, native=True) + return _a4_plan(mask, gate, h, w, dpi, gate_dpi) + middle = w // 2 + halves = [] + early = [] + for a, b in ((0, middle), (middle, w)): + mask, gate = quick_page_content(rgb[:, a:b], dpi, gate_dpi, native=True) + halves.append(mask) + early.append(gate) + return _assemble_fast( + halves, + early, + h, + w, + dpi, + fold_fraction, + gate_dpi, + lambda x0, x1: rgb[:, x0:x1], + lambda: _analyse_page_full(rgb, dpi, kind), + ) + + +def _assemble_fast( + halves, early, h, w, dpi, fold_fraction, gate_dpi, render_roi, reference +): + middle = w // 2 + half = max(1, round(w * fold_fraction / 2)) + x0 = max(0, middle - half) + x1 = min(w, middle + half) + roi = render_roi(x0, x1) + content = np.zeros(roi.shape[:2], bool) + for i, (a, b) in enumerate(((0, middle), (middle, w))): + lo = max(a, x0) + hi = min(b, x1) + mask = halves[i] + ys = np.arange(h) * mask.shape[0] // h + xs = (np.arange(lo, hi) - a) * mask.shape[1] // (b - a) + content[:, lo - x0 : hi - x0] = mask[ys[:, None], xs[None, :]] + ink, _ = analyse_ink(roi, dpi, spread=True) + remaining = {} + plan = plan_masks( + ink, dpi, "a3", content_ink=content, rgb=roi, _remaining_content=remaining + ) + # If a proposed overflow reaches the strip's edge or ownership remains + # ambiguous, use the complete reference analysis rather than truncate it. + margin = round(2 * dpi / 25.4) + if ( + plan["review_reasons"] + or plan["outputs"][0]["rect_px"][2] >= x1 - x0 - margin + or plan["outputs"][1]["rect_px"][0] <= margin + ): + plan = reference() + plan["analysis"] = dict(mode="fast", fallback="full-spread") + return plan + for i, (a, b) in enumerate(((0, middle), (middle, w))): + mask = halves[i] + side = "left" if i == 0 else "right" + if not early[i]: + lo = max(a, x0) + hi = min(b, x1) + mask[:, lo - a : hi - a] = remaining[side] + count = int(mask.sum()) + else: + # Exact count of the implied nearest-neighbour full-resolution mask, + # without allocating it. Coarse masks establish obvious content only. + wy = np.diff( + np.ceil(np.arange(mask.shape[0] + 1) * h / mask.shape[0]).astype( + "int64" + ) + ) + wx = np.diff( + np.ceil(np.arange(mask.shape[1] + 1) * (b - a) / mask.shape[1]).astype( + "int64" + ) + ) + count = int(np.sum(mask * wy[:, None] * wx[None, :])) + o = plan["outputs"][i] + o["ink_pixels"] = count + o["keep"] = count > 0 + o["state"] = "content" if count else "blank" + o["rect_px"] = ( + [0, 0, o["rect_px"][2] + x0, h] + if i == 0 + else [o["rect_px"][0] + x0, 0, w, h] + ) + for c in plan["crossings"]: + c["box"][0] += x0 + c["box"][2] += x0 + for box in c.get("anchor_boxes", []): + if box is not None: + box[0] += x0 + box[2] += x0 + plan["size_px"] = [w, h] + plan["analysis"] = dict( + mode="fast", + gate_dpi=gate_dpi, + fold_fraction=fold_fraction, + early_written=early, + fallback=None, + ) + return plan + + +def analyse_pdf_page_fast(page, dpi, kind, *, fold_fraction=0.1, gate_dpi=75): + """Render a small overview and the precise fold directly from the PDF. + + Full-resolution halves are rendered only when the positive overview gate + cannot establish writing. Raster clips are aligned to the same integer + pixel grid as a complete reference render, including rotated CropBoxes. + """ + gate_dpi = min(gate_dpi, dpi) + bounds = (page.rect * pymupdf.Matrix(dpi / 72, dpi / 72)).irect + h, w = bounds.height, bounds.width + coarse_pix = page.get_pixmap( + dpi=round(gate_dpi), colorspace=pymupdf.csRGB, alpha=False + ) + coarse = np.frombuffer(coarse_pix.samples, np.uint8).reshape( + coarse_pix.height, coarse_pix.width, 3 + ) + + def render(x0, x1): + rect = pymupdf.Rect(x0 * 72 / dpi, 0, x1 * 72 / dpi, page.rect.height) + pix = page.get_pixmap( + dpi=round(dpi), clip=rect, colorspace=pymupdf.csRGB, alpha=False + ) + data = np.frombuffer(pix.samples, np.uint8).reshape(pix.height, pix.width, 3) + out = np.full((h, x1 - x0, 3), 255, np.uint8) + left = max(x0, pix.x) + top = max(0, pix.y) + right = min(x1, pix.x + pix.width) + bottom = min(h, pix.y + pix.height) + out[top:bottom, left - x0 : right - x0] = data[ + top - pix.y : bottom - pix.y, left - pix.x : right - pix.x + ] + return out + + def reference(): + return _analyse_page_full(render(0, w), dpi, kind) + + if kind not in {"a3", "a4"}: + return reference(), coarse + if kind == "a4": + mask, gate = quick_page_content( + coarse, + gate_dpi, + gate_dpi, + native=True, + _fallback=lambda: analyse_ink(render(0, w), dpi)[0], + ) + return _a4_plan(mask, gate, h, w, dpi, gate_dpi), coarse + middle = w // 2 + cm = coarse.shape[1] // 2 + halves = [] + early = [] + for a, b, ca, cb in ((0, middle, 0, cm), (middle, w, cm, coarse.shape[1])): + mask, gate = quick_page_content( + coarse[:, ca:cb], + gate_dpi, + gate_dpi, + native=True, + _fallback=lambda a=a, b=b: analyse_ink(render(a, b), dpi)[0], + ) + halves.append(mask) + early.append(gate) + plan = _assemble_fast( + halves, early, h, w, dpi, fold_fraction, gate_dpi, render, reference + ) + plan["analysis"]["rendering"] = "direct-pdf-roi" + return plan, coarse diff --git a/copienator/ink_detection.py b/copienator/ink_detection.py index 9709784..1fd8eb3 100644 --- a/copienator/ink_detection.py +++ b/copienator/ink_detection.py @@ -45,12 +45,15 @@ def _paper_blur(gray: np.ndarray, sigma: float) -> np.ndarray: return blurred.astype(np.uint8) -def _large_blank_ink(gray: np.ndarray, clean: np.ndarray, dpi: float): +def _large_blank_ink(gray: np.ndarray, clean: np.ndarray, dpi: float, *, + content_only: bool = False): """Refine confirmed ruling, only accepting substantial extra blank margins. Short directional openings tolerate locally bent/broken paper lines. A physical component-size threshold rejects their remaining tiny fragments. - This is deliberately limited to already-confirmed ruled paper. + Crop refinement is limited to already-confirmed ruled paper. Content-only + analysis also uses this stroke filter on seedless plain pages and can return + an empty mask; the normal crop path deliberately preserves its input then. """ px = dpi / 25.4 height, width = gray.shape @@ -120,6 +123,14 @@ def _large_blank_ink(gray: np.ndarray, clean: np.ndarray, dpi: float): overlap = np.bincount(lab[strong_ruling > 0], minlength=count) bars &= overlap < st[:, 4]*.5 refined |= bars[lab] + if content_only: + # A crop must preserve its input when there is no reliable content. + # A blank-page classifier needs that empty result instead. Recover the + # same nearby original strokes without restoring remote paper artifacts. + radius = max(1, round(2*px)) + nearby = cv2.dilate(refined.astype(np.uint8), + cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(2*radius+1,2*radius+1))) > 0 + return refined | ((clean > 0) & nearby), len(holes), True ys = np.flatnonzero(np.any(refined, axis=1)) original = np.flatnonzero(np.any(clean, axis=1)) if not len(ys) or not len(original): @@ -138,8 +149,171 @@ def _large_blank_ink(gray: np.ndarray, clean: np.ndarray, dpi: float): return result, len(holes), changed +def _blank_content_support(content: np.ndarray, dpi: float) -> tuple[np.ndarray, dict]: + """Require a glyph, neighbouring strokes, or a substantial diagram. + + A few remote dots and a short loose curve do not establish a written page. + Grouping protects fragmented pencil letters without a page-wide ink quota. + This evidence filter is for blank classification, never margin cropping. + """ + px = dpi/25.4 + n,labels,stats,_ = cv2.connectedComponentsWithStats(content.astype(np.uint8),8) + if n == 1: + return content,dict(content_supported_groups=0,content_unsupported_pixels=0) + radius = max(1,round(1.5*px)) + joined = cv2.dilate(content.astype(np.uint8), + cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(2*radius+1,2*radius+1))) + _,groups = cv2.connectedComponents(joined,8) + ys,xs = np.nonzero(content) + component_groups = np.zeros(n,np.int32) + component_groups[labels[ys,xs]] = groups[ys,xs] + area = stats[:,4] + bw,bh = stats[:,2],stats[:,3] + density = area/np.maximum(1,bw*bh) + letter = ((area >= .8*px*px) & (np.minimum(bw,bh) >= .6*px) + & (density >= .18)) + # A dense sub-2 mm ink spot is not a glyph on its own. Keep it eligible + # for a supported group (decimal points, dots over letters, punctuation). + compact_spot = np.zeros(n,bool) + spot_candidates = np.flatnonzero((density > .55) & (np.maximum(bw,bh) < 2*px)) + for i in spot_candidates: + if i == 0: + continue + x,y,ww,hh,_ = stats[i] + yy,xx = np.nonzero(labels[y:y+hh,x:x+ww] == i) + hull_area = cv2.contourArea(cv2.convexHull(np.c_[xx,yy].astype(np.int32))) + # Small x's have empty wedges inside their convex hull; solid spots + # fill it. Pixel-centre hull area can be smaller than the pixel count. + compact_spot[i] = area[i] >= .9*max(1.,hull_area) + letter &= ~compact_spot + substantial = (area >= 4*px*px) & (np.maximum(bw,bh) >= 5*px) + # A lone short smear at the physical top/bottom edge can pass both area + # tests. Require its pixels to be almost collinear before discounting it; + # normal digits, letters and closed diagrams have broader stroke geometry. + # Neighbouring meaningful strokes still support it through the group rule. + near_edge = (stats[:,1] < 15*px) | (stats[:,1]+bh > content.shape[0]-15*px) + edge_smear = np.zeros(n,bool) + candidates = np.flatnonzero(near_edge & (area < 6*px*px) + & (np.maximum(bw,bh) < 8*px) + & (np.minimum(bw,bh) > 2*px) & (density < .35)) + for i in candidates: + if i == 0: + continue + x,y,ww,hh,_ = stats[i] + yy,xx = np.nonzero(labels[y:y+hh,x:x+ww] == i) + values = np.linalg.eigvalsh(np.cov(np.c_[xx,yy].T)) + edge_smear[i] = values[0] < .05*values[1] + letter &= ~edge_smear + substantial &= ~edge_smear + credible = area >= .2*px*px + credible[0] = False + counts = np.bincount(component_groups[credible],minlength=int(groups.max())+1) + totals = np.bincount(component_groups,weights=area,minlength=len(counts)) + supported = (counts >= 2) & (totals >= 1.2*px*px) + supported[component_groups[letter | substantial]] = True + supported[0] = False + retained = supported[groups] & content + return retained,dict(content_supported_groups=int(supported.sum()), + content_unsupported_pixels=int(content.sum()-retained.sum()), + content_compact_spots=int(compact_spot[1:].sum()), + content_edge_smears=int(edge_smear.sum())) + + +def detect_page_content(rgb: np.ndarray, dpi: float = 200) -> tuple[np.ndarray, dict]: + """Expose rognage's content evidence on a single logical page. + +Normal crop behaviour stays unchanged. On confirmed ruled paper, use its +large-blank refinement even when that refinement finds no content, instead of +the crop-specific fallback that keeps the original page. No separate weak-mark +detector is used. The returned mask is in the input image coordinates. +""" + content_masks = {} + paper_geometry = {} + result = detect_bounds(rgb,dpi,content_masks=content_masks,_paper_geometry=paper_geometry) + content = content_masks['foreground'] + gray = cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY) + # Use the same deskew and paper-family confirmation as the crop pipeline. + h,w = gray.shape + if paper_geometry: + matrix,size,deskewed = (paper_geometry[key] for key in ('matrix','size','deskewed')) + else: + angle = _skew(gray,angle_step=.5) + matrix = cv2.getRotationMatrix2D((w/2,h/2),angle,1) + corners = cv2.transform(np.array([[[0,0],[w,0],[0,h],[w,h]]],float),matrix)[0] + origin = np.floor(corners.min(axis=0)) + size = np.ceil(corners.max(axis=0)-origin).astype(int) + matrix[:,2] -= origin + deskewed = cv2.warpAffine(gray,matrix,tuple(size),borderValue=255) + binary = cv2.adaptiveThreshold(deskewed,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C, + cv2.THRESH_BINARY_INV,max(15,int(dpi/5)|1),9) + horizontal,nh = _ruling(binary,True) + vertical,nv = _ruling(binary,False) + confirmed = ((nh and _edge_connected_family(binary,horizontal,True)) or + (nv and _edge_connected_family(binary,vertical,False))) + if not confirmed: + # Slight local warp breaks one-pixel pale grid lines. Confirm the same + # periodic, edge-reaching family with a small physical-width tolerance. + tolerance = max(3,round(.3*dpi/25.4)|1) + tolerant = cv2.dilate(binary,np.ones((tolerance,tolerance),np.uint8)) + th,tnh = _ruling(tolerant,True) + tv,tnv = _ruling(tolerant,False) + confirmed = ((tnh and _edge_connected_family(tolerant,th,True)) or + (tnv and _edge_connected_family(tolerant,tv,False))) + result['content_tolerant_ruling'] = bool(confirmed) + # With no seeds, the same coherent-stroke filter can recover pencil-only + # writing on plain paper. Arbitrary weak pixels alone are not a veto. + refine = bool(confirmed) or not content.any() + result['content_refinement'] = refine + result['content_ruling_confirmed'] = bool(confirmed) + if refine: + clean = cv2.warpAffine(content.astype(np.uint8),matrix,tuple(size), + flags=cv2.INTER_NEAREST,borderValue=0) + refined,holes,_ = _large_blank_ink(deskewed,clean,dpi,content_only=True) + content = cv2.warpAffine(refined.astype(np.uint8),cv2.invertAffineTransform(matrix), + (w,h),flags=cv2.INTER_NEAREST,borderValue=0)>0 + result['content_holes'] = holes + result['content_pixels'] = int(content.sum()) + if not confirmed and not content.any(): + # The directional refinement can erase very thin pencil letters on + # plain paper. Reuse the crop pipeline's adaptive foreground instead, + # requiring coherent letter-sized components rather than arbitrary + # weak pixels. This fallback is confined to seedless/plain pages. + weak = _foreground(gray,dpi,9,np.zeros_like(gray)) + weak,_ = _content_mask(weak,dpi) + count,labels,stats,_ = cv2.connectedComponentsWithStats(weak,8) + keep = np.zeros(count,bool) + px = dpi/25.4 + keep[1:] = ((stats[1:,4] >= .5*px*px) + & (stats[1:,2] >= px) & (stats[1:,3] >= px)) + # Faint scanner borders can be one huge connected component. They are + # not coherent writing merely because their bounding box is large. + x,y,bw,bh,area = stats[1:].T + sparse = area / np.maximum(1,bw*bh) < .1 + edge_frame = (((bw > .9*w) & ((y < px) | (y+bh > h-px))) + | ((bh > .9*h) & ((x < px) | (x+bw > w-px)))) + edge_frame |= ((bw > .8*w) & ((x < 2*px) | (x+bw > w-2*px))) + keep[1:] &= ~(sparse & edge_frame) + # A faint narrow scanner-edge band is not a pencil letter. Require + # both edge placement and pale neutral appearance, not position alone. + for i in np.flatnonzero(keep[1:])+1: + xx,yy,ww,hh,_ = stats[i] + if ww < 3*px and (xx < 2*px or xx+ww > w-2*px) and hh > ww: + values = rgb[yy:yy+hh,xx:xx+ww][labels[yy:yy+hh,xx:xx+ww]==i] + if np.median(values.min(axis=1)) > 205 and np.median(np.ptp(values.astype(np.int16),axis=1)) < 30: + keep[i] = False + content = keep[labels] + result['content_pixels'] = int(content.sum()) + result['content_pencil_recovery'] = bool(content.any()) + content,support = _blank_content_support(content,dpi) + result.update(support) + result['content_pixels'] = int(content.sum()) + result['content_state'] = 'content' if content.any() else 'blank' + return content,result + + def _neutral_paper_foreground(gray: np.ndarray, chroma: np.ndarray, dpi: float, - *, diagnostics: dict | None = None): + *, diagnostics: dict | None = None, + geometry: dict | None = None): """Clean confirmed dark ruling before it can seed whole pages. Returns None for ordinary ink components or unconfirmed paper geometry. @@ -163,6 +337,8 @@ def _neutral_paper_foreground(gray: np.ndarray, chroma: np.ndarray, dpi: float, size = np.ceil(corners.max(axis=0)-origin).astype(int) matrix[:, 2] -= origin deskewed = cv2.warpAffine(gray, matrix, tuple(size), borderValue=255) + if geometry is not None: + geometry.update(matrix=matrix,size=size,deskewed=deskewed) background = _paper_blur(deskewed, dpi/8) normalized = cv2.divide(deskewed, np.maximum(background,1), scale=255) block = max(15, int(dpi/5) | 1) @@ -220,7 +396,9 @@ def _seed_mask(mask: np.ndarray, px: float, *, paper_nearby=None, def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6, min_crop_mm: float = 5, *, diagnostics: dict | None = None, - colour_cleanup: bool = False, edge_mark_cleanup: bool = True) -> dict: + colour_cleanup: bool = False, edge_mark_cleanup: bool = True, + content_masks: dict | None = None, + _paper_geometry: dict | None = None) -> dict: """Locate strong ink, recover adjacent faint strokes, and retain padding. Neutral punched-hole shadows usually have neither sufficient chroma nor @@ -237,6 +415,9 @@ def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6, Optional diagnostics are written to a separate caller-owned dictionary. Its masks use the original image coordinates and never influence decisions or enter the JSON-serializable return value. + + content_masks requests only foreground and weak-candidate masks, avoiding + diagnostic component explanations and intermediate paper-mask exports. """ px = dpi/25.4 h, w = rgb.shape[:2] @@ -259,7 +440,8 @@ def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6, diagnostics['masks'] = {} diagnostics['thresholds'] = [] masks = diagnostics['masks'] if diagnostics is not None else None - cleaned, paper_info = _neutral_paper_foreground(gray, chroma, dpi, diagnostics=masks) + cleaned, paper_info = _neutral_paper_foreground(gray, chroma, dpi, diagnostics=masks, + geometry=_paper_geometry) if cleaned is not None: # Adaptive neutral cleanup includes faint paper fragments. Such pixels # need local contrast or substantial darkness before they seed a crop. @@ -279,6 +461,9 @@ def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6, masks['isolated_edge_marks'] = np.zeros((h,w), bool) weak = ((chroma > 60) | ((gray < 175) & (residual > 25))).astype(np.uint8) weak[suppressed] = 0 + if content_masks is not None: + content_masks.clear() + content_masks.update(foreground=np.zeros((h,w),bool),weak_candidates=weak>0) if masks is not None: masks['strong_seeds'] = np.zeros((h,w), bool) masks['foreground'] = np.zeros((h,w), bool) @@ -329,6 +514,8 @@ def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6, masks['isolated_edge_marks'] |= discarded[labels] seeded_labels = remaining retained = seeded_labels[labels] + if content_masks is not None: + content_masks['foreground'] |= retained if masks is not None: masks['strong_seeds'] |= seeds & retained ys = np.flatnonzero(np.any(retained, axis=1)) diff --git a/copienator/paper_ruling.py b/copienator/paper_ruling.py index fe4fce4..82a254a 100644 --- a/copienator/paper_ruling.py +++ b/copienator/paper_ruling.py @@ -93,7 +93,6 @@ def ruling_suppression(rgb: np.ndarray, dpi: float): matrix[:,2] -= origin image = cv2.warpAffine(rgb, matrix, tuple(size), borderValue=(255,255,255)) dark = 255-image - dark16 = dark.astype(np.int16) darkness = _maximum(dark) binary = cv2.adaptiveThreshold(_minimum(image), 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, @@ -119,7 +118,7 @@ def ruling_suppression(rgb: np.ndarray, dpi: float): support = _maximum(model) > 9 if count: support &= bands > 0 - close = _maximum(np.abs(dark16-model.astype(np.int16))) < 40 + close = _maximum(cv2.absdiff(dark,model)) < 40 sample = support & (darkness > 9) & close & ~thick # Neutral, red, green and blue line families are modelled separately so # a red margin rule cannot be mistaken for unusually dark blue writing. @@ -128,7 +127,10 @@ def ruling_suppression(rgb: np.ndarray, dpi: float): classes[second <= third] = 1 classes[(first <= second) & (first <= third)] = 0 classes[cv2.subtract(_maximum(model),_minimum(model)) < 25] = 3 - credible = np.zeros(binary.shape, bool) + # Rejected classes have ceiling -1, below every uint8 darkness value. + # Compute each class ceiling once, then test pixels by lookup instead + # of subtracting and splitting full RGB images for every colour class. + ceilings = np.full((4,3),-1,np.int16) for colour in range(4): selected = sample & (classes == colour) if selected.sum() < 100: @@ -143,9 +145,10 @@ def ruling_suppression(rgb: np.ndarray, dpi: float): # An isolated long fraction bar must not define its own paper colour. # The upper quantile admits darker major rules within an established # family while rejecting excess ink on most of its pixels. - ceiling = np.floor(np.quantile(dark[selected],.99,axis=0)+25).astype(np.int16) - credible |= ((classes == colour) - & (_maximum(dark16-ceiling) <= 0)) + ceilings[colour] = np.floor(np.quantile(dark[selected],.99,axis=0)+25).astype(np.int16) + credible = ((dark[:,:,0] <= ceilings[classes,0]) + & (dark[:,:,1] <= ceilings[classes,1]) + & (dark[:,:,2] <= ceilings[classes,2])) match = close & credible & (darkness > 9) & ~thick removed |= support & match protected |= support & ~match & (binary > 0) diff --git a/copienator_gui/workflow.py b/copienator_gui/workflow.py index 6521323..9bac1fd 100644 --- a/copienator_gui/workflow.py +++ b/copienator_gui/workflow.py @@ -259,8 +259,13 @@ def build_workflow(show_personal_steps: bool) -> list[StepDefinition]: "page_splitter", "Prétraitement des copies", "Séparer et réordonner les pages", - "Ouvre l’outil interactif de découpage A3 vers A4. La cible peut être un dossier ou un PDF.", - (python("default", "Séparation des pages", "page-split"),), + "Choisissez le découpage manuel ou automatique. Le traitement automatique sépare les A3, " + "préserve les débordements et retire les pages vides, sans réordonner les pages. " + "Le mode manuel permet ensuite de vérifier et modifier ses propositions.", + ( + python("default", "Manuellement", "page-split"), + python("automatic", "Automatiquement", "page-split", fixed_args=("--automatic",)), + ), arguments=( arg_target(), ArgumentSpec( @@ -270,6 +275,11 @@ def build_workflow(show_personal_steps: bool) -> list[StepDefinition]: "--marked", help="Limite l’opération aux copies signalées dans l’interface au lieu de traiter toutes les copies.", ), + ArgumentSpec( + "workers", "Copies traitées en parallèle", "int", "--workers", default=4, + variants=("automatic",), + help="Nombre de copies analysées simultanément pendant le découpage automatique.", + ), ), artifacts=("Copies", "Copies Originales"), ), diff --git a/docs/auto_page_split.md b/docs/auto_page_split.md new file mode 100644 index 0000000..4029d65 --- /dev/null +++ b/docs/auto_page_split.md @@ -0,0 +1,241 @@ +# Experimental automatic page splitting + +## Python GUI + +The “Séparer et réordonner les pages” step now offers **Manuellement** and +**Automatiquement**. Automatic processing runs in the background using the +standard splitter, publishes PDFs in `Copies`, and preserves source scans in +`Copies Originales`. It retains source-page order. The parallel-copy setting +appears only for automatic processing. Existing single-copy and marked-copy +selection work in both modes. + +Choose Manuellement afterward to review the saved proposals against the +preserved original. The editor displays independent overlapping output crops +and crosses out removed blank pages. Advancing accepts the displayed decision; +the existing keep/discard shortcuts override it. Moving a splitter resumes +manual splitting on that page. Reviewing automatic proposals preserves their +source-page order; the original manual workflow retains its booklet ordering. +Proposals are reused only while both source and output fingerprints still match. + +Each automatic run saves its inputs, report/gallery, and previous outputs under +`.copienator/runs/page-split-*`. All PDFs are analysed before publication, and +PDFs and provenance are published together with rollback on installation errors. +An entirely blank copy keeps its preserved source and receives a manual-review +flag instead of an invalid empty output PDF. + +The equivalent CLI command is: + +```sh +python -m copienator page-split EVALUATION --automatic --workers 4 +``` + +Omit `--automatic` for the interactive editor. The standalone review command +below remains available for experiments without installing results into `Copies`. + +Run from the repository root, on original scans rather than already split copies: + +```sh +python -m copienator.auto_page_split 'Interro07/Copies Originales' tmp/split-review --workers 4 +``` + +The output directory must not already exist. Input PDFs are never overwritten. +This command is deliberately separate from the GUI and from the interactive +`page-split` command. It does not reorder booklets or use secondary splitters. +Outputs follow source-page order, left then right. They must not replace the +current processed evaluation without review and the appropriate later ordering. + +## Outputs + +- `index.html`: source-page previews, proposed boundaries, blank decisions, + uncertainty reasons, and links to exported PDFs. +- `pdfs/`: clipped PDFs retaining source scan data, without rasterizing output. +- `report.json`: decisions, original file hashes, original rotations/CropBoxes, + source page numbers, displayed clip coordinates, and output page numbers. +- `review/`: JPEG previews. Green marks the left output, purple the right/whole + output, orange the detected centre groups; red shading marks a blank removal. + +A wholly blank copy has no output PDF, but still has report entries. An interrupted +run may leave partial outputs; use a new output directory for a subsequent run. + +## Decisions + +Displayed physical dimensions distinguish A3 landscape spreads from A4 sheets, +with a 5% tolerance. A4 landscape sheets are also retained whole. Unexpected +sizes and portrait A3 pages are retained whole and flagged. PDF rotation is +respected; handwriting orientation is not inferred. + +The standard splitter renders a 75 dpi overview, accepts clearly written pages, +and analyses a central 10% strip at the requested 200 dpi for overflow ownership. +Sparse logical pages use `ink_detection.detect_page_content` at full resolution. +Ambiguous ownership, overflow reaching the strip edge, and unexpected geometry +use the internal full-page fallback. Coloured-ruling cleanup prevents grid +fragments from linking text across the fold. + +That shared entry point runs normal rognage with its default artifact filtering. +On confirmed ruled paper it exposes the existing large-blank refinement's retained +content, including an empty result. Ordinary cropping deliberately falls back to +the original mask when refinement is empty; the new entry point does not. Normal +cropping behaviour and defaults remain unchanged. The same coherent-stroke filter +checks seedless pages for plausible pencil writing. On plain paper, the existing +adaptive foreground extractor recovers letter-sized pencil components; faint +page-spanning scanner borders do not qualify. It does not retain a page +merely because unrelated faint pixels survive. There is no separate weak-pixel +veto or splitter-specific hole-removal algorithm. + +Small initial stroke groups near the midpoint are followed towards supported +writing inside either page. Connected faint strokes can recover beyond the +cropper's local 2 mm radius, while page-wide ruling networks are rejected. The +ownership search allows larger horizontal gaps between strong pen regions, +penalises vertical gaps, and stops at a page anchor rather than joining entire +lines across both pages. Pale thin ruling is filtered before word grouping. + +An isolated group with no page anchor is treated as a fold artifact. Comparable +support from both pages still produces an explicit review flag. Output rectangles +cover the actual terminal strokes plus 2 mm padding, not the expanded search +area. Each side's boundary remains independent. The regression suite and reviewed +real examples are documented in `cropping-test/page-splitting/README.md`. + +Blank decisions use logical-page evidence after subtracting actual strokes +attributed to the opposite side. The report includes the crop analysis and the +content-refinement result for each output. A spread artifact that is absent from +the logical page's credible content cannot by itself veto blank removal. + +## Limitations and interpretation + +The report's flags are heuristic reasons, not calibrated confidence scores. +Repeated-hole detection and ruling suppression can be ambiguous around unusual +handwriting. Extremely faint isolated writing can escape detection. Horizontal +grouping can miss disconnected parts of long formulae or merge adjacent blocks. +Ambiguous cases intentionally duplicate content rather than choosing an owner. +Full-height rectangular overlap can include neighbouring text at other heights. + +Blank removal is intentionally conservative. Do not interpret a retained page +as proof it contains student work. In particular, residual scan marks can keep +otherwise blank pages. Interro07's saved page-origin mappings provide a useful +comparison with earlier processing, but are not independent handwritten ground +truth and do not validate overflow attribution. + +## Verification + +```sh +python -m unittest tests.test_auto_page_split tests.test_ink_detection tests.test_crop_blank_margins tests.test_page_splitter_reverse tests.test_page_splitter_secondary -q +``` + +Tests cover both overflow directions at once, disconnected letters, ambiguous +ownership, opposite-side blank attribution, weak scan noise, pencil-only writing, +blank grids with binding holes, faint grid notes and fraction bars, A4/unknown +sizes, all four PDF rotations with CropBox +offsets, rendered export fidelity, input preservation, existing-output refusal, +and wholly blank copies. + +## Updated full Interro07 batch (2026-11-03) + +The current batch is `tmp/auto-page-split-interro07-support-20261103/`. +All 47 original PDFs were processed: 214 scanned pages, with 206 A3 spreads +and 8 A4 sheets. The output contains **347 logical pages**, removing **73 blank +halves**. All 345 pages kept by the earlier workflow remain kept. The two +additional retained halves contain exercise headings (Copie08 source 6/left and +Copie11 source 4/left), consistent with the reviewed content policy. + +Exactly seven keep/remove decisions change from the preceding 66-removal batch, +matching the seven reviewed difficult blank fixtures. Their source halves were +visually checked; the two heading exports were rendered with Poppler and checked. +Original hashes, output page counts and displayed dimensions were verified. +`comparison.json` records the comparisons. The 26 source-page overlap review +flags remain; blank detection results do not validate all overflow attribution. + +## Shared rognage trial (2026-11-02) + +The current results are in `tmp/auto-page-split-interro07-shared-rognage/`. +Blank detection now uses the shared crop content analysis on each logical page. +The splitter-specific weak-pixel veto and hole cleanup have been removed. +Normal cropping still uses its original empty-refinement fallback. + +Against Interro07's saved page-origin mappings, the updated detector removes +66 of the 75 previously omitted halves (88%), compared with 5 in the initial +prototype. All 345 previously retained pages are preserved. Nine previously +omitted halves remain kept, including two very pale pages retained by the +pencil-writing recovery check. These are conservative keep decisions; no +previously retained page was removed. The output has 354 logical pages. + +All proposed removals were reviewed in source-page contact sheets. The final +PDF page counts, dimensions, and source hashes are checked, with results in +`comparison.json`. The 44 regression tests pass, covering normal crop behaviour, +blank ruled sheets with holes, faint writing, scanner borders, large faint +diagrams, overlap geometry, and rotation. This validates blank detection against +the existing Interro07 workflow; it is not an exhaustive annotation of handwriting +or overflow ownership. Centre-binding artifacts can still cause false overlap +proposals. No GUI integration or change to the existing processed PDFs was made. + +## Initial Interro07 trial (2026-11-02; superseded) + +The final trial is in `tmp/auto-page-split-interro07-final/` (ignored by Git). +It processed all 47 original PDFs: 214 scanned pages, comprising 206 A3 spreads +and 8 A4 sheets. Original-file hashes were verified unchanged. The exported +PDFs contain 415 logical pages. There are 47 extended spreads, including 21 with +both boundaries extended, and 69 source pages flagged for review. These counts +describe proposals, not validated correct overflow detections. + +Comparison with the saved `Interro07/.copienator/page_origins/` mappings: + +| Decision | Logical pages | +| --- | ---: | +| Previously retained, still retained | 345 | +| Previously retained, now removed | 0 | +| Previously omitted, now removed | 5 | +| Previously omitted, retained as uncertain | 58 | +| Previously omitted, classified as content | 12 | + +All five proposed removals were visually inspected and appear blank. Selected +overflow previews and Poppler-rendered exports were also inspected. The coloured +ruling cleanup fixed a false 88.65 mm left extension on Copie36, source page 4: +the final 4.45 mm extension follows the equation at the centre. Binding artifacts +still generate false extensions elsewhere. No independent exhaustive annotation +of overflow ownership was made; preserving previously retained pages does not +prove that every stroke is correctly assigned or unclipped. + +The 38 focused and existing regression tests pass, including pixel comparisons +of exported pages with rotated, offset-CropBox sources. The prototype is **not +ready for unattended GUI integration**. The next priorities are better blank-page +recall without erasing faint work, and better separation of binding artifacts +from centre handwriting. `comparison.json` lists the 70 keep/remove disagreements +for targeted follow-up. Neither the GUI nor the existing processed copies changed. + +## Standard coarse-to-fine analysis + +The former fast mode is now the only command-line analysis path. It renders a +75 dpi overview and only a central 10% strip at the requested 200 dpi. A positive +content gate accepts obvious writing; sparse or ambiguous pages still use full +resolution content detection. Ambiguous ownership or overflow reaching the strip +boundary uses the complete reference analysis. Unexpected geometry keeps the +reference path. Exported scan data remains at its original resolution. + +```sh +python -m copienator.auto_page_split 'Interro07/Copies Originales' tmp/fast-review \ + --workers 4 +``` + +The full Interro07 run in `tmp/auto-page-split-interro07-fast-final-20261103/` +takes 37.87 s including exports/previews, retains 347 pages and removes 73 blank +halves. All 420 keep/remove decisions agree with a fresh reference analysis. +Seven output boundaries differ: five by one pixel, and two inspected differences +in ruling/padding without observed loss of writing. This mode is not pixel-identical +to the reference. There is no analysis-mode selector; full-spread analysis remains an internal fallback. +See `cropping-test/page-splitting/experiments/FAST_ANALYSIS.md` for matched timing +comparisons, rejected low-resolution variants, quality checks and limitations. + + +## Interro06 blank-detection follow-up + +The reviewed Interro06 suite originally passed 15/19 with fast analysis. All four +failures were unwritten halves retained for isolated ink marks. Compact solid +spots smaller than 2 mm no longer qualify as glyphs by themselves; their convex +hull must be filled, so small x-shaped glyphs remain eligible. A short almost +collinear smear within 15 mm of the top/bottom edge also needs neighbouring +writing support. The smear rule has additional area, size and density limits. +Both filters affect blank classification only; normal margin cropping is unchanged. + +The four targets now pass, along with all 19 Interro06 cases, all 15 Interro07 +blank cases and the writing-preservation controls. Inputs and reviewed expectations +are unchanged. The exact marks and initial measurements are saved under +`cropping-test/interro06/blank-investigation/`. diff --git a/tests/test_auto_page_split.py b/tests/test_auto_page_split.py new file mode 100644 index 0000000..1dc9ec4 --- /dev/null +++ b/tests/test_auto_page_split.py @@ -0,0 +1,167 @@ +import tempfile +import unittest +import json +from pathlib import Path + +import numpy as np +import pymupdf + +from copienator.auto_page_split import analyse_ink, analyse_page, classify_format, plan_masks, run + + +class AutomaticSplitTests(unittest.TestCase): + def mask(self): + return np.zeros((300, 420), bool) + + def plan(self, mask, kind='a3'): + return plan_masks(mask, 25.4, kind) + + def test_formats_use_size_not_landscape_alone(self): + pt = 72/25.4 + for w,h,expected in [(420,297,'a3'),(210,297,'a4'),(297,210,'a4'), + (297,420,'unknown'),(320,200,'unknown')]: + self.assertEqual(classify_format(w*pt,h*pt),expected) + + def test_both_directions_extend_independently(self): + mask = self.mask() + mask[40:43,150:218] = True + mask[100:103,203:280] = True + plan = self.plan(mask) + self.assertEqual([x['owner'] for x in plan['crossings']],['left','right']) + self.assertEqual(plan['extensions_mm'],[10,9]) + self.assertTrue(all(o['keep'] for o in plan['outputs'])) + + def test_disconnected_letters_form_a_line(self): + mask = self.mask() + for x in range(170,219,4): + mask[40:44,x:x+2] = True + plan = self.plan(mask) + self.assertGreater(plan['extensions_mm'][0],0) + self.assertEqual(plan['crossings'][0]['owner'],'left') + + def test_detached_terminal_strokes_follow_supported_text_on_either_side(self): + mask=self.mask() + mask[40:44,160:199]=True + mask[41:45,205:219]=True + mask[100:104,221:260]=True + mask[101:105,201:215]=True + plan=self.plan(mask) + self.assertEqual([c['owner'] for c in plan['crossings']],['left','right']) + self.assertGreater(plan['extensions_mm'][0],0) + self.assertGreater(plan['extensions_mm'][1],0) + + def test_fold_only_groups_are_artifacts_without_page_anchors(self): + mask=self.mask() + mask[40:43,205:215]=True + plan=self.plan(mask) + self.assertEqual(plan['crossings'][0]['owner'],'artifact') + self.assertEqual(plan['extensions_mm'],[0,0]) + self.assertFalse(plan['review_reasons']) + + def test_pale_ruling_does_not_attach_to_nearby_text_but_dark_bar_does(self): + mask=np.zeros((600,840),bool) + mask[100:103,418:468]=True + mask[100:120,490:580]=True + credible=np.zeros_like(mask) + credible[100:120,490:580]=True + for colour,owner in [((134,162,200),'artifact'),((80,80,80),'right')]: + with self.subTest(colour=colour): + rgb=np.full((*mask.shape,3),255,np.uint8) + rgb[mask]=(20,20,20) + rgb[100:103,418:468]=colour + plan=plan_masks(mask,200,'a3',content_ink=credible,rgb=rgb) + self.assertEqual(plan['crossings'][0]['owner'],owner) + + def test_overflow_does_not_make_other_page_nonblank(self): + mask = self.mask() + mask[40:43,150:218] = True + plan = self.plan(mask) + self.assertEqual([o['state'] for o in plan['outputs']],['content','blank']) + self.assertFalse(plan['outputs'][1]['keep']) + + def test_ambiguous_crossing_preserved_on_both(self): + mask = self.mask() + mask[40:43,180:250] = True + plan = self.plan(mask) + self.assertIn('ambiguous-centre-writing',plan['review_reasons']) + self.assertLess(plan['outputs'][1]['rect_px'][0],180) + self.assertGreater(plan['outputs'][0]['rect_px'][2],250) + + def test_unknown_format_and_a4(self): + mask = self.mask() + self.assertTrue(self.plan(mask,kind='unknown')['outputs'][0]['keep']) + mask[40:44,50:54] = True + self.assertEqual(self.plan(mask,kind='a4')['outputs'][0]['rect_px'],[0,0,420,300]) + + def test_unseeded_scan_noise_does_not_veto_blank(self): + rgb = np.full((600,420,3),255,np.uint8) + rgb[200:203,200:202] = 195 + plan = analyse_page(rgb,100,'a4') + self.assertEqual(plan['outputs'][0]['state'],'blank') + self.assertEqual(plan['analysis']['early_written'],[False]) + + def test_faint_strokes_near_reliable_ink_are_recovered(self): + rgb = np.full((600,420,3),255,np.uint8) + rgb[200:210,200:204] = 195 + rgb[200:210,204:208] = 30 + ink,_ = analyse_ink(rgb,100) + self.assertTrue(ink[200:210,204:208].any()) + self.assertEqual(analyse_page(rgb,100,'a4')['outputs'][0]['state'],'content') + + def test_spread_artifacts_cannot_veto_blank_logical_half(self): + spread=self.mask() + spread[100:105,208:213]=True + content=self.mask() + content[30:40,300:350]=True + plan=plan_masks(spread,25.4,'a3',content_ink=content) + self.assertEqual([o['state'] for o in plan['outputs']],['blank','content']) + self.assertEqual(plan['crossings'][0]['owner'],'artifact') + self.assertFalse(plan['review_reasons']) + + def test_export_preserves_visible_rotation_cropbox_and_sources(self): + for rotation in (0,90,180,270): + with self.subTest(rotation=rotation), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root/'source.pdf' + with pymupdf.open() as doc: + # Crop offsets and all rotations; displayed geometry is A3. + w,h = (420,297) if rotation%180==0 else (297,420) + pt = 72/25.4 + p = doc.new_page(width=w*pt+20,height=h*pt+30) + p.draw_rect(pymupdf.Rect(50,50,w*pt-50,h*pt-50),fill=(0,0,1)) + p.draw_rect(pymupdf.Rect(100,100,150,180),fill=(1,0,0)) + p.set_cropbox(pymupdf.Rect(10,15,w*pt+10,h*pt+15)) + p.set_rotation(rotation) + doc.save(source) + before = source.read_bytes() + summary = run(source,root/'result',dpi=100) + self.assertEqual(summary['output_pages'],2) + self.assertEqual(source.read_bytes(),before) + with pymupdf.open(source) as original, pymupdf.open(root/'result/pdfs/source.pdf') as result: + outputs=json.loads((root/'result/report.json').read_text())['pages'][0]['outputs'] + for page,output in zip(result,outputs): + self.assertEqual(page.rotation,0) + self.assertAlmostEqual(page.rect.height,original[0].rect.height,places=3) + # Ambiguous full-width rectangle is deliberately duplicated. + a = original[0].get_pixmap(matrix=pymupdf.Matrix(.3,.3)) + b = page.get_pixmap(matrix=pymupdf.Matrix(.3,.3)) + self.assertGreater(b.width, a.width*.8) + expected=original[0].get_pixmap(clip=pymupdf.Rect(output['crop_points'])) + actual=page.get_pixmap() + aa=np.frombuffer(expected.samples,np.uint8).reshape(expected.height,expected.width,3) + bb=np.frombuffer(actual.samples,np.uint8).reshape(actual.height,actual.width,3) + h,w=min(aa.shape[0],bb.shape[0]),min(aa.shape[1],bb.shape[1]) + self.assertLess(np.abs(aa[:h,:w].astype(float)-bb[:h,:w]).mean(),2) + with self.assertRaises(FileExistsError): + run(source,root/'result',dpi=100) + + def test_all_blank_copy_has_report_without_invalid_empty_pdf(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory) + with pymupdf.open() as doc: + doc.new_page(width=210*72/25.4,height=297*72/25.4) + doc.save(root/'blank.pdf') + summary=run(root/'blank.pdf',root/'result',dpi=100) + self.assertEqual(summary['blank_removed'],1) + self.assertFalse((root/'result/pdfs/blank.pdf').exists()) + self.assertTrue((root/'result/report.json').exists()) diff --git a/tests/test_automatic_page_splitter.py b/tests/test_automatic_page_splitter.py new file mode 100644 index 0000000..522901b --- /dev/null +++ b/tests/test_automatic_page_splitter.py @@ -0,0 +1,204 @@ +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +import numpy as np +import pymupdf + +from copienator import EvaluationWorkspace, ExitCode +from copienator.commands import automatic_page_splitter as automatic +from copienator.commands.page_splitter import PDFPreviewer, _selected_inputs, main +from copienator.copy_errors import copy_errors +from copienator.orientation import load_provenance +from copienator_gui.workflow import build_command, build_workflow + + +class AutomaticPageSplitterTests(unittest.TestCase): + def make_source(self, root, *, blank=False): + (root / "labels").write_text("Ex 1\n") + source = root / "Copie01.pdf" + with pymupdf.open() as doc: + page = doc.new_page(width=420 * 72 / 25.4, height=297 * 72 / 25.4) + if not blank: + page.insert_text((60, 100), "Answer on the left page", fontsize=16) + page.insert_text((750, 100), "Answer on the right page", fontsize=16) + page.insert_text((560, 300), "x + y = z", fontsize=16) + doc.new_page(width=420 * 72 / 25.4, height=297 * 72 / 25.4) + page = doc.new_page(width=210 * 72 / 25.4, height=297 * 72 / 25.4) + page.insert_text((60, 100), "Exercise 3", fontsize=16) + doc.save(source) + return source + + def test_gui_choices_build_manual_and_automatic_commands(self): + step = next(s for s in build_workflow(False) if s.id == "page_splitter") + self.assertEqual( + [v.label for v in step.variants], ["Manuellement", "Automatiquement"] + ) + manual, auto = [ + build_command(Path.cwd(), step, v, {"workers": 3}, "Exam") + for v in step.variants + ] + self.assertNotIn("--automatic", manual) + self.assertNotIn("--workers", manual) + self.assertIn("--automatic", auto) + self.assertEqual(auto[auto.index("--workers") + 1], "3") + + def test_cli_publishes_without_tk_and_preserves_original_and_provenance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = self.make_source(root) + before = source.read_bytes() + workspace = EvaluationWorkspace(root) + with patch("copienator.commands.page_splitter.tk.Tk") as tk: + self.assertEqual(main([str(root), "--automatic", "--workers", "1"]), 0) + tk.assert_not_called() + preserved = workspace.original_copies_dir / source.name + self.assertEqual(preserved.read_bytes(), before) + self.assertFalse(source.exists()) + output = workspace.copies_dir / source.name + with pymupdf.open(output) as doc: + self.assertEqual(len(doc), 3) + origins = load_provenance(workspace, output, len(doc)) + self.assertEqual([o["origin"] for o in origins], [1, 1, 3]) + self.assertGreater(doc[0].rect.width, 210 * 72 / 25.4) + saved = automatic.load_decisions(workspace, preserved) + self.assertEqual(len(saved), 3) + self.assertEqual(_selected_inputs(workspace, root), [preserved]) + self.assertFalse(copy_errors(workspace)) + runs = list(workspace.runs_dir.glob("page-split-*")) + self.assertTrue((runs[0] / "result/pdfs" / source.name).exists()) + self.assertEqual( + automatic.run(workspace, [preserved], workers=1), ExitCode.SUCCESS + ) + self.assertEqual(preserved.read_bytes(), before) + + def test_processing_failure_leaves_existing_files_in_place(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = self.make_source(root) + before = source.read_bytes() + workspace = EvaluationWorkspace(root) + workspace.copies_dir.mkdir() + output = workspace.copies_dir / source.name + output.write_bytes(b"previous processed copy") + with ( + patch.object( + automatic, + "analyse_batch", + side_effect=RuntimeError("analysis failed"), + ), + self.assertRaisesRegex(RuntimeError, "analysis failed"), + ): + automatic.run(workspace, [source], workers=1) + self.assertEqual(source.read_bytes(), before) + self.assertEqual(output.read_bytes(), b"previous processed copy") + self.assertFalse(workspace.original_copies_dir.exists()) + + def test_parallel_run_preserves_unselected_copies_and_ignores_stale_proposals(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = self.make_source(root) + second = root / "Copie02.pdf" + shutil.copy2(source, second) + workspace = EvaluationWorkspace(root) + workspace.copies_dir.mkdir() + other = workspace.copies_dir / "Copie99.pdf" + other.write_bytes(b"unselected copy") + automatic.run(workspace, [source, second], workers=2) + self.assertEqual(other.read_bytes(), b"unselected copy") + original = workspace.original_copies_dir / source.name + self.assertIsNotNone(automatic.load_decisions(workspace, original)) + output = workspace.copies_dir / source.name + output.write_bytes(b"edited after automatic splitting") + self.assertIsNone(automatic.load_decisions(workspace, original)) + + def test_wholly_blank_copy_is_preserved_and_flagged_without_stale_output(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = self.make_source(root, blank=True) + before = source.read_bytes() + workspace = EvaluationWorkspace(root) + workspace.copies_dir.mkdir() + (workspace.copies_dir / source.name).write_bytes(b"stale output") + automatic.run(workspace, [source], workers=1) + self.assertFalse((workspace.copies_dir / source.name).exists()) + self.assertEqual( + (workspace.original_copies_dir / source.name).read_bytes(), before + ) + self.assertIn(source.name, copy_errors(workspace)) + run = next(workspace.runs_dir.glob("page-split-*")) + self.assertEqual( + (run / "previous_copies" / source.name).read_bytes(), b"stale output" + ) + + def test_manual_acceptance_preserves_saved_overlap_blanks_and_order(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = self.make_source(root) + workspace = EvaluationWorkspace(root) + automatic.run(workspace, [source], workers=1) + original = workspace.original_copies_dir / source.name + before = original.read_bytes() + output = workspace.copies_dir / source.name + expected = pymupdf.open(output) + self.addCleanup(expected.close) + preview = PDFPreviewer.__new__(PDFPreviewer) + preview.workspace = workspace + preview.inputs = [original] + preview.num = preview.global_rotation = 0 + preview.history = [] + preview.output_dir = None + preview.failed = False + preview.master = Mock() + preview.load_page = Mock() + self.assertTrue(preview.setup_next_file()) + self.assertTrue(preview.automatic_review) + preview._initialize_current_page_settings() + for _ in range(3): + preview.confirm_and_next_page() + self.assertFalse(preview.failed) + self.assertEqual( + [s["keep"] for s in preview.page_settings], ["both", "none", "as_is"] + ) + self.assertEqual(original.read_bytes(), before) + with pymupdf.open(output) as actual: + self.assertEqual(len(actual), len(expected)) + for a, b in zip(actual, expected): + self.assertAlmostEqual(a.rect.width, b.rect.width, places=3) + aa, bb = a.get_pixmap(dpi=40), b.get_pixmap(dpi=40) + self.assertEqual((aa.width, aa.height), (bb.width, bb.height)) + difference = np.abs( + np.frombuffer(aa.samples, np.uint8).astype(float) + - np.frombuffer(bb.samples, np.uint8) + ) + self.assertLess(difference.mean(), 0.1) + + def test_accepting_wholly_blank_proposal_never_creates_an_empty_pdf(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = self.make_source(root, blank=True) + workspace = EvaluationWorkspace(root) + automatic.run(workspace, [source], workers=1) + original = workspace.original_copies_dir / source.name + before = original.read_bytes() + preview = PDFPreviewer.__new__(PDFPreviewer) + preview.workspace = workspace + preview.inputs = [original] + preview.num = preview.global_rotation = 0 + preview.history = [] + preview.failed = False + preview.master = Mock() + preview.load_page = Mock() + preview.setup_next_file() + preview._initialize_current_page_settings() + preview.confirm_and_next_page() + self.assertFalse(preview.failed) + self.assertEqual(original.read_bytes(), before) + self.assertFalse((workspace.copies_dir / source.name).exists()) + self.assertNotIn(source.name, copy_errors(workspace)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fast_page_split.py b/tests/test_fast_page_split.py new file mode 100644 index 0000000..de77816 --- /dev/null +++ b/tests/test_fast_page_split.py @@ -0,0 +1,88 @@ +import unittest + +import cv2 +import numpy as np +import pymupdf + +from copienator.auto_page_split import _analyse_page_full as analyse_page +from copienator.fast_page_split import analyse_page_fast, analyse_pdf_page_fast, quick_page_content + + +class FastPageSplitTests(unittest.TestCase): + def test_direct_pdf_strip_respects_rotation_and_crop_offsets(self): + image = np.full((1400, 2000, 3), 255, np.uint8) + for y in range(150, 650, 70): + for x in (150, 1300): + cv2.putText(image, 'answer x y z', (x, y), + cv2.FONT_HERSHEY_SIMPLEX, 1, (20, 20, 20), 2) + cv2.putText(image, 'x + y = z', (870, 1000), + cv2.FONT_HERSHEY_SIMPLEX, 1, (20, 20, 20), 2) + for rotation in (0, 90, 180, 270): + with self.subTest(rotation=rotation), pymupdf.open() as doc: + source = np.ascontiguousarray(np.rot90(image, rotation//90)) + h,w = source.shape[:2] + page = doc.new_page(width=w*72/150+20,height=h*72/150+30) + box = pymupdf.Rect(10,15,w*72/150+10,h*72/150+15) + pix = pymupdf.Pixmap(pymupdf.csRGB,w,h,source.tobytes(),False) + page.insert_image(box,pixmap=pix) + page.set_cropbox(box) + page.set_rotation(rotation) + rendered = page.get_pixmap(dpi=150,alpha=False) + full = np.frombuffer(rendered.samples,np.uint8).reshape(rendered.height,rendered.width,3) + reference = analyse_page(full,150,'a3') + fast,overview = analyse_pdf_page_fast(page,150,'a3') + self.assertIsNone(fast['analysis']['fallback']) + self.assertEqual(fast['size_px'],reference['size_px']) + self.assertLess(overview.size,full.size) + self.assertEqual([o['keep'] for o in fast['outputs']],[True,True]) + for a,b in zip(fast['outputs'],reference['outputs']): + self.assertLessEqual(max(abs(x-y) for x,y in zip(a['rect_px'],b['rect_px'])),2) + + def test_plain_blank_and_sparse_pencil_use_full_detector(self): + for pencil in (False, True): + with self.subTest(pencil=pencil): + image = np.full((1200, 850, 3), 255, np.uint8) + if pencil: + cv2.putText(image, 'pencil', (100, 600), + cv2.FONT_HERSHEY_SIMPLEX, 1, (195, 195, 195), 2) + mask, early = quick_page_content(image, 150) + self.assertFalse(early) + expected = analyse_page(image, 150, 'a4') + actual = analyse_page_fast(image, 150, 'a4') + self.assertEqual(actual['outputs'][0]['keep'], expected['outputs'][0]['keep']) + self.assertEqual(actual['outputs'][0]['rect_px'], [0, 0, 850, 1200]) + self.assertEqual(bool(mask.any()), pencil) + + def test_compact_roi_maps_bounds_back_to_full_spread(self): + image = np.full((1400, 2000, 3), 255, np.uint8) + for y in range(150, 650, 70): + for x in (150, 1300): + cv2.putText(image, 'answer x y z', (x, y), + cv2.FONT_HERSHEY_SIMPLEX, 1, (20, 20, 20), 2) + cv2.putText(image, 'x + y = z', (870, 1000), + cv2.FONT_HERSHEY_SIMPLEX, 1, (20, 20, 20), 2) + for mirrored in (False, True): + with self.subTest(mirrored=mirrored): + rgb = image[:, ::-1].copy() if mirrored else image + reference = analyse_page(rgb, 150, 'a3') + fast = analyse_page_fast(rgb, 150, 'a3') + self.assertIsNone(fast['analysis']['fallback']) + self.assertEqual(fast['size_px'], [2000, 1400]) + self.assertEqual(fast['extensions_mm'], reference['extensions_mm']) + self.assertEqual([o['rect_px'] for o in fast['outputs']], + [o['rect_px'] for o in reference['outputs']]) + self.assertTrue(all(o['keep'] for o in fast['outputs'])) + + def test_ambiguous_or_wide_writing_falls_back_without_truncation(self): + image = np.full((700, 1000, 3), 255, np.uint8) + image[200:230, 300:650] = 20 + reference = analyse_page(image, 100, 'a3') + fast = analyse_page_fast(image, 100, 'a3') + self.assertEqual(fast['analysis']['fallback'], 'full-spread') + self.assertEqual(fast['outputs'], reference['outputs']) + self.assertEqual(fast['crossings'], reference['crossings']) + + def test_unknown_geometry_keeps_reference_behaviour(self): + image = np.full((400, 500, 3), 255, np.uint8) + self.assertEqual(analyse_page_fast(image, 100, 'unknown'), + analyse_page(image, 100, 'unknown')) diff --git a/tests/test_ink_detection.py b/tests/test_ink_detection.py index ce3820e..3dcaafa 100644 --- a/tests/test_ink_detection.py +++ b/tests/test_ink_detection.py @@ -3,10 +3,116 @@ import unittest import cv2 import numpy as np -from copienator.ink_detection import detect_bounds, _paper_blur +from copienator.ink_detection import detect_bounds, detect_page_content, _paper_blur, _blank_content_support class InkDetectionTests(unittest.TestCase): + def test_lightweight_content_masks_match_full_diagnostics(self): + for image,colour in [(self.scan(),False),(self.dark_grid(),False), + (self.scan(),True),(np.full((300,220,3),255,np.uint8),False)]: + with self.subTest(colour_cleanup=colour,shape=image.shape): + full,light = {},{} + expected=detect_bounds(image,dpi=150,diagnostics=full,colour_cleanup=colour) + actual=detect_bounds(image,dpi=150,content_masks=light,colour_cleanup=colour) + self.assertEqual(actual,expected) + self.assertEqual(set(light),{'foreground','weak_candidates'}) + for name in light: + np.testing.assert_array_equal(light[name],full['masks'][name]) + + def test_blank_detection_ignores_remote_specks(self): + image = np.full((1200,850,3),255,np.uint8) + for x,y in ((150,200),(400,600),(700,1000)): + cv2.rectangle(image,(x,y),(x+3,y+4),(30,30,30),-1) + mask,result = detect_page_content(image,dpi=150) + self.assertFalse(mask.any()) + self.assertGreater(result['content_unsupported_pixels'],0) + + def test_isolated_question_numbers_remain_content(self): + for text in ('1','7','16','x'): + with self.subTest(text=text): + image = np.full((1200,850,3),255,np.uint8) + cv2.putText(image,text,(300,600),cv2.FONT_HERSHEY_SIMPLEX,.7,(30,30,30),2) + self.assertTrue(detect_page_content(image,dpi=150)[0].any()) + + def test_faint_edge_question_number_remains_content(self): + image = np.full((1200,850,3),255,np.uint8) + cv2.putText(image,'1',(1,600),cv2.FONT_HERSHEY_SIMPLEX,.7,(190,190,190),2) + self.assertTrue(detect_page_content(image,dpi=150)[0].any()) + + def test_isolated_circle_diagram_remains_content(self): + image = np.full((1200,850,3),255,np.uint8) + cv2.circle(image,(400,600),20,(30,30,30),2) + self.assertTrue(detect_page_content(image,dpi=150)[0].any()) + + def test_solid_ink_dots_do_not_establish_content_at_multiple_resolutions(self): + for dpi in (150,200,300): + px=dpi/25.4 + mask=np.zeros((round(100*px),round(70*px)),np.uint8) + cv2.ellipse(mask,(round(30*px),round(45*px)), + (round(.6*px),round(.8*px)),0,0,360,1,-1) + self.assertFalse(_blank_content_support(mask>0,dpi)[0].any()) + + def test_compact_dot_is_kept_with_neighbouring_question_number(self): + mask=np.zeros((400,300),np.uint8) + cv2.putText(mask,'7',(100,200),cv2.FONT_HERSHEY_SIMPLEX,.7,1,2) + mask[194:200,117:122]=1 + retained,_=_blank_content_support(mask>0,150) + self.assertTrue(retained[194:200,117:122].all()) + + def test_large_isolated_numbers_and_diagram_near_bottom_remain_content(self): + for text in ('1','7','x','?','2','16'): + with self.subTest(text=text): + image=np.full((1200,850,3),255,np.uint8) + cv2.putText(image,text,(300,1150),cv2.FONT_HERSHEY_SIMPLEX, + 1.5,(30,30,30),2) + self.assertTrue(detect_page_content(image,200)[0].any()) + image=np.full((1200,850,3),255,np.uint8) + cv2.circle(image,(400,1140),20,(30,30,30),2) + self.assertTrue(detect_page_content(image,200)[0].any()) + + def test_content_analysis_recognizes_blank_grid_without_changing_crop_fallback(self): + image = np.full((1200,850,3),255,np.uint8) + for y in range(20,1200,20): + cv2.line(image,(0,y),(849,y),(65,65,65),1) + for x in range(10,850,20): + cv2.line(image,(x,0),(x,1199),(65,65,65),1) + for y in (110,370,630,890,1130): + cv2.ellipse(image,(25,y),(12,20),0,0,300,(35,35,35),2) + before = detect_bounds(image,dpi=150) + mask,result = detect_page_content(image,dpi=150) + self.assertTrue(result['content_ruling_confirmed']) + self.assertFalse(mask.any()) + self.assertEqual(detect_bounds(image,dpi=150),before) + + def test_content_analysis_preserves_pencil_only_writing(self): + image = np.full((1200,850,3),255,np.uint8) + cv2.putText(image,'pencil',(100,600),cv2.FONT_HERSHEY_SIMPLEX, + 1,(195,195,195),2) + mask,result = detect_page_content(image,dpi=150) + self.assertTrue(mask[560:610,90:250].any()) + self.assertEqual(result['content_state'],'content') + + def test_faint_scanner_border_does_not_count_as_pencil(self): + image = np.full((1200,850,3),255,np.uint8) + cv2.rectangle(image,(1,1),(848,300),(195,195,195),1) + mask,_ = detect_page_content(image,dpi=150) + self.assertFalse(mask.any()) + + def test_faint_unruled_diagram_is_retained(self): + image = np.full((1200,850,3),255,np.uint8) + cv2.rectangle(image,(100,100),(750,1100),(195,195,195),2) + mask,_ = detect_page_content(image,dpi=150) + self.assertTrue(mask.any()) + + def test_content_analysis_preserves_faint_grid_note_and_fraction_bar(self): + image = self.dark_grid() + cv2.putText(image,'pencil',(100,1100),cv2.FONT_HERSHEY_SIMPLEX, + .7,(190,190,190),2) + cv2.line(image,(150,1020),(700,1020),(0,0,0),3) + mask,_ = detect_page_content(image,dpi=150) + self.assertTrue(mask[1070:1110,90:250].any()) + self.assertTrue(mask[1018:1023,150:700].any()) + def test_fast_background_blur_matches_opencv_pixel_for_pixel(self): rng = np.random.default_rng(42) for shape in ((97,131), (241,319)):