Automatic page splitting

This commit is contained in:
sebastien committed 2026-11-03 22:08:36 +01:00
1 parent d4404e4c27
commit 78f3e4b2f0
13 files changed
+1959 -40

No files matched your search

+377
View File
@@ -0,0 +1,377 @@
"""Experimental, non-destructive spread splitting with an HTML review gallery.
python -m copienator.auto_page_split INPUT OUTPUT [--workers 4]
No booklet reordering, secondary cuts, or GUI changes. Unknown formats and
uncertain content are retained. OUTPUT must not already exist.
"""
from __future__ import annotations
import argparse
from concurrent.futures import ProcessPoolExecutor
import hashlib
import html
import json
import multiprocessing
from pathlib import Path
from urllib.parse import quote
import cv2
import numpy as np
import pymupdf
from PIL import Image, ImageDraw
from copienator.ink_detection import detect_bounds, detect_page_content
def classify_format(width: float, height: float) -> str:
"""Dimensions are displayed PDF points; tolerate 5% scan size error."""
w, h = width * 25.4 / 72, height * 25.4 / 72
if abs(min(w, h) / 210 - 1) < .05 and abs(max(w, h) / 297 - 1) < .05:
return 'a4'
if abs(w / 420 - 1) < .05 and abs(h / 297 - 1) < .05:
return 'a3'
return 'unknown'
def analyse_ink(rgb: np.ndarray, dpi: float, *, spread: bool = False) -> tuple[np.ndarray, dict]:
"""Use normal rognage on logical pages; colour cleanup only for overflow.
Blankness uses the shared crop detector's content-analysis entry point.
No independent weak-pixel veto or secondary artifact detector is applied.
"""
if not spread:
return detect_page_content(rgb,dpi)
masks = {}
result = detect_bounds(rgb, dpi, content_masks=masks,
colour_cleanup=True, edge_mark_cleanup=False)
ink = masks['foreground']
# For ownership, follow connected faint pen strokes beyond rognage's local
# 2 mm recovery radius. Do not restore page-wide connected ruling networks.
candidate = ink | masks['weak_candidates']
n,labels,stats,_ = cv2.connectedComponentsWithStats(candidate.astype(np.uint8),8)
seeded = np.zeros(n,bool)
seeded[np.unique(labels[ink])] = True
seeded[0] = False
giant = (stats[:,2] > .65*ink.shape[1]) & (stats[:,3] > .4*ink.shape[0])
seeded[giant] = False
ink = ink | seeded[labels]
return ink, result
def _analyse_page_full(rgb: np.ndarray, dpi: float, kind: str) -> dict:
"""Keep spread geometry analysis separate from logical-page blankness."""
h, w = rgb.shape[:2]
boundaries = [0, w//2, w] if kind == 'a3' else [0, w]
content = np.zeros((h,w), bool)
evidence = []
for x0,x1 in zip(boundaries,boundaries[1:]):
foreground, result = analyse_ink(rgb[:,x0:x1],dpi)
content[:,x0:x1] = foreground
evidence.append(result)
ink = analyse_ink(rgb,dpi,spread=True)[0] if kind == 'a3' else content
plan = plan_masks(ink,dpi,kind,content_ink=content,rgb=rgb)
for output,result in zip(plan['outputs'],evidence):
output['crop_analysis'] = result
return plan
def analyse_page(rgb: np.ndarray, dpi: float, kind: str) -> dict:
"""Analyse a raster with the standard coarse-to-fine splitter."""
from copienator.fast_page_split import analyse_page_fast
return analyse_page_fast(rgb, dpi, kind)
def _fold_regions(ink: np.ndarray, credible: np.ndarray, dpi: float,
rgb: np.ndarray | None = None) -> tuple[list[dict], np.ndarray]:
"""Grow local word groups towards page anchors, stopping at supported text.
Broader gaps are allowed for ownership evidence without merging both pages
into one giant line or changing output bounds to include the anchor word.
"""
import heapq
px = dpi/25.4
h,w = ink.shape
middle = w//2
ignored = []
if rgb is not None:
# Remove weak thin ruling *before* word grouping: otherwise a nearby
# dark word can lend its colour/anchor evidence to a pale grid fragment.
count,raw_labels,raw_stats,_ = cv2.connectedComponentsWithStats(ink.astype(np.uint8),8)
ink = ink.copy()
for i,(x,y,bw,bh,area) in enumerate(raw_stats[1:],1):
if not (x < middle+2*px and x+bw > middle-2*px and bh<px and bw>4*bh):
continue
region = raw_labels[y:y+bh,x:x+bw]==i
values = rgb[y:y+bh,x:x+bw][region].astype(np.int16)
if np.median(np.ptp(values,axis=1))<=80 and np.median(values.min(axis=1))>=110:
ink[y:y+bh,x:x+bw][region] = False
ignored.append(dict(owner='artifact',box=[int(x),int(y),int(x+bw),int(y+bh)],
component_id=0,anchor_boxes=[None,None],anchor_distance_mm=[None,None],reason='weak-thin-ruling'))
kernel = np.ones((max(1,round(.5*px)),max(1,round(3*px))),np.uint8)
joined = cv2.dilate(ink.astype(np.uint8),kernel)
n,labels,stats,_ = cv2.connectedComponentsWithStats(joined,8)
boxes = np.zeros((n,4),np.int32)
areas = np.bincount(labels[ink],minlength=n)
pen = np.zeros(n,bool)
for i in range(1,n):
x,y,bw,bh,_ = stats[i]
yy,xx = np.nonzero((labels[y:y+bh,x:x+bw]==i) & ink[y:y+bh,x:x+bw])
boxes[i] = [x+xx.min(),y+yy.min(),x+xx.max()+1,y+yy.max()+1]
if rgb is not None and len(xx)>=.4*px*px:
values=rgb[y+yy,x+xx].astype(np.int16)
# Between the crop detector's weak (60) and strong (100) colour
# thresholds: pale blue ruling must not become an ownership anchor.
pen[i] = np.median(np.ptp(values,axis=1))>80 or np.median(values.min(axis=1))<110
support = credible & ink
# The blank classifier deliberately ignores short isolated pen marks.
# A two-dimensional letter/terminal stroke can still link to its page here.
support |= pen[labels] & ink
left_limit,right_limit = max(0,round(middle-12*px)),min(w,round(middle+12*px))
left_counts = np.bincount(labels[:,:left_limit][support[:,:left_limit]],minlength=n)
right_counts = np.bincount(labels[:,right_limit:][support[:,right_limit:]],minlength=n)
anchors = np.c_[left_counts>=max(2,.2*px*px),right_counts>=max(2,.2*px*px)]
backed = np.bincount(labels[support],minlength=n)>=max(2,.12*px*px)
backed[0] = False
near = max(1,round(2*px))
roots = np.unique(labels[:,max(0,middle-near):middle+near])
regions = ignored
for root in roots:
if root == 0:continue
x0,y0,x1,y1 = boxes[root]
if x1 < middle-near or x0 > middle+near:continue
thin_unanchored = (not anchors[root].any() and
((not pen[root] and y1-y0<px and x1-x0>4*(y1-y0)) or areas[root]<.4*px*px))
if thin_unanchored:
regions.append(dict(owner='artifact',box=boxes[root].tolist(),component_id=int(root),anchor_boxes=[None,None],anchor_distance_mm=[None,None]))
continue
distances = np.full(n,np.inf)
distances[root]=0
queue=[(0.,int(root))]
reached=[np.inf,np.inf]
evidence=[None,None]
# Only supported writing may carry a connection onwards. Otherwise
# rows of grid fragments could form a bridge all the way into an answer.
allowed = backed.copy();allowed[root]=True
while queue:
cost,node=heapq.heappop(queue)
if cost != distances[node] or cost>45*px:continue
if anchors[node].any():
for side in (0,1):
if anchors[node,side] and cost<reached[side]:
reached[side]=cost;evidence[side]=boxes[node].tolist()
continue
a,b,c,d = boxes[node]
dx=np.maximum(0,np.maximum(boxes[:,0]-c,a-boxes[:,2]))
dy=np.maximum(0,np.maximum(boxes[:,1]-d,b-boxes[:,3]))
# Wide mathematical spaces are plausible between strong pen
# regions; unsupported shadows get only the shorter word gap.
gap = 32*px if pen[root] else 8*px
neighbours=np.flatnonzero(allowed & (dx<=gap) & (dy<=2*px))
for other in neighbours:
step=float(np.hypot(dx[other],3*dy[other]))+1
new=cost+step
if new<distances[other]:
distances[other]=new;heapq.heappush(queue,(new,int(other)))
if not np.isfinite(reached).any():owner='artifact'
elif abs(reached[0]-reached[1])<=px:owner='uncertain'
else:owner='left' if reached[0]<reached[1] else 'right'
regions.append(dict(owner=owner,box=[int(x0),int(y0),int(x1),int(y1)],
component_id=int(root),anchor_boxes=evidence,anchor_distance_mm=[round(c/px,2) if np.isfinite(c) else None for c in reached]))
return regions,labels
def plan_masks(ink: np.ndarray, dpi: float, kind: str, *,
content_ink: np.ndarray | None = None,
rgb: np.ndarray | None = None,
_remaining_content: dict | None = None) -> dict:
"""Produce rectangles in image coordinates and explicit review reasons."""
h, w = ink.shape
px = dpi / 25.4
middle = w // 2
reasons = []
crossings = []
left_end, right_start = middle, middle
owned = (ink if content_ink is None else content_ink).copy()
left_owned, right_owned = owned[:, :middle].copy(), owned[:, middle:].copy()
if kind == 'a3':
pad = round(2*px)
crossings,group_labels = _fold_regions(ink,owned,dpi,rgb)
for crossing in crossings:
owner = crossing['owner']
x0,y0,x1,y1 = crossing['box']
if owner == 'artifact':
continue
if owner == 'left':
left_end = max(left_end,min(w,x1+pad))
if x1>middle:
right_owned[y0:y1,max(x0,middle)-middle:x1-middle] &= ~(ink[y0:y1,max(x0,middle):x1] & (group_labels[y0:y1,max(x0,middle):x1]==crossing['component_id']))
elif owner == 'right':
right_start = min(right_start,max(0,x0-pad))
if x0<middle:
left_owned[y0:y1,x0:min(x1,middle)] &= ~(ink[y0:y1,x0:min(x1,middle)] & (group_labels[y0:y1,x0:min(x1,middle)]==crossing['component_id']))
else:
left_end = max(left_end,min(w,x1+pad))
right_start = min(right_start,max(0,x0-pad))
reasons.append('ambiguous-centre-writing')
regions = [('left', [0,0,left_end,h], left_owned),
('right', [right_start,0,w,h], right_owned)]
else:
regions = [('whole', [0,0,w,h], owned)]
if kind == 'unknown':
reasons.append('unknown-format-or-orientation')
outputs = []
if _remaining_content is not None:
_remaining_content.update({side: content for side,_,content in regions})
for side, rect, content in regions:
state = 'content' if content.any() else 'blank'
if kind == 'unknown':
state = 'uncertain'
outputs.append(dict(side=side, rect_px=rect, state=state,
keep=state != 'blank', ink_pixels=int(content.sum())))
return dict(format=kind, size_px=[w,h], outputs=outputs, crossings=crossings,
review_reasons=sorted(set(reasons)),
extensions_mm=[round((left_end-middle)/px,2),
round((middle-right_start)/px,2)] if kind == 'a3' else [0,0])
def save_preview(rgb: np.ndarray, plan: dict, path: Path) -> None:
im = Image.fromarray(rgb)
im.thumbnail((1200, 1000))
sx, sy = im.width/plan['size_px'][0], im.height/plan['size_px'][1]
overlay = Image.new('RGBA', im.size)
draw = ImageDraw.Draw(overlay)
for output in plan['outputs']:
x0,y0,x1,y1 = output['rect_px']
box = (x0*sx,y0*sy,x1*sx-1,y1*sy-1)
colour = '#187941' if output['side'] == 'left' else '#a832a6'
draw.rectangle(box, outline=colour, width=3,
fill=(255,30,30,65) if not output['keep'] else None)
for crossing in plan['crossings']:
x0,y0,x1,y1 = crossing['box']
draw.rectangle((x0*sx,y0*sy,x1*sx,y1*sy), outline='#999999' if crossing['owner']=='artifact' else '#e88b00', width=2)
Image.alpha_composite(im.convert('RGBA'), overlay).convert('RGB').save(path, quality=90)
def process_pdf(job: tuple[str, str, int]) -> list[dict]:
source_s, destination_s, dpi = job
source, destination = Path(source_s), Path(destination_s)
cv2.setNumThreads(1)
digest = hashlib.sha256(source.read_bytes()).hexdigest()
records = []
with pymupdf.open(source) as doc, pymupdf.open() as result:
for i, page in enumerate(doc):
rotation = page.rotation
kind = classify_format(page.rect.width, page.rect.height)
from copienator.fast_page_split import analyse_pdf_page_fast
plan,rgb = analyse_pdf_page_fast(page,dpi,kind)
pixel_width,pixel_height = plan['size_px']
plan.update(file=source.name, source_page=i+1, source_sha256=digest,
source_rotation=rotation, source_cropbox=list(page.cropbox))
preview = f'{source.stem}-{i+1:03}.jpg'
plan['preview'] = f'review/{preview}'
save_preview(rgb, plan, destination/'review'/preview)
# Normalize through a full-page PDF form: remove_rotation() loses
# non-default CropBoxes on quarter-turn pages in some MuPDF builds.
w,h = page.rect.width,page.rect.height
normalized = pymupdf.open()
flat = normalized.new_page(width=w,height=h)
page.set_rotation(0)
if page.get_contents():
flat.show_pdf_page(flat.rect,doc,i,rotate=-rotation)
for output in plan['outputs']:
x0,y0,x1,y1 = output['rect_px']
clip = pymupdf.Rect(x0/pixel_width*w,y0/pixel_height*h,
x1/pixel_width*w,y1/pixel_height*h)
output['crop_points'] = list(clip)
if output['keep']:
target = result.new_page(width=clip.width,height=clip.height)
# Truly empty PDF pages have no content stream to show.
if flat.get_contents():
target.show_pdf_page(target.rect,normalized,0,clip=clip)
output['output_page'] = len(result)
records.append(plan)
normalized.close()
# An entirely blank copy has no output PDF; its report is still kept.
if len(result):
result.save(destination/'pdfs'/source.name, garbage=3, deflate=True)
if hashlib.sha256(source.read_bytes()).hexdigest() != digest:
raise RuntimeError(f'Source changed during analysis: {source}')
output_path = destination/'pdfs'/source.name
expected = sum(o['keep'] for r in records for o in r['outputs'])
if expected:
with pymupdf.open(output_path) as check:
if len(check) != expected or any(p.rect.is_empty for p in check):
raise RuntimeError(f'Invalid output: {output_path}')
return records
def run(source: Path, destination: Path, *, dpi: int = 200, workers: int = 1) -> dict:
if dpi < 100 or workers < 1:
raise ValueError('Use dpi >= 100 and workers >= 1')
sources = sorted(source.glob('*.pdf')) if source.is_dir() else [source]
if not sources or any(not p.is_file() or p.suffix.lower() != '.pdf' for p in sources):
raise ValueError('No input PDFs found')
# Refuse reuse: this makes accidental source replacement impossible.
destination.mkdir(parents=True, exist_ok=False)
(destination/'review').mkdir()
(destination/'pdfs').mkdir()
jobs = [(str(p),str(destination),dpi) for p in sources]
records = []
if workers == 1:
batches = map(process_pdf, jobs)
for batch in batches:
records.extend(batch)
print(f'{batch[0]["file"]}: {len(batch)} scanned pages', flush=True)
else:
with ProcessPoolExecutor(max_workers=workers,
mp_context=multiprocessing.get_context('spawn')) as pool:
for batch in pool.map(process_pdf,jobs):
records.extend(batch)
print(f'{batch[0]["file"]}: {len(batch)} scanned pages', flush=True)
return write_report(records,destination,len(sources))
def write_report(records: list[dict], destination: Path, file_count: int) -> dict:
"""Rebuild the gallery from saved decisions without reanalysing scans."""
summary = dict(files=file_count, source_pages=len(records),
a3=sum(r['format']=='a3' for r in records), a4=sum(r['format']=='a4' for r in records),
output_pages=sum(o['keep'] for r in records for o in r['outputs']),
blank_removed=sum(not o['keep'] for r in records for o in r['outputs']),
review_pages=sum(bool(r['review_reasons']) for r in records),
extended_spreads=sum(any(r['extensions_mm']) for r in records))
(destination/'report.json').write_text(json.dumps(dict(summary=summary,pages=records),indent=2)+'\n')
cards = []
for row in records:
name = html.escape(row['file'])
states = ', '.join(f'{o["side"]}: {o["state"]}' for o in row['outputs'])
reasons = ', '.join(row['review_reasons']) or 'No review flag'
links = ' · '.join(f'<a href="pdfs/{quote(row["file"])}#page={o["output_page"]}">{o["side"]} PDF</a>'
for o in row['outputs'] if o['keep'])
cards.append(f'<article class="{"review" if row["review_reasons"] else "ok"}">'
f'<h2>{name} / {row["source_page"]}</h2><p>{row["format"]} | {states}</p>'
f'<p>Extensions L/R: {row["extensions_mm"]} mm | {reasons}</p><p>{links}</p>'
f'<a href="{html.escape(row["preview"])}"><img loading="lazy" src="{html.escape(row["preview"])}"></a></article>')
(destination/'index.html').write_text('<!doctype html><meta charset="utf-8">'
'<title>Automatic page splitting - review</title><style>'
'body{font:16px system-ui;margin:24px;background:#eee}main{display:grid;grid-template-columns:repeat(auto-fit,minmax(550px,1fr));gap:16px}'
'article{background:white;padding:16px}article.review{border-top:5px solid #e88b00}img{width:100%}h2{font-size:18px}p{font-size:13px}</style>'
'<h1>Automatic page splitting - experimental</h1><p>Source order; no booklet reordering. '
'Green: left output. Purple: right/whole output. Orange: centre writing. Red shading: removed blank.</p>'
f'<pre>{html.escape(json.dumps(summary,indent=2))}</pre><main>'+''.join(cards)+'</main>')
return summary
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('input',type=Path)
parser.add_argument('output',type=Path)
parser.add_argument('--dpi',type=int,default=200)
parser.add_argument('--workers',type=int,default=1)
args = parser.parse_args(argv)
print(json.dumps(run(args.input,args.output,dpi=args.dpi,workers=args.workers),indent=2))
return 0
if __name__ == '__main__':
raise SystemExit(main())
@@ -0,0 +1,183 @@
"""Publish automatic splits into an evaluation, preserving sources and review data."""
from __future__ import annotations
import hashlib
import json
import shutil
import tempfile
from pathlib import Path
from copienator import CliError, ExitCode
from copienator.auto_page_split import run as analyse_batch
from copienator.copy_errors import clear_copy_error, copy_errors, mark_copy_error
from copienator.filesystem import publish_file_sets
from copienator.orientation import report_path
def decision_path(workspace, stem):
return workspace.metadata_dir / "page_splitting" / f"{stem}.json"
def load_decisions(workspace, source):
"""Only reuse decisions for the exact preserved input and published output."""
try:
data = json.loads(decision_path(workspace, source.stem).read_text())
if data["source_sha256"] != hashlib.sha256(source.read_bytes()).hexdigest():
return None
output = workspace.copies_dir / source.name
digest = (
hashlib.sha256(output.read_bytes()).hexdigest() if output.exists() else None
)
if digest != data["output_sha256"]:
return None
pages = data["pages"]
if not isinstance(pages, list) or not all(
isinstance(row, dict)
and type(row.get("source_page")) is int
and isinstance(row.get("outputs"), list)
and all(
isinstance(o, dict)
and o.get("side") in {"left", "right", "whole"}
and type(o.get("keep")) is bool
and isinstance(o.get("crop_points"), list)
and len(o["crop_points"]) == 4
for o in row["outputs"]
)
for row in pages
):
return None
return pages
except (OSError, ValueError, KeyError, TypeError, AttributeError):
return None
def run(workspace, inputs: list[Path], *, workers=4):
if workers < 1:
raise CliError(
"Le nombre de traitements parallèles doit être positif.",
ExitCode.INVALID_ARGUMENTS,
)
if not inputs:
print("Aucune copie PDF à traiter.", flush=True)
return ExitCode.SUCCESS
if len({p.name for p in inputs}) != len(inputs):
raise CliError(
"Plusieurs copies portent le même nom.", ExitCode.INVALID_ARGUMENTS
)
copy_errors(workspace) # Validate existing review metadata before publishing.
workspace.runs_dir.mkdir(parents=True, exist_ok=True)
run_dir = Path(tempfile.mkdtemp(prefix="page-split-", dir=workspace.runs_dir))
snapshots = run_dir / "inputs"
snapshots.mkdir()
for source in inputs:
shutil.copy2(source, snapshots / source.name)
result = run_dir / "result"
print(f"Découpage automatique de {len(inputs)} copie(s).", flush=True)
summary = analyse_batch(snapshots, result, workers=workers)
records = json.loads((result / "report.json").read_text())["pages"]
stages = {
name: run_dir / name for name in ("copies", "originals", "origins", "decisions")
}
for stage in stages.values():
stage.mkdir()
for source in inputs:
rows = [r for r in records if r["file"] == source.name]
expected = rows[0]["source_sha256"]
if hashlib.sha256(source.read_bytes()).hexdigest() != expected:
raise CliError(
f"{source.name} a changé pendant l’analyse. Aucun PDF remplacé."
)
shutil.copy2(snapshots / source.name, stages["originals"] / source.name)
for directory, label in (
(workspace.copies_dir, "previous_copies"),
(workspace.original_copies_dir, "previous_originals"),
):
prior = directory / source.name
if prior.exists():
backup = run_dir / label
backup.mkdir(exist_ok=True)
shutil.copy2(prior, backup / source.name)
output = result / "pdfs" / source.name
digest = (
hashlib.sha256(output.read_bytes()).hexdigest() if output.exists() else None
)
if digest:
shutil.copy2(output, stages["copies"] / source.name)
origins = [
{"origin": r["source_page"], "crop_points": o["crop_points"]}
for r in rows
for o in r["outputs"]
if o["keep"]
]
if digest:
(stages["origins"] / (source.stem + ".json")).write_text(
json.dumps({"version": 1, "sha256": digest, "pages": origins}),
encoding="utf-8",
)
(stages["decisions"] / (source.stem + ".json")).write_text(
json.dumps(
{
"version": 1,
"source_sha256": expected,
"output_sha256": digest,
"pages": rows,
}
),
encoding="utf-8",
)
names = [p.name for p in inputs]
for source in inputs:
if source.read_bytes() != (snapshots / source.name).read_bytes():
raise CliError(
f"{source.name} a changé pendant l’analyse. Aucun PDF remplacé."
)
publish_file_sets(
{
workspace.copies_dir: (stages["copies"], names),
workspace.original_copies_dir: (stages["originals"], []),
workspace.metadata_dir / "page_origins": (
stages["origins"],
[p.stem + ".json" for p in inputs],
),
workspace.metadata_dir / "page_splitting": (stages["decisions"], []),
}
)
for source in inputs:
if (
source.parent.resolve()
not in {
workspace.copies_dir.resolve(),
workspace.original_copies_dir.resolve(),
}
and source.read_bytes() == (snapshots / source.name).read_bytes()
):
# Root scans become preserved originals, as in the manual workflow.
source.unlink()
rows = [r for r in records if r["file"] == source.name]
output = workspace.copies_dir / source.name
clear_copy_error(workspace, output, source="orientation")
report_path(workspace, source.stem).unlink(missing_ok=True)
reasons = {reason for r in rows for reason in r["review_reasons"]}
if reasons or not output.exists():
mark_copy_error(
workspace,
output,
"Découpage automatique à vérifier manuellement."
if reasons
else "Toutes les pages semblent vides : vérifier l’original.",
source="page_splitter",
)
else:
clear_copy_error(workspace, output, source="page_splitter")
print(
f"Terminé : {summary['output_pages']} pages conservées, {summary['blank_removed']} pages vides retirées. "
"Ordre des pages inchangé.",
flush=True,
)
print(
f"Rapport et sauvegardes : {run_dir}\n"
"Choisissez Manuellement pour vérifier ou modifier les propositions.",
flush=True,
)
return ExitCode.SUCCESS
+100 -23
View File
@@ -27,7 +27,7 @@ from copienator import (
)
from copienator.platform import launch_pdf_arranger
from copienator.copy_errors import marked_copy_paths, clear_copy_error
from copienator.orientation import save_provenance, report_path
from copienator.orientation import save_provenance, report_path, provenance_path
# --- Constants ---
# Conversion factor: 1 cm to points (1 inch = 2.54 cm, 72 points = 1 inch)
@@ -49,7 +49,7 @@ def _temporary_sibling(path: Path, purpose: str) -> Path:
def commit_processed_pdf(
workspace: EvaluationWorkspace,
original_path: Path,
generated_path: Path,
generated_path: Path | None,
) -> Path:
"""Commit a processed copy and its original backup with rollback."""
backup_path = workspace.original_copies_dir / original_path.name
@@ -73,8 +73,9 @@ def commit_processed_pdf(
backup_replaced = True
if output_path.exists():
output_path.replace(saved_output)
generated_path.replace(output_path)
output_replaced = True
if generated_path is not None:
generated_path.replace(output_path)
output_replaced = True
if original_path.resolve() not in {
backup_path.resolve(),
output_path.resolve(),
@@ -120,6 +121,10 @@ class PDFPreviewer:
try:
self.doc = pymupdf.open(self.pdf_path)
self.original_page_ids = {page.xref: index + 1 for index, page in enumerate(self.doc)}
from copienator.commands.automatic_page_splitter import load_decisions
saved = load_decisions(self.workspace, self.pdf_path)
self.automatic_plans = {r['source_page']: r for r in saved} if saved else {}
self.automatic_review = bool(self.automatic_plans)
except (OSError, RuntimeError, ValueError) as e:
self.failed = True
self._temporary_directory.cleanup()
@@ -176,6 +181,12 @@ class PDFPreviewer:
f"{fmt('add_secondary_splitter')}: add secondary line, {fmt('delete_secondary_splitter')}: delete newest (arrows move newest; main line red, active secondary orange)\n"
f"'{fmt('reverse_pages')}': reverse page order and restart from the new first page\n"
)
if self.automatic_review:
instructions += (
"\nPropositions automatiques : la touche suivante valide le choix affiché. "
"Croix rouge : page retirée.\n"
"Déplacer une ligne reprend le découpage manuel pour cette page."
)
self.info_label = tk.Label(master, text=instructions, justify=tk.LEFT)
self.info_label.pack(pady=5, side=tk.TOP)
@@ -255,6 +266,25 @@ class PDFPreviewer:
self.current_line_x = page.rect.width / 2
self.current_rotation = 0
self.secondary_splitters = []
origin = getattr(self, 'original_page_ids', {}).get(page.xref, self.current_page_index+1)
plan = getattr(self, 'automatic_plans', {}).get(origin)
self.current_automatic_outputs = plan['outputs'] if plan else None
def _automatic_outputs(self):
"""Express saved asymmetric crops in the current preview orientation."""
outputs = getattr(self, 'current_automatic_outputs', None)
if outputs is None:
return None
flipped = (self.current_rotation+self.file_rotation+self.global_rotation) % 360
if not flipped:
return outputs
width, height = self.doc[self.current_page_index].rect.width, self.doc[self.current_page_index].rect.height
result = []
for output in outputs:
x0,y0,x1,y1 = output['crop_points']
result.append(dict(output, side={'left':'right','right':'left'}.get(output['side'], 'whole'),
crop_points=[width-x1,height-y1,width-x0,height-y0]))
return result
def load_page(self):
"""Loads and displays the current page on the canvas, scaled to fit."""
@@ -296,9 +326,20 @@ class PDFPreviewer:
# --- Draw Line and Rotate ---
draw = ImageDraw.Draw(img)
# The line position is scaled by the same zoom factor
line_x_scaled = self.current_line_x * self.current_zoom
draw.line([(line_x_scaled, 0), (line_x_scaled, pix.height)],
fill="red", width=3)
proposed = self._automatic_outputs()
if proposed is not None:
for output in proposed:
x0,y0,x1,y1 = output['crop_points']
box = tuple(v*self.current_zoom for v in (x0,y0,x1,y1))
draw.rectangle(box, outline='green' if output['side']=='left' else 'purple', width=3)
if not output['keep']:
draw.line((box[0],box[1],box[2],box[3]), fill='red', width=3)
draw.line((box[0],box[3],box[2],box[1]), fill='red', width=3)
self.page_label.config(text=self.page_label.cget('text')+' — Proposition automatique (croix rouge : page retirée)')
else:
line_x_scaled = self.current_line_x * self.current_zoom
draw.line([(line_x_scaled, 0), (line_x_scaled, pix.height)],
fill="red", width=3)
for index, position in enumerate(self.secondary_splitters):
x = position * self.current_zoom
active = index == len(self.secondary_splitters) - 1
@@ -353,6 +394,7 @@ class PDFPreviewer:
def add_secondary_splitter(self, event=None):
"""Start a new line in the widest remaining interval of this page."""
width = self.doc.load_page(self.current_page_index).rect.width
self.current_automatic_outputs = None
boundaries = sorted({0, width, self.current_line_x, *self.secondary_splitters})
left, right = max(zip(boundaries, boundaries[1:]), key=lambda pair: pair[1] - pair[0])
self.secondary_splitters.append((left + right) / 2)
@@ -364,6 +406,7 @@ class PDFPreviewer:
self.load_page()
def _move_active_line(self, delta):
self.current_automatic_outputs = None
width = self.doc.load_page(self.current_page_index).rect.width
if self.secondary_splitters:
self.secondary_splitters[-1] = min(width, max(0, self.secondary_splitters[-1] + delta))
@@ -415,13 +458,19 @@ class PDFPreviewer:
# Reload the canvas to show the new page
self.load_page()
def confirm_and_next_page(self, event=None, keep="both"):
def confirm_and_next_page(self, event=None, keep=None):
"""Saves the settings for the current page and moves to the next."""
proposed = self._automatic_outputs()
if keep is None:
sides = {o['side'] for o in proposed if o['keep']} if proposed is not None else {'left','right'}
keep = ('as_is' if 'whole' in sides else 'both' if len(sides)==2
else next(iter(sides)) if sides else 'none')
self.page_settings.append({
"line_x": self.current_line_x,
"secondary_splitters": list(self.secondary_splitters),
"rotation": self.current_rotation,
"keep": keep
"keep": keep,
"automatic_outputs": proposed,
})
self.current_page_index += 1
@@ -442,12 +491,18 @@ class PDFPreviewer:
def finish_and_process(self) -> bool:
"""Render and transactionally install the processed PDF."""
try:
self.split_pdf()
self.reorder_pdfs()
self.concate_files()
output = commit_processed_pdf(self.workspace, self.pdf_path, self.final_file)
save_provenance(self.workspace, output, self.output_origins)
if all(s['keep']=='none' for s in self.page_settings):
self.doc.close()
output = commit_processed_pdf(self.workspace, self.pdf_path, None)
provenance_path(self.workspace, output.stem).unlink(missing_ok=True)
else:
self.split_pdf()
self.reorder_pdfs()
self.concate_files()
output = commit_processed_pdf(self.workspace, self.pdf_path, self.final_file)
save_provenance(self.workspace, output, self.output_origins)
clear_copy_error(self.workspace, output, source='orientation')
clear_copy_error(self.workspace, output, source='page_splitter')
report_path(self.workspace, output.stem).unlink(missing_ok=True)
except Exception as exc: # noqa: BLE001 - interactive boundary
self.failed = True
@@ -514,15 +569,27 @@ class PDFPreviewer:
line_x = settings['line_x']
rotation_settings = settings['rotation']
keep = settings['keep']
# Bake source rotation and the preview's extra rotation into content,
# so all cut coordinates refer to the displayed page.
page.remove_rotation()
page.set_rotation((rotation_settings + self.file_rotation + self.global_rotation) % 360)
page.remove_rotation()
# Normalize the displayed page without losing an offset CropBox.
# This matches the automatic export's coordinate system.
source_rotation = page.rotation
width, height = page.rect.width, page.rect.height
normalized = pymupdf.open()
flat = normalized.new_page(width=width, height=height)
page.set_rotation(0)
if page.get_contents():
flat.show_pdf_page(flat.rect, self.doc, i, rotate=-source_rotation)
flat.set_rotation((rotation_settings+self.file_rotation+self.global_rotation) % 360)
flat.remove_rotation()
sides = []
if keep == "as_is":
sides.append((0, width, self.split_filename_left(i)))
elif settings.get('automatic_outputs') is not None:
for output in settings['automatic_outputs']:
side = output['side']
if keep == 'both' or keep == side:
x0,_,x1,_ = output['crop_points']
filename = self.split_filename_left(i) if side=='left' else self.split_filename_right(i)
sides.append((x0,x1,filename))
else:
if keep in {"both", "left"} and line_x > 0:
sides.append((0, line_x, self.split_filename_left(i)))
@@ -539,10 +606,12 @@ class PDFPreviewer:
for x0, x1 in zip(boundaries, boundaries[1:]):
clip = pymupdf.Rect(x0, 0, x1, height)
target = output.new_page(width=clip.width, height=clip.height)
target.show_pdf_page(target.rect, self.doc, i, clip=clip)
if flat.get_contents():
target.show_pdf_page(target.rect, normalized, 0, clip=clip)
origins.append({'origin': origin, 'crop_points': list(clip)})
output.save(output_path)
self.fragment_origins[str(output_path)] = origins
normalized.close()
self.doc.close()
print(f"\nProcessing complete. Files are in '{self.split_dir}' directory.")
@@ -564,7 +633,7 @@ class PDFPreviewer:
psk = ps[i]['keep']
# Si c'est une copie double (on s'assure qu'on a bien 2 pages consécutives modifiables)
if psk in ["both", "right", "left", "none"] and i < len(ps)-1 and ps[i+1]['keep'] in ["both", "right", "left", "none"]:
if not getattr(self, 'automatic_review', False) and psk in ["both", "right", "left", "none"] and i < len(ps)-1 and ps[i+1]['keep'] in ["both", "right", "left", "none"]:
# 1. Page de garde (Extérieur Droit)
if ps[i]['keep'] in ["both", "right"]:
@@ -678,11 +747,17 @@ def _selected_inputs(
]
else:
candidates = list_pdf_files(directory, workspace)
if target == workspace.root and not candidates:
candidates = list_pdf_files(workspace.original_copies_dir, workspace)
return list(reversed(candidates))
def run(workspace: EvaluationWorkspace, target: Path, *, marked: bool = False) -> ExitCode:
def run(workspace: EvaluationWorkspace, target: Path, *, marked: bool = False,
automatic: bool = False, workers: int = 4) -> ExitCode:
inputs = list(reversed(marked_copy_paths(workspace, originals=True))) if marked else _selected_inputs(workspace, target)
if automatic:
from copienator.commands.automatic_page_splitter import run as run_automatic
return run_automatic(workspace, list(reversed(inputs)), workers=workers)
if not inputs:
print(f"No PDF files found in {target}")
return ExitCode.SUCCESS
@@ -695,6 +770,8 @@ def run(workspace: EvaluationWorkspace, target: Path, *, marked: bool = False) -
def build_parser() -> argparse.ArgumentParser:
parser = target_parser("Interactively split and reorder scanned PDF pages")
parser.add_argument("--marked", action="store_true", help="Process only copies flagged during margin review")
parser.add_argument("--automatic", action="store_true", help="Split automatically and save editable proposals without reordering")
parser.add_argument("--workers", type=int, default=4, help="Copies analysed simultaneously in automatic mode")
return parser
@@ -703,7 +780,7 @@ def main(argv: Sequence[str] | None = None) -> int:
def handle(args: argparse.Namespace) -> ExitCode:
workspace, target = workspace_from_target(args)
return run(workspace, target, marked=args.marked)
return run(workspace, target, marked=args.marked, automatic=args.automatic, workers=args.workers)
return execute(parser, argv, handle)
+3 -3
View File
@@ -20,7 +20,7 @@ def _validate(value) -> dict[str, dict[str, str]]:
if isinstance(flags, str):
flags = {"cutleft": flags}
if (not isinstance(flags, dict) or not flags
or any(step not in {"cutleft", "plotting", "orientation"} or not isinstance(reason, str)
or any(step not in {"cutleft", "plotting", "orientation", "page_splitter"} or not isinstance(reason, str)
for step, reason in flags.items())):
raise CliError("Invalid copy_errors.json: invalid step flags")
normalized[name] = dict(flags)
@@ -35,7 +35,7 @@ def copy_errors(workspace: EvaluationWorkspace) -> dict[str, str]:
def mark_copy_error(workspace: EvaluationWorkspace, pdf: Path, reason: str,
*, source: str = "cutleft") -> None:
if source not in {"cutleft", "plotting", "orientation"}:
if source not in {"cutleft", "plotting", "orientation", "page_splitter"}:
raise ValueError(f"Unknown copy flag source: {source}")
def update(errors):
normalized = _validate(errors)
@@ -47,7 +47,7 @@ def mark_copy_error(workspace: EvaluationWorkspace, pdf: Path, reason: str,
def clear_copy_error(workspace: EvaluationWorkspace, pdf: Path,
*, source: str = "cutleft") -> None:
if source not in {"cutleft", "plotting", "orientation"}:
if source not in {"cutleft", "plotting", "orientation", "page_splitter"}:
raise ValueError(f"Unknown copy flag source: {source}")
if not _path(workspace).exists():
return
+276
View File
@@ -0,0 +1,276 @@
"""Standard coarse-to-fine splitting: cheap positive content gate, precise fold.
The whole-page gate only establishes obvious writing. All uncertain/sparse pages
use the reference blank detector. Full-resolution fold processing is restricted
to a central strip; edge-reaching overflow and ambiguous ownership fall back to
the complete reference analysis. Full-spread analysis remains an internal fallback.
"""
from __future__ import annotations
import cv2
import numpy as np
import pymupdf
from copienator.auto_page_split import analyse_ink, _analyse_page_full, plan_masks
def quick_page_content(rgb, dpi, target_dpi=75, *, native=True, _fallback=None):
"""Positive-only coarse gate. Ambiguous/sparse pages use full detection."""
h, w = rgb.shape[:2]
small = cv2.resize(
rgb,
(round(w * target_dpi / dpi), round(h * target_dpi / dpi)),
interpolation=cv2.INTER_AREA,
)
px = target_dpi / 25.4
lo = small.min(axis=2)
ch = small.max(axis=2) - lo
gray = cv2.cvtColor(small, cv2.COLOR_RGB2GRAY)
strong = (ch > 100) | (gray < 95)
n, lab, stats, _ = cv2.connectedComponentsWithStats(strong.astype("uint8"), 8)
ok = np.zeros(n, bool)
x, y, bw, bh, area = stats[1:].T
# Only compact interior pen components can establish an obvious written page.
ok[1:] = (
(area >= 0.25 * px * px)
& (bw >= 0.35 * px)
& (bh >= 0.5 * px)
& (bw < 20 * px)
& (bh < 15 * px)
& (x > 0.09 * small.shape[1])
& (x + bw < 0.91 * small.shape[1])
)
selected = ok[lab]
grouped = cv2.dilate(
selected.astype("uint8"),
np.ones((max(1, round(px)), max(1, round(3 * px))), np.uint8),
)
ng, groups, gs, _ = cv2.connectedComponentsWithStats(grouped, 8)
comp_group = np.zeros(n, np.int32)
ys, xs = np.nonzero(selected)
comp_group[lab[ys, xs]] = groups[ys, xs]
counts = np.bincount(comp_group[ok], minlength=ng)
amounts = np.bincount(groups[selected], minlength=ng)
good = (
(counts >= 4)
& (amounts >= 3 * px * px)
& (gs[:, 2] >= 8 * px)
& (gs[:, 3] >= 2 * px)
)
good[0] = False
if good.any():
# Retain all strong ink for ownership support; the gate alone is interior-only.
return (
strong
if native
else cv2.resize(
strong.astype("uint8"), (w, h), interpolation=cv2.INTER_NEAREST
)
> 0
), True
return (_fallback() if _fallback is not None else analyse_ink(rgb, dpi)[0]), False
def _a4_plan(mask, gate, h, w, dpi, gate_dpi):
count = int(mask.sum())
return dict(
format="a4",
size_px=[w, h],
outputs=[
dict(
side="whole",
rect_px=[0, 0, w, h],
state="content" if count else "blank",
keep=bool(count),
ink_pixels=count,
content_dpi=gate_dpi if gate else dpi,
)
],
crossings=[],
review_reasons=[],
extensions_mm=[0, 0],
analysis=dict(
mode="fast",
early_written=[gate],
fold_fraction=0,
gate_dpi=gate_dpi,
fallback=None,
),
)
def analyse_page_fast(rgb, dpi, kind, *, fold_fraction=0.1, gate_dpi=75):
"""Fast analysis for an existing raster image."""
if kind not in {"a3", "a4"}:
return _analyse_page_full(rgb, dpi, kind)
gate_dpi = min(gate_dpi, dpi)
h, w = rgb.shape[:2]
if kind == "a4":
mask, gate = quick_page_content(rgb, dpi, gate_dpi, native=True)
return _a4_plan(mask, gate, h, w, dpi, gate_dpi)
middle = w // 2
halves = []
early = []
for a, b in ((0, middle), (middle, w)):
mask, gate = quick_page_content(rgb[:, a:b], dpi, gate_dpi, native=True)
halves.append(mask)
early.append(gate)
return _assemble_fast(
halves,
early,
h,
w,
dpi,
fold_fraction,
gate_dpi,
lambda x0, x1: rgb[:, x0:x1],
lambda: _analyse_page_full(rgb, dpi, kind),
)
def _assemble_fast(
halves, early, h, w, dpi, fold_fraction, gate_dpi, render_roi, reference
):
middle = w // 2
half = max(1, round(w * fold_fraction / 2))
x0 = max(0, middle - half)
x1 = min(w, middle + half)
roi = render_roi(x0, x1)
content = np.zeros(roi.shape[:2], bool)
for i, (a, b) in enumerate(((0, middle), (middle, w))):
lo = max(a, x0)
hi = min(b, x1)
mask = halves[i]
ys = np.arange(h) * mask.shape[0] // h
xs = (np.arange(lo, hi) - a) * mask.shape[1] // (b - a)
content[:, lo - x0 : hi - x0] = mask[ys[:, None], xs[None, :]]
ink, _ = analyse_ink(roi, dpi, spread=True)
remaining = {}
plan = plan_masks(
ink, dpi, "a3", content_ink=content, rgb=roi, _remaining_content=remaining
)
# If a proposed overflow reaches the strip's edge or ownership remains
# ambiguous, use the complete reference analysis rather than truncate it.
margin = round(2 * dpi / 25.4)
if (
plan["review_reasons"]
or plan["outputs"][0]["rect_px"][2] >= x1 - x0 - margin
or plan["outputs"][1]["rect_px"][0] <= margin
):
plan = reference()
plan["analysis"] = dict(mode="fast", fallback="full-spread")
return plan
for i, (a, b) in enumerate(((0, middle), (middle, w))):
mask = halves[i]
side = "left" if i == 0 else "right"
if not early[i]:
lo = max(a, x0)
hi = min(b, x1)
mask[:, lo - a : hi - a] = remaining[side]
count = int(mask.sum())
else:
# Exact count of the implied nearest-neighbour full-resolution mask,
# without allocating it. Coarse masks establish obvious content only.
wy = np.diff(
np.ceil(np.arange(mask.shape[0] + 1) * h / mask.shape[0]).astype(
"int64"
)
)
wx = np.diff(
np.ceil(np.arange(mask.shape[1] + 1) * (b - a) / mask.shape[1]).astype(
"int64"
)
)
count = int(np.sum(mask * wy[:, None] * wx[None, :]))
o = plan["outputs"][i]
o["ink_pixels"] = count
o["keep"] = count > 0
o["state"] = "content" if count else "blank"
o["rect_px"] = (
[0, 0, o["rect_px"][2] + x0, h]
if i == 0
else [o["rect_px"][0] + x0, 0, w, h]
)
for c in plan["crossings"]:
c["box"][0] += x0
c["box"][2] += x0
for box in c.get("anchor_boxes", []):
if box is not None:
box[0] += x0
box[2] += x0
plan["size_px"] = [w, h]
plan["analysis"] = dict(
mode="fast",
gate_dpi=gate_dpi,
fold_fraction=fold_fraction,
early_written=early,
fallback=None,
)
return plan
def analyse_pdf_page_fast(page, dpi, kind, *, fold_fraction=0.1, gate_dpi=75):
"""Render a small overview and the precise fold directly from the PDF.
Full-resolution halves are rendered only when the positive overview gate
cannot establish writing. Raster clips are aligned to the same integer
pixel grid as a complete reference render, including rotated CropBoxes.
"""
gate_dpi = min(gate_dpi, dpi)
bounds = (page.rect * pymupdf.Matrix(dpi / 72, dpi / 72)).irect
h, w = bounds.height, bounds.width
coarse_pix = page.get_pixmap(
dpi=round(gate_dpi), colorspace=pymupdf.csRGB, alpha=False
)
coarse = np.frombuffer(coarse_pix.samples, np.uint8).reshape(
coarse_pix.height, coarse_pix.width, 3
)
def render(x0, x1):
rect = pymupdf.Rect(x0 * 72 / dpi, 0, x1 * 72 / dpi, page.rect.height)
pix = page.get_pixmap(
dpi=round(dpi), clip=rect, colorspace=pymupdf.csRGB, alpha=False
)
data = np.frombuffer(pix.samples, np.uint8).reshape(pix.height, pix.width, 3)
out = np.full((h, x1 - x0, 3), 255, np.uint8)
left = max(x0, pix.x)
top = max(0, pix.y)
right = min(x1, pix.x + pix.width)
bottom = min(h, pix.y + pix.height)
out[top:bottom, left - x0 : right - x0] = data[
top - pix.y : bottom - pix.y, left - pix.x : right - pix.x
]
return out
def reference():
return _analyse_page_full(render(0, w), dpi, kind)
if kind not in {"a3", "a4"}:
return reference(), coarse
if kind == "a4":
mask, gate = quick_page_content(
coarse,
gate_dpi,
gate_dpi,
native=True,
_fallback=lambda: analyse_ink(render(0, w), dpi)[0],
)
return _a4_plan(mask, gate, h, w, dpi, gate_dpi), coarse
middle = w // 2
cm = coarse.shape[1] // 2
halves = []
early = []
for a, b, ca, cb in ((0, middle, 0, cm), (middle, w, cm, coarse.shape[1])):
mask, gate = quick_page_content(
coarse[:, ca:cb],
gate_dpi,
gate_dpi,
native=True,
_fallback=lambda a=a, b=b: analyse_ink(render(a, b), dpi)[0],
)
halves.append(mask)
early.append(gate)
plan = _assemble_fast(
halves, early, h, w, dpi, fold_fraction, gate_dpi, render, reference
)
plan["analysis"]["rendering"] = "direct-pdf-roi"
return plan, coarse
+192 -5
View File
@@ -45,12 +45,15 @@ def _paper_blur(gray: np.ndarray, sigma: float) -> np.ndarray:
return blurred.astype(np.uint8)
def _large_blank_ink(gray: np.ndarray, clean: np.ndarray, dpi: float):
def _large_blank_ink(gray: np.ndarray, clean: np.ndarray, dpi: float, *,
content_only: bool = False):
"""Refine confirmed ruling, only accepting substantial extra blank margins.
Short directional openings tolerate locally bent/broken paper lines. A
physical component-size threshold rejects their remaining tiny fragments.
This is deliberately limited to already-confirmed ruled paper.
Crop refinement is limited to already-confirmed ruled paper. Content-only
analysis also uses this stroke filter on seedless plain pages and can return
an empty mask; the normal crop path deliberately preserves its input then.
"""
px = dpi / 25.4
height, width = gray.shape
@@ -120,6 +123,14 @@ def _large_blank_ink(gray: np.ndarray, clean: np.ndarray, dpi: float):
overlap = np.bincount(lab[strong_ruling > 0], minlength=count)
bars &= overlap < st[:, 4]*.5
refined |= bars[lab]
if content_only:
# A crop must preserve its input when there is no reliable content.
# A blank-page classifier needs that empty result instead. Recover the
# same nearby original strokes without restoring remote paper artifacts.
radius = max(1, round(2*px))
nearby = cv2.dilate(refined.astype(np.uint8),
cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(2*radius+1,2*radius+1))) > 0
return refined | ((clean > 0) & nearby), len(holes), True
ys = np.flatnonzero(np.any(refined, axis=1))
original = np.flatnonzero(np.any(clean, axis=1))
if not len(ys) or not len(original):
@@ -138,8 +149,171 @@ def _large_blank_ink(gray: np.ndarray, clean: np.ndarray, dpi: float):
return result, len(holes), changed
def _blank_content_support(content: np.ndarray, dpi: float) -> tuple[np.ndarray, dict]:
"""Require a glyph, neighbouring strokes, or a substantial diagram.
A few remote dots and a short loose curve do not establish a written page.
Grouping protects fragmented pencil letters without a page-wide ink quota.
This evidence filter is for blank classification, never margin cropping.
"""
px = dpi/25.4
n,labels,stats,_ = cv2.connectedComponentsWithStats(content.astype(np.uint8),8)
if n == 1:
return content,dict(content_supported_groups=0,content_unsupported_pixels=0)
radius = max(1,round(1.5*px))
joined = cv2.dilate(content.astype(np.uint8),
cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(2*radius+1,2*radius+1)))
_,groups = cv2.connectedComponents(joined,8)
ys,xs = np.nonzero(content)
component_groups = np.zeros(n,np.int32)
component_groups[labels[ys,xs]] = groups[ys,xs]
area = stats[:,4]
bw,bh = stats[:,2],stats[:,3]
density = area/np.maximum(1,bw*bh)
letter = ((area >= .8*px*px) & (np.minimum(bw,bh) >= .6*px)
& (density >= .18))
# A dense sub-2 mm ink spot is not a glyph on its own. Keep it eligible
# for a supported group (decimal points, dots over letters, punctuation).
compact_spot = np.zeros(n,bool)
spot_candidates = np.flatnonzero((density > .55) & (np.maximum(bw,bh) < 2*px))
for i in spot_candidates:
if i == 0:
continue
x,y,ww,hh,_ = stats[i]
yy,xx = np.nonzero(labels[y:y+hh,x:x+ww] == i)
hull_area = cv2.contourArea(cv2.convexHull(np.c_[xx,yy].astype(np.int32)))
# Small x's have empty wedges inside their convex hull; solid spots
# fill it. Pixel-centre hull area can be smaller than the pixel count.
compact_spot[i] = area[i] >= .9*max(1.,hull_area)
letter &= ~compact_spot
substantial = (area >= 4*px*px) & (np.maximum(bw,bh) >= 5*px)
# A lone short smear at the physical top/bottom edge can pass both area
# tests. Require its pixels to be almost collinear before discounting it;
# normal digits, letters and closed diagrams have broader stroke geometry.
# Neighbouring meaningful strokes still support it through the group rule.
near_edge = (stats[:,1] < 15*px) | (stats[:,1]+bh > content.shape[0]-15*px)
edge_smear = np.zeros(n,bool)
candidates = np.flatnonzero(near_edge & (area < 6*px*px)
& (np.maximum(bw,bh) < 8*px)
& (np.minimum(bw,bh) > 2*px) & (density < .35))
for i in candidates:
if i == 0:
continue
x,y,ww,hh,_ = stats[i]
yy,xx = np.nonzero(labels[y:y+hh,x:x+ww] == i)
values = np.linalg.eigvalsh(np.cov(np.c_[xx,yy].T))
edge_smear[i] = values[0] < .05*values[1]
letter &= ~edge_smear
substantial &= ~edge_smear
credible = area >= .2*px*px
credible[0] = False
counts = np.bincount(component_groups[credible],minlength=int(groups.max())+1)
totals = np.bincount(component_groups,weights=area,minlength=len(counts))
supported = (counts >= 2) & (totals >= 1.2*px*px)
supported[component_groups[letter | substantial]] = True
supported[0] = False
retained = supported[groups] & content
return retained,dict(content_supported_groups=int(supported.sum()),
content_unsupported_pixels=int(content.sum()-retained.sum()),
content_compact_spots=int(compact_spot[1:].sum()),
content_edge_smears=int(edge_smear.sum()))
def detect_page_content(rgb: np.ndarray, dpi: float = 200) -> tuple[np.ndarray, dict]:
"""Expose rognage's content evidence on a single logical page.
Normal crop behaviour stays unchanged. On confirmed ruled paper, use its
large-blank refinement even when that refinement finds no content, instead of
the crop-specific fallback that keeps the original page. No separate weak-mark
detector is used. The returned mask is in the input image coordinates.
"""
content_masks = {}
paper_geometry = {}
result = detect_bounds(rgb,dpi,content_masks=content_masks,_paper_geometry=paper_geometry)
content = content_masks['foreground']
gray = cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY)
# Use the same deskew and paper-family confirmation as the crop pipeline.
h,w = gray.shape
if paper_geometry:
matrix,size,deskewed = (paper_geometry[key] for key in ('matrix','size','deskewed'))
else:
angle = _skew(gray,angle_step=.5)
matrix = cv2.getRotationMatrix2D((w/2,h/2),angle,1)
corners = cv2.transform(np.array([[[0,0],[w,0],[0,h],[w,h]]],float),matrix)[0]
origin = np.floor(corners.min(axis=0))
size = np.ceil(corners.max(axis=0)-origin).astype(int)
matrix[:,2] -= origin
deskewed = cv2.warpAffine(gray,matrix,tuple(size),borderValue=255)
binary = cv2.adaptiveThreshold(deskewed,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY_INV,max(15,int(dpi/5)|1),9)
horizontal,nh = _ruling(binary,True)
vertical,nv = _ruling(binary,False)
confirmed = ((nh and _edge_connected_family(binary,horizontal,True)) or
(nv and _edge_connected_family(binary,vertical,False)))
if not confirmed:
# Slight local warp breaks one-pixel pale grid lines. Confirm the same
# periodic, edge-reaching family with a small physical-width tolerance.
tolerance = max(3,round(.3*dpi/25.4)|1)
tolerant = cv2.dilate(binary,np.ones((tolerance,tolerance),np.uint8))
th,tnh = _ruling(tolerant,True)
tv,tnv = _ruling(tolerant,False)
confirmed = ((tnh and _edge_connected_family(tolerant,th,True)) or
(tnv and _edge_connected_family(tolerant,tv,False)))
result['content_tolerant_ruling'] = bool(confirmed)
# With no seeds, the same coherent-stroke filter can recover pencil-only
# writing on plain paper. Arbitrary weak pixels alone are not a veto.
refine = bool(confirmed) or not content.any()
result['content_refinement'] = refine
result['content_ruling_confirmed'] = bool(confirmed)
if refine:
clean = cv2.warpAffine(content.astype(np.uint8),matrix,tuple(size),
flags=cv2.INTER_NEAREST,borderValue=0)
refined,holes,_ = _large_blank_ink(deskewed,clean,dpi,content_only=True)
content = cv2.warpAffine(refined.astype(np.uint8),cv2.invertAffineTransform(matrix),
(w,h),flags=cv2.INTER_NEAREST,borderValue=0)>0
result['content_holes'] = holes
result['content_pixels'] = int(content.sum())
if not confirmed and not content.any():
# The directional refinement can erase very thin pencil letters on
# plain paper. Reuse the crop pipeline's adaptive foreground instead,
# requiring coherent letter-sized components rather than arbitrary
# weak pixels. This fallback is confined to seedless/plain pages.
weak = _foreground(gray,dpi,9,np.zeros_like(gray))
weak,_ = _content_mask(weak,dpi)
count,labels,stats,_ = cv2.connectedComponentsWithStats(weak,8)
keep = np.zeros(count,bool)
px = dpi/25.4
keep[1:] = ((stats[1:,4] >= .5*px*px)
& (stats[1:,2] >= px) & (stats[1:,3] >= px))
# Faint scanner borders can be one huge connected component. They are
# not coherent writing merely because their bounding box is large.
x,y,bw,bh,area = stats[1:].T
sparse = area / np.maximum(1,bw*bh) < .1
edge_frame = (((bw > .9*w) & ((y < px) | (y+bh > h-px)))
| ((bh > .9*h) & ((x < px) | (x+bw > w-px))))
edge_frame |= ((bw > .8*w) & ((x < 2*px) | (x+bw > w-2*px)))
keep[1:] &= ~(sparse & edge_frame)
# A faint narrow scanner-edge band is not a pencil letter. Require
# both edge placement and pale neutral appearance, not position alone.
for i in np.flatnonzero(keep[1:])+1:
xx,yy,ww,hh,_ = stats[i]
if ww < 3*px and (xx < 2*px or xx+ww > w-2*px) and hh > ww:
values = rgb[yy:yy+hh,xx:xx+ww][labels[yy:yy+hh,xx:xx+ww]==i]
if np.median(values.min(axis=1)) > 205 and np.median(np.ptp(values.astype(np.int16),axis=1)) < 30:
keep[i] = False
content = keep[labels]
result['content_pixels'] = int(content.sum())
result['content_pencil_recovery'] = bool(content.any())
content,support = _blank_content_support(content,dpi)
result.update(support)
result['content_pixels'] = int(content.sum())
result['content_state'] = 'content' if content.any() else 'blank'
return content,result
def _neutral_paper_foreground(gray: np.ndarray, chroma: np.ndarray, dpi: float,
*, diagnostics: dict | None = None):
*, diagnostics: dict | None = None,
geometry: dict | None = None):
"""Clean confirmed dark ruling before it can seed whole pages.
Returns None for ordinary ink components or unconfirmed paper geometry.
@@ -163,6 +337,8 @@ def _neutral_paper_foreground(gray: np.ndarray, chroma: np.ndarray, dpi: float,
size = np.ceil(corners.max(axis=0)-origin).astype(int)
matrix[:, 2] -= origin
deskewed = cv2.warpAffine(gray, matrix, tuple(size), borderValue=255)
if geometry is not None:
geometry.update(matrix=matrix,size=size,deskewed=deskewed)
background = _paper_blur(deskewed, dpi/8)
normalized = cv2.divide(deskewed, np.maximum(background,1), scale=255)
block = max(15, int(dpi/5) | 1)
@@ -220,7 +396,9 @@ def _seed_mask(mask: np.ndarray, px: float, *, paper_nearby=None,
def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6,
min_crop_mm: float = 5, *, diagnostics: dict | None = None,
colour_cleanup: bool = False, edge_mark_cleanup: bool = True) -> dict:
colour_cleanup: bool = False, edge_mark_cleanup: bool = True,
content_masks: dict | None = None,
_paper_geometry: dict | None = None) -> dict:
"""Locate strong ink, recover adjacent faint strokes, and retain padding.
Neutral punched-hole shadows usually have neither sufficient chroma nor
@@ -237,6 +415,9 @@ def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6,
Optional diagnostics are written to a separate caller-owned dictionary.
Its masks use the original image coordinates and never influence decisions
or enter the JSON-serializable return value.
content_masks requests only foreground and weak-candidate masks, avoiding
diagnostic component explanations and intermediate paper-mask exports.
"""
px = dpi/25.4
h, w = rgb.shape[:2]
@@ -259,7 +440,8 @@ def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6,
diagnostics['masks'] = {}
diagnostics['thresholds'] = []
masks = diagnostics['masks'] if diagnostics is not None else None
cleaned, paper_info = _neutral_paper_foreground(gray, chroma, dpi, diagnostics=masks)
cleaned, paper_info = _neutral_paper_foreground(gray, chroma, dpi, diagnostics=masks,
geometry=_paper_geometry)
if cleaned is not None:
# Adaptive neutral cleanup includes faint paper fragments. Such pixels
# need local contrast or substantial darkness before they seed a crop.
@@ -279,6 +461,9 @@ def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6,
masks['isolated_edge_marks'] = np.zeros((h,w), bool)
weak = ((chroma > 60) | ((gray < 175) & (residual > 25))).astype(np.uint8)
weak[suppressed] = 0
if content_masks is not None:
content_masks.clear()
content_masks.update(foreground=np.zeros((h,w),bool),weak_candidates=weak>0)
if masks is not None:
masks['strong_seeds'] = np.zeros((h,w), bool)
masks['foreground'] = np.zeros((h,w), bool)
@@ -329,6 +514,8 @@ def detect_bounds(rgb: np.ndarray, dpi: float = 200, padding_mm: float = 6,
masks['isolated_edge_marks'] |= discarded[labels]
seeded_labels = remaining
retained = seeded_labels[labels]
if content_masks is not None:
content_masks['foreground'] |= retained
if masks is not None:
masks['strong_seeds'] |= seeds & retained
ys = np.flatnonzero(np.any(retained, axis=1))
+9 -6
View File
@@ -93,7 +93,6 @@ def ruling_suppression(rgb: np.ndarray, dpi: float):
matrix[:,2] -= origin
image = cv2.warpAffine(rgb, matrix, tuple(size), borderValue=(255,255,255))
dark = 255-image
dark16 = dark.astype(np.int16)
darkness = _maximum(dark)
binary = cv2.adaptiveThreshold(_minimum(image), 255,
cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV,
@@ -119,7 +118,7 @@ def ruling_suppression(rgb: np.ndarray, dpi: float):
support = _maximum(model) > 9
if count:
support &= bands > 0
close = _maximum(np.abs(dark16-model.astype(np.int16))) < 40
close = _maximum(cv2.absdiff(dark,model)) < 40
sample = support & (darkness > 9) & close & ~thick
# Neutral, red, green and blue line families are modelled separately so
# a red margin rule cannot be mistaken for unusually dark blue writing.
@@ -128,7 +127,10 @@ def ruling_suppression(rgb: np.ndarray, dpi: float):
classes[second <= third] = 1
classes[(first <= second) & (first <= third)] = 0
classes[cv2.subtract(_maximum(model),_minimum(model)) < 25] = 3
credible = np.zeros(binary.shape, bool)
# Rejected classes have ceiling -1, below every uint8 darkness value.
# Compute each class ceiling once, then test pixels by lookup instead
# of subtracting and splitting full RGB images for every colour class.
ceilings = np.full((4,3),-1,np.int16)
for colour in range(4):
selected = sample & (classes == colour)
if selected.sum() < 100:
@@ -143,9 +145,10 @@ def ruling_suppression(rgb: np.ndarray, dpi: float):
# An isolated long fraction bar must not define its own paper colour.
# The upper quantile admits darker major rules within an established
# family while rejecting excess ink on most of its pixels.
ceiling = np.floor(np.quantile(dark[selected],.99,axis=0)+25).astype(np.int16)
credible |= ((classes == colour)
& (_maximum(dark16-ceiling) <= 0))
ceilings[colour] = np.floor(np.quantile(dark[selected],.99,axis=0)+25).astype(np.int16)
credible = ((dark[:,:,0] <= ceilings[classes,0])
& (dark[:,:,1] <= ceilings[classes,1])
& (dark[:,:,2] <= ceilings[classes,2]))
match = close & credible & (darkness > 9) & ~thick
removed |= support & match
protected |= support & ~match & (binary > 0)
+12 -2
View File
@@ -259,8 +259,13 @@ def build_workflow(show_personal_steps: bool) -> list[StepDefinition]:
"page_splitter",
"Prétraitement des copies",
"Séparer et réordonner les pages",
"Ouvre l’outil interactif de découpage A3 vers A4. La cible peut être un dossier ou un PDF.",
(python("default", "Séparation des pages", "page-split"),),
"Choisissez le découpage manuel ou automatique. Le traitement automatique sépare les A3, "
"préserve les débordements et retire les pages vides, sans réordonner les pages. "
"Le mode manuel permet ensuite de vérifier et modifier ses propositions.",
(
python("default", "Manuellement", "page-split"),
python("automatic", "Automatiquement", "page-split", fixed_args=("--automatic",)),
),
arguments=(
arg_target(),
ArgumentSpec(
@@ -270,6 +275,11 @@ def build_workflow(show_personal_steps: bool) -> list[StepDefinition]:
"--marked",
help="Limite l’opération aux copies signalées dans l’interface au lieu de traiter toutes les copies.",
),
ArgumentSpec(
"workers", "Copies traitées en parallèle", "int", "--workers", default=4,
variants=("automatic",),
help="Nombre de copies analysées simultanément pendant le découpage automatique.",
),
),
artifacts=("Copies", "Copies Originales"),
),
+241
View File
@@ -0,0 +1,241 @@
# Experimental automatic page splitting
## Python GUI
The “Séparer et réordonner les pages” step now offers **Manuellement** and
**Automatiquement**. Automatic processing runs in the background using the
standard splitter, publishes PDFs in `Copies`, and preserves source scans in
`Copies Originales`. It retains source-page order. The parallel-copy setting
appears only for automatic processing. Existing single-copy and marked-copy
selection work in both modes.
Choose Manuellement afterward to review the saved proposals against the
preserved original. The editor displays independent overlapping output crops
and crosses out removed blank pages. Advancing accepts the displayed decision;
the existing keep/discard shortcuts override it. Moving a splitter resumes
manual splitting on that page. Reviewing automatic proposals preserves their
source-page order; the original manual workflow retains its booklet ordering.
Proposals are reused only while both source and output fingerprints still match.
Each automatic run saves its inputs, report/gallery, and previous outputs under
`.copienator/runs/page-split-*`. All PDFs are analysed before publication, and
PDFs and provenance are published together with rollback on installation errors.
An entirely blank copy keeps its preserved source and receives a manual-review
flag instead of an invalid empty output PDF.
The equivalent CLI command is:
```sh
python -m copienator page-split EVALUATION --automatic --workers 4
```
Omit `--automatic` for the interactive editor. The standalone review command
below remains available for experiments without installing results into `Copies`.
Run from the repository root, on original scans rather than already split copies:
```sh
python -m copienator.auto_page_split 'Interro07/Copies Originales' tmp/split-review --workers 4
```
The output directory must not already exist. Input PDFs are never overwritten.
This command is deliberately separate from the GUI and from the interactive
`page-split` command. It does not reorder booklets or use secondary splitters.
Outputs follow source-page order, left then right. They must not replace the
current processed evaluation without review and the appropriate later ordering.
## Outputs
- `index.html`: source-page previews, proposed boundaries, blank decisions,
uncertainty reasons, and links to exported PDFs.
- `pdfs/`: clipped PDFs retaining source scan data, without rasterizing output.
- `report.json`: decisions, original file hashes, original rotations/CropBoxes,
source page numbers, displayed clip coordinates, and output page numbers.
- `review/`: JPEG previews. Green marks the left output, purple the right/whole
output, orange the detected centre groups; red shading marks a blank removal.
A wholly blank copy has no output PDF, but still has report entries. An interrupted
run may leave partial outputs; use a new output directory for a subsequent run.
## Decisions
Displayed physical dimensions distinguish A3 landscape spreads from A4 sheets,
with a 5% tolerance. A4 landscape sheets are also retained whole. Unexpected
sizes and portrait A3 pages are retained whole and flagged. PDF rotation is
respected; handwriting orientation is not inferred.
The standard splitter renders a 75 dpi overview, accepts clearly written pages,
and analyses a central 10% strip at the requested 200 dpi for overflow ownership.
Sparse logical pages use `ink_detection.detect_page_content` at full resolution.
Ambiguous ownership, overflow reaching the strip edge, and unexpected geometry
use the internal full-page fallback. Coloured-ruling cleanup prevents grid
fragments from linking text across the fold.
That shared entry point runs normal rognage with its default artifact filtering.
On confirmed ruled paper it exposes the existing large-blank refinement's retained
content, including an empty result. Ordinary cropping deliberately falls back to
the original mask when refinement is empty; the new entry point does not. Normal
cropping behaviour and defaults remain unchanged. The same coherent-stroke filter
checks seedless pages for plausible pencil writing. On plain paper, the existing
adaptive foreground extractor recovers letter-sized pencil components; faint
page-spanning scanner borders do not qualify. It does not retain a page
merely because unrelated faint pixels survive. There is no separate weak-pixel
veto or splitter-specific hole-removal algorithm.
Small initial stroke groups near the midpoint are followed towards supported
writing inside either page. Connected faint strokes can recover beyond the
cropper's local 2 mm radius, while page-wide ruling networks are rejected. The
ownership search allows larger horizontal gaps between strong pen regions,
penalises vertical gaps, and stops at a page anchor rather than joining entire
lines across both pages. Pale thin ruling is filtered before word grouping.
An isolated group with no page anchor is treated as a fold artifact. Comparable
support from both pages still produces an explicit review flag. Output rectangles
cover the actual terminal strokes plus 2 mm padding, not the expanded search
area. Each side's boundary remains independent. The regression suite and reviewed
real examples are documented in `cropping-test/page-splitting/README.md`.
Blank decisions use logical-page evidence after subtracting actual strokes
attributed to the opposite side. The report includes the crop analysis and the
content-refinement result for each output. A spread artifact that is absent from
the logical page's credible content cannot by itself veto blank removal.
## Limitations and interpretation
The report's flags are heuristic reasons, not calibrated confidence scores.
Repeated-hole detection and ruling suppression can be ambiguous around unusual
handwriting. Extremely faint isolated writing can escape detection. Horizontal
grouping can miss disconnected parts of long formulae or merge adjacent blocks.
Ambiguous cases intentionally duplicate content rather than choosing an owner.
Full-height rectangular overlap can include neighbouring text at other heights.
Blank removal is intentionally conservative. Do not interpret a retained page
as proof it contains student work. In particular, residual scan marks can keep
otherwise blank pages. Interro07's saved page-origin mappings provide a useful
comparison with earlier processing, but are not independent handwritten ground
truth and do not validate overflow attribution.
## Verification
```sh
python -m unittest tests.test_auto_page_split tests.test_ink_detection tests.test_crop_blank_margins tests.test_page_splitter_reverse tests.test_page_splitter_secondary -q
```
Tests cover both overflow directions at once, disconnected letters, ambiguous
ownership, opposite-side blank attribution, weak scan noise, pencil-only writing,
blank grids with binding holes, faint grid notes and fraction bars, A4/unknown
sizes, all four PDF rotations with CropBox
offsets, rendered export fidelity, input preservation, existing-output refusal,
and wholly blank copies.
## Updated full Interro07 batch (2026-11-03)
The current batch is `tmp/auto-page-split-interro07-support-20261103/`.
All 47 original PDFs were processed: 214 scanned pages, with 206 A3 spreads
and 8 A4 sheets. The output contains **347 logical pages**, removing **73 blank
halves**. All 345 pages kept by the earlier workflow remain kept. The two
additional retained halves contain exercise headings (Copie08 source 6/left and
Copie11 source 4/left), consistent with the reviewed content policy.
Exactly seven keep/remove decisions change from the preceding 66-removal batch,
matching the seven reviewed difficult blank fixtures. Their source halves were
visually checked; the two heading exports were rendered with Poppler and checked.
Original hashes, output page counts and displayed dimensions were verified.
`comparison.json` records the comparisons. The 26 source-page overlap review
flags remain; blank detection results do not validate all overflow attribution.
## Shared rognage trial (2026-11-02)
The current results are in `tmp/auto-page-split-interro07-shared-rognage/`.
Blank detection now uses the shared crop content analysis on each logical page.
The splitter-specific weak-pixel veto and hole cleanup have been removed.
Normal cropping still uses its original empty-refinement fallback.
Against Interro07's saved page-origin mappings, the updated detector removes
66 of the 75 previously omitted halves (88%), compared with 5 in the initial
prototype. All 345 previously retained pages are preserved. Nine previously
omitted halves remain kept, including two very pale pages retained by the
pencil-writing recovery check. These are conservative keep decisions; no
previously retained page was removed. The output has 354 logical pages.
All proposed removals were reviewed in source-page contact sheets. The final
PDF page counts, dimensions, and source hashes are checked, with results in
`comparison.json`. The 44 regression tests pass, covering normal crop behaviour,
blank ruled sheets with holes, faint writing, scanner borders, large faint
diagrams, overlap geometry, and rotation. This validates blank detection against
the existing Interro07 workflow; it is not an exhaustive annotation of handwriting
or overflow ownership. Centre-binding artifacts can still cause false overlap
proposals. No GUI integration or change to the existing processed PDFs was made.
## Initial Interro07 trial (2026-11-02; superseded)
The final trial is in `tmp/auto-page-split-interro07-final/` (ignored by Git).
It processed all 47 original PDFs: 214 scanned pages, comprising 206 A3 spreads
and 8 A4 sheets. Original-file hashes were verified unchanged. The exported
PDFs contain 415 logical pages. There are 47 extended spreads, including 21 with
both boundaries extended, and 69 source pages flagged for review. These counts
describe proposals, not validated correct overflow detections.
Comparison with the saved `Interro07/.copienator/page_origins/` mappings:
| Decision | Logical pages |
| --- | ---: |
| Previously retained, still retained | 345 |
| Previously retained, now removed | 0 |
| Previously omitted, now removed | 5 |
| Previously omitted, retained as uncertain | 58 |
| Previously omitted, classified as content | 12 |
All five proposed removals were visually inspected and appear blank. Selected
overflow previews and Poppler-rendered exports were also inspected. The coloured
ruling cleanup fixed a false 88.65 mm left extension on Copie36, source page 4:
the final 4.45 mm extension follows the equation at the centre. Binding artifacts
still generate false extensions elsewhere. No independent exhaustive annotation
of overflow ownership was made; preserving previously retained pages does not
prove that every stroke is correctly assigned or unclipped.
The 38 focused and existing regression tests pass, including pixel comparisons
of exported pages with rotated, offset-CropBox sources. The prototype is **not
ready for unattended GUI integration**. The next priorities are better blank-page
recall without erasing faint work, and better separation of binding artifacts
from centre handwriting. `comparison.json` lists the 70 keep/remove disagreements
for targeted follow-up. Neither the GUI nor the existing processed copies changed.
## Standard coarse-to-fine analysis
The former fast mode is now the only command-line analysis path. It renders a
75 dpi overview and only a central 10% strip at the requested 200 dpi. A positive
content gate accepts obvious writing; sparse or ambiguous pages still use full
resolution content detection. Ambiguous ownership or overflow reaching the strip
boundary uses the complete reference analysis. Unexpected geometry keeps the
reference path. Exported scan data remains at its original resolution.
```sh
python -m copienator.auto_page_split 'Interro07/Copies Originales' tmp/fast-review \
--workers 4
```
The full Interro07 run in `tmp/auto-page-split-interro07-fast-final-20261103/`
takes 37.87 s including exports/previews, retains 347 pages and removes 73 blank
halves. All 420 keep/remove decisions agree with a fresh reference analysis.
Seven output boundaries differ: five by one pixel, and two inspected differences
in ruling/padding without observed loss of writing. This mode is not pixel-identical
to the reference. There is no analysis-mode selector; full-spread analysis remains an internal fallback.
See `cropping-test/page-splitting/experiments/FAST_ANALYSIS.md` for matched timing
comparisons, rejected low-resolution variants, quality checks and limitations.
## Interro06 blank-detection follow-up
The reviewed Interro06 suite originally passed 15/19 with fast analysis. All four
failures were unwritten halves retained for isolated ink marks. Compact solid
spots smaller than 2 mm no longer qualify as glyphs by themselves; their convex
hull must be filled, so small x-shaped glyphs remain eligible. A short almost
collinear smear within 15 mm of the top/bottom edge also needs neighbouring
writing support. The smear rule has additional area, size and density limits.
Both filters affect blank classification only; normal margin cropping is unchanged.
The four targets now pass, along with all 19 Interro06 cases, all 15 Interro07
blank cases and the writing-preservation controls. Inputs and reviewed expectations
are unchanged. The exact marks and initial measurements are saved under
`cropping-test/interro06/blank-investigation/`.
+167
View File
@@ -0,0 +1,167 @@
import tempfile
import unittest
import json
from pathlib import Path
import numpy as np
import pymupdf
from copienator.auto_page_split import analyse_ink, analyse_page, classify_format, plan_masks, run
class AutomaticSplitTests(unittest.TestCase):
def mask(self):
return np.zeros((300, 420), bool)
def plan(self, mask, kind='a3'):
return plan_masks(mask, 25.4, kind)
def test_formats_use_size_not_landscape_alone(self):
pt = 72/25.4
for w,h,expected in [(420,297,'a3'),(210,297,'a4'),(297,210,'a4'),
(297,420,'unknown'),(320,200,'unknown')]:
self.assertEqual(classify_format(w*pt,h*pt),expected)
def test_both_directions_extend_independently(self):
mask = self.mask()
mask[40:43,150:218] = True
mask[100:103,203:280] = True
plan = self.plan(mask)
self.assertEqual([x['owner'] for x in plan['crossings']],['left','right'])
self.assertEqual(plan['extensions_mm'],[10,9])
self.assertTrue(all(o['keep'] for o in plan['outputs']))
def test_disconnected_letters_form_a_line(self):
mask = self.mask()
for x in range(170,219,4):
mask[40:44,x:x+2] = True
plan = self.plan(mask)
self.assertGreater(plan['extensions_mm'][0],0)
self.assertEqual(plan['crossings'][0]['owner'],'left')
def test_detached_terminal_strokes_follow_supported_text_on_either_side(self):
mask=self.mask()
mask[40:44,160:199]=True
mask[41:45,205:219]=True
mask[100:104,221:260]=True
mask[101:105,201:215]=True
plan=self.plan(mask)
self.assertEqual([c['owner'] for c in plan['crossings']],['left','right'])
self.assertGreater(plan['extensions_mm'][0],0)
self.assertGreater(plan['extensions_mm'][1],0)
def test_fold_only_groups_are_artifacts_without_page_anchors(self):
mask=self.mask()
mask[40:43,205:215]=True
plan=self.plan(mask)
self.assertEqual(plan['crossings'][0]['owner'],'artifact')
self.assertEqual(plan['extensions_mm'],[0,0])
self.assertFalse(plan['review_reasons'])
def test_pale_ruling_does_not_attach_to_nearby_text_but_dark_bar_does(self):
mask=np.zeros((600,840),bool)
mask[100:103,418:468]=True
mask[100:120,490:580]=True
credible=np.zeros_like(mask)
credible[100:120,490:580]=True
for colour,owner in [((134,162,200),'artifact'),((80,80,80),'right')]:
with self.subTest(colour=colour):
rgb=np.full((*mask.shape,3),255,np.uint8)
rgb[mask]=(20,20,20)
rgb[100:103,418:468]=colour
plan=plan_masks(mask,200,'a3',content_ink=credible,rgb=rgb)
self.assertEqual(plan['crossings'][0]['owner'],owner)
def test_overflow_does_not_make_other_page_nonblank(self):
mask = self.mask()
mask[40:43,150:218] = True
plan = self.plan(mask)
self.assertEqual([o['state'] for o in plan['outputs']],['content','blank'])
self.assertFalse(plan['outputs'][1]['keep'])
def test_ambiguous_crossing_preserved_on_both(self):
mask = self.mask()
mask[40:43,180:250] = True
plan = self.plan(mask)
self.assertIn('ambiguous-centre-writing',plan['review_reasons'])
self.assertLess(plan['outputs'][1]['rect_px'][0],180)
self.assertGreater(plan['outputs'][0]['rect_px'][2],250)
def test_unknown_format_and_a4(self):
mask = self.mask()
self.assertTrue(self.plan(mask,kind='unknown')['outputs'][0]['keep'])
mask[40:44,50:54] = True
self.assertEqual(self.plan(mask,kind='a4')['outputs'][0]['rect_px'],[0,0,420,300])
def test_unseeded_scan_noise_does_not_veto_blank(self):
rgb = np.full((600,420,3),255,np.uint8)
rgb[200:203,200:202] = 195
plan = analyse_page(rgb,100,'a4')
self.assertEqual(plan['outputs'][0]['state'],'blank')
self.assertEqual(plan['analysis']['early_written'],[False])
def test_faint_strokes_near_reliable_ink_are_recovered(self):
rgb = np.full((600,420,3),255,np.uint8)
rgb[200:210,200:204] = 195
rgb[200:210,204:208] = 30
ink,_ = analyse_ink(rgb,100)
self.assertTrue(ink[200:210,204:208].any())
self.assertEqual(analyse_page(rgb,100,'a4')['outputs'][0]['state'],'content')
def test_spread_artifacts_cannot_veto_blank_logical_half(self):
spread=self.mask()
spread[100:105,208:213]=True
content=self.mask()
content[30:40,300:350]=True
plan=plan_masks(spread,25.4,'a3',content_ink=content)
self.assertEqual([o['state'] for o in plan['outputs']],['blank','content'])
self.assertEqual(plan['crossings'][0]['owner'],'artifact')
self.assertFalse(plan['review_reasons'])
def test_export_preserves_visible_rotation_cropbox_and_sources(self):
for rotation in (0,90,180,270):
with self.subTest(rotation=rotation), tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = root/'source.pdf'
with pymupdf.open() as doc:
# Crop offsets and all rotations; displayed geometry is A3.
w,h = (420,297) if rotation%180==0 else (297,420)
pt = 72/25.4
p = doc.new_page(width=w*pt+20,height=h*pt+30)
p.draw_rect(pymupdf.Rect(50,50,w*pt-50,h*pt-50),fill=(0,0,1))
p.draw_rect(pymupdf.Rect(100,100,150,180),fill=(1,0,0))
p.set_cropbox(pymupdf.Rect(10,15,w*pt+10,h*pt+15))
p.set_rotation(rotation)
doc.save(source)
before = source.read_bytes()
summary = run(source,root/'result',dpi=100)
self.assertEqual(summary['output_pages'],2)
self.assertEqual(source.read_bytes(),before)
with pymupdf.open(source) as original, pymupdf.open(root/'result/pdfs/source.pdf') as result:
outputs=json.loads((root/'result/report.json').read_text())['pages'][0]['outputs']
for page,output in zip(result,outputs):
self.assertEqual(page.rotation,0)
self.assertAlmostEqual(page.rect.height,original[0].rect.height,places=3)
# Ambiguous full-width rectangle is deliberately duplicated.
a = original[0].get_pixmap(matrix=pymupdf.Matrix(.3,.3))
b = page.get_pixmap(matrix=pymupdf.Matrix(.3,.3))
self.assertGreater(b.width, a.width*.8)
expected=original[0].get_pixmap(clip=pymupdf.Rect(output['crop_points']))
actual=page.get_pixmap()
aa=np.frombuffer(expected.samples,np.uint8).reshape(expected.height,expected.width,3)
bb=np.frombuffer(actual.samples,np.uint8).reshape(actual.height,actual.width,3)
h,w=min(aa.shape[0],bb.shape[0]),min(aa.shape[1],bb.shape[1])
self.assertLess(np.abs(aa[:h,:w].astype(float)-bb[:h,:w]).mean(),2)
with self.assertRaises(FileExistsError):
run(source,root/'result',dpi=100)
def test_all_blank_copy_has_report_without_invalid_empty_pdf(self):
with tempfile.TemporaryDirectory() as directory:
root=Path(directory)
with pymupdf.open() as doc:
doc.new_page(width=210*72/25.4,height=297*72/25.4)
doc.save(root/'blank.pdf')
summary=run(root/'blank.pdf',root/'result',dpi=100)
self.assertEqual(summary['blank_removed'],1)
self.assertFalse((root/'result/pdfs/blank.pdf').exists())
self.assertTrue((root/'result/report.json').exists())
+204
View File
@@ -0,0 +1,204 @@
import shutil
import tempfile
import unittest
from pathlib import Path
from unittest.mock import Mock, patch
import numpy as np
import pymupdf
from copienator import EvaluationWorkspace, ExitCode
from copienator.commands import automatic_page_splitter as automatic
from copienator.commands.page_splitter import PDFPreviewer, _selected_inputs, main
from copienator.copy_errors import copy_errors
from copienator.orientation import load_provenance
from copienator_gui.workflow import build_command, build_workflow
class AutomaticPageSplitterTests(unittest.TestCase):
def make_source(self, root, *, blank=False):
(root / "labels").write_text("Ex 1\n")
source = root / "Copie01.pdf"
with pymupdf.open() as doc:
page = doc.new_page(width=420 * 72 / 25.4, height=297 * 72 / 25.4)
if not blank:
page.insert_text((60, 100), "Answer on the left page", fontsize=16)
page.insert_text((750, 100), "Answer on the right page", fontsize=16)
page.insert_text((560, 300), "x + y = z", fontsize=16)
doc.new_page(width=420 * 72 / 25.4, height=297 * 72 / 25.4)
page = doc.new_page(width=210 * 72 / 25.4, height=297 * 72 / 25.4)
page.insert_text((60, 100), "Exercise 3", fontsize=16)
doc.save(source)
return source
def test_gui_choices_build_manual_and_automatic_commands(self):
step = next(s for s in build_workflow(False) if s.id == "page_splitter")
self.assertEqual(
[v.label for v in step.variants], ["Manuellement", "Automatiquement"]
)
manual, auto = [
build_command(Path.cwd(), step, v, {"workers": 3}, "Exam")
for v in step.variants
]
self.assertNotIn("--automatic", manual)
self.assertNotIn("--workers", manual)
self.assertIn("--automatic", auto)
self.assertEqual(auto[auto.index("--workers") + 1], "3")
def test_cli_publishes_without_tk_and_preserves_original_and_provenance(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = self.make_source(root)
before = source.read_bytes()
workspace = EvaluationWorkspace(root)
with patch("copienator.commands.page_splitter.tk.Tk") as tk:
self.assertEqual(main([str(root), "--automatic", "--workers", "1"]), 0)
tk.assert_not_called()
preserved = workspace.original_copies_dir / source.name
self.assertEqual(preserved.read_bytes(), before)
self.assertFalse(source.exists())
output = workspace.copies_dir / source.name
with pymupdf.open(output) as doc:
self.assertEqual(len(doc), 3)
origins = load_provenance(workspace, output, len(doc))
self.assertEqual([o["origin"] for o in origins], [1, 1, 3])
self.assertGreater(doc[0].rect.width, 210 * 72 / 25.4)
saved = automatic.load_decisions(workspace, preserved)
self.assertEqual(len(saved), 3)
self.assertEqual(_selected_inputs(workspace, root), [preserved])
self.assertFalse(copy_errors(workspace))
runs = list(workspace.runs_dir.glob("page-split-*"))
self.assertTrue((runs[0] / "result/pdfs" / source.name).exists())
self.assertEqual(
automatic.run(workspace, [preserved], workers=1), ExitCode.SUCCESS
)
self.assertEqual(preserved.read_bytes(), before)
def test_processing_failure_leaves_existing_files_in_place(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = self.make_source(root)
before = source.read_bytes()
workspace = EvaluationWorkspace(root)
workspace.copies_dir.mkdir()
output = workspace.copies_dir / source.name
output.write_bytes(b"previous processed copy")
with (
patch.object(
automatic,
"analyse_batch",
side_effect=RuntimeError("analysis failed"),
),
self.assertRaisesRegex(RuntimeError, "analysis failed"),
):
automatic.run(workspace, [source], workers=1)
self.assertEqual(source.read_bytes(), before)
self.assertEqual(output.read_bytes(), b"previous processed copy")
self.assertFalse(workspace.original_copies_dir.exists())
def test_parallel_run_preserves_unselected_copies_and_ignores_stale_proposals(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = self.make_source(root)
second = root / "Copie02.pdf"
shutil.copy2(source, second)
workspace = EvaluationWorkspace(root)
workspace.copies_dir.mkdir()
other = workspace.copies_dir / "Copie99.pdf"
other.write_bytes(b"unselected copy")
automatic.run(workspace, [source, second], workers=2)
self.assertEqual(other.read_bytes(), b"unselected copy")
original = workspace.original_copies_dir / source.name
self.assertIsNotNone(automatic.load_decisions(workspace, original))
output = workspace.copies_dir / source.name
output.write_bytes(b"edited after automatic splitting")
self.assertIsNone(automatic.load_decisions(workspace, original))
def test_wholly_blank_copy_is_preserved_and_flagged_without_stale_output(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = self.make_source(root, blank=True)
before = source.read_bytes()
workspace = EvaluationWorkspace(root)
workspace.copies_dir.mkdir()
(workspace.copies_dir / source.name).write_bytes(b"stale output")
automatic.run(workspace, [source], workers=1)
self.assertFalse((workspace.copies_dir / source.name).exists())
self.assertEqual(
(workspace.original_copies_dir / source.name).read_bytes(), before
)
self.assertIn(source.name, copy_errors(workspace))
run = next(workspace.runs_dir.glob("page-split-*"))
self.assertEqual(
(run / "previous_copies" / source.name).read_bytes(), b"stale output"
)
def test_manual_acceptance_preserves_saved_overlap_blanks_and_order(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = self.make_source(root)
workspace = EvaluationWorkspace(root)
automatic.run(workspace, [source], workers=1)
original = workspace.original_copies_dir / source.name
before = original.read_bytes()
output = workspace.copies_dir / source.name
expected = pymupdf.open(output)
self.addCleanup(expected.close)
preview = PDFPreviewer.__new__(PDFPreviewer)
preview.workspace = workspace
preview.inputs = [original]
preview.num = preview.global_rotation = 0
preview.history = []
preview.output_dir = None
preview.failed = False
preview.master = Mock()
preview.load_page = Mock()
self.assertTrue(preview.setup_next_file())
self.assertTrue(preview.automatic_review)
preview._initialize_current_page_settings()
for _ in range(3):
preview.confirm_and_next_page()
self.assertFalse(preview.failed)
self.assertEqual(
[s["keep"] for s in preview.page_settings], ["both", "none", "as_is"]
)
self.assertEqual(original.read_bytes(), before)
with pymupdf.open(output) as actual:
self.assertEqual(len(actual), len(expected))
for a, b in zip(actual, expected):
self.assertAlmostEqual(a.rect.width, b.rect.width, places=3)
aa, bb = a.get_pixmap(dpi=40), b.get_pixmap(dpi=40)
self.assertEqual((aa.width, aa.height), (bb.width, bb.height))
difference = np.abs(
np.frombuffer(aa.samples, np.uint8).astype(float)
- np.frombuffer(bb.samples, np.uint8)
)
self.assertLess(difference.mean(), 0.1)
def test_accepting_wholly_blank_proposal_never_creates_an_empty_pdf(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = self.make_source(root, blank=True)
workspace = EvaluationWorkspace(root)
automatic.run(workspace, [source], workers=1)
original = workspace.original_copies_dir / source.name
before = original.read_bytes()
preview = PDFPreviewer.__new__(PDFPreviewer)
preview.workspace = workspace
preview.inputs = [original]
preview.num = preview.global_rotation = 0
preview.history = []
preview.failed = False
preview.master = Mock()
preview.load_page = Mock()
preview.setup_next_file()
preview._initialize_current_page_settings()
preview.confirm_and_next_page()
self.assertFalse(preview.failed)
self.assertEqual(original.read_bytes(), before)
self.assertFalse((workspace.copies_dir / source.name).exists())
self.assertNotIn(source.name, copy_errors(workspace))
if __name__ == "__main__":
unittest.main()
+88
View File
@@ -0,0 +1,88 @@
import unittest
import cv2
import numpy as np
import pymupdf
from copienator.auto_page_split import _analyse_page_full as analyse_page
from copienator.fast_page_split import analyse_page_fast, analyse_pdf_page_fast, quick_page_content
class FastPageSplitTests(unittest.TestCase):
def test_direct_pdf_strip_respects_rotation_and_crop_offsets(self):
image = np.full((1400, 2000, 3), 255, np.uint8)
for y in range(150, 650, 70):
for x in (150, 1300):
cv2.putText(image, 'answer x y z', (x, y),
cv2.FONT_HERSHEY_SIMPLEX, 1, (20, 20, 20), 2)
cv2.putText(image, 'x + y = z', (870, 1000),
cv2.FONT_HERSHEY_SIMPLEX, 1, (20, 20, 20), 2)
for rotation in (0, 90, 180, 270):
with self.subTest(rotation=rotation), pymupdf.open() as doc:
source = np.ascontiguousarray(np.rot90(image, rotation//90))
h,w = source.shape[:2]
page = doc.new_page(width=w*72/150+20,height=h*72/150+30)
box = pymupdf.Rect(10,15,w*72/150+10,h*72/150+15)
pix = pymupdf.Pixmap(pymupdf.csRGB,w,h,source.tobytes(),False)
page.insert_image(box,pixmap=pix)
page.set_cropbox(box)
page.set_rotation(rotation)
rendered = page.get_pixmap(dpi=150,alpha=False)
full = np.frombuffer(rendered.samples,np.uint8).reshape(rendered.height,rendered.width,3)
reference = analyse_page(full,150,'a3')
fast,overview = analyse_pdf_page_fast(page,150,'a3')
self.assertIsNone(fast['analysis']['fallback'])
self.assertEqual(fast['size_px'],reference['size_px'])
self.assertLess(overview.size,full.size)
self.assertEqual([o['keep'] for o in fast['outputs']],[True,True])
for a,b in zip(fast['outputs'],reference['outputs']):
self.assertLessEqual(max(abs(x-y) for x,y in zip(a['rect_px'],b['rect_px'])),2)
def test_plain_blank_and_sparse_pencil_use_full_detector(self):
for pencil in (False, True):
with self.subTest(pencil=pencil):
image = np.full((1200, 850, 3), 255, np.uint8)
if pencil:
cv2.putText(image, 'pencil', (100, 600),
cv2.FONT_HERSHEY_SIMPLEX, 1, (195, 195, 195), 2)
mask, early = quick_page_content(image, 150)
self.assertFalse(early)
expected = analyse_page(image, 150, 'a4')
actual = analyse_page_fast(image, 150, 'a4')
self.assertEqual(actual['outputs'][0]['keep'], expected['outputs'][0]['keep'])
self.assertEqual(actual['outputs'][0]['rect_px'], [0, 0, 850, 1200])
self.assertEqual(bool(mask.any()), pencil)
def test_compact_roi_maps_bounds_back_to_full_spread(self):
image = np.full((1400, 2000, 3), 255, np.uint8)
for y in range(150, 650, 70):
for x in (150, 1300):
cv2.putText(image, 'answer x y z', (x, y),
cv2.FONT_HERSHEY_SIMPLEX, 1, (20, 20, 20), 2)
cv2.putText(image, 'x + y = z', (870, 1000),
cv2.FONT_HERSHEY_SIMPLEX, 1, (20, 20, 20), 2)
for mirrored in (False, True):
with self.subTest(mirrored=mirrored):
rgb = image[:, ::-1].copy() if mirrored else image
reference = analyse_page(rgb, 150, 'a3')
fast = analyse_page_fast(rgb, 150, 'a3')
self.assertIsNone(fast['analysis']['fallback'])
self.assertEqual(fast['size_px'], [2000, 1400])
self.assertEqual(fast['extensions_mm'], reference['extensions_mm'])
self.assertEqual([o['rect_px'] for o in fast['outputs']],
[o['rect_px'] for o in reference['outputs']])
self.assertTrue(all(o['keep'] for o in fast['outputs']))
def test_ambiguous_or_wide_writing_falls_back_without_truncation(self):
image = np.full((700, 1000, 3), 255, np.uint8)
image[200:230, 300:650] = 20
reference = analyse_page(image, 100, 'a3')
fast = analyse_page_fast(image, 100, 'a3')
self.assertEqual(fast['analysis']['fallback'], 'full-spread')
self.assertEqual(fast['outputs'], reference['outputs'])
self.assertEqual(fast['crossings'], reference['crossings'])
def test_unknown_geometry_keeps_reference_behaviour(self):
image = np.full((400, 500, 3), 255, np.uint8)
self.assertEqual(analyse_page_fast(image, 100, 'unknown'),
analyse_page(image, 100, 'unknown'))
+107 -1
View File
@@ -3,10 +3,116 @@ import unittest
import cv2
import numpy as np
from copienator.ink_detection import detect_bounds, _paper_blur
from copienator.ink_detection import detect_bounds, detect_page_content, _paper_blur, _blank_content_support
class InkDetectionTests(unittest.TestCase):
def test_lightweight_content_masks_match_full_diagnostics(self):
for image,colour in [(self.scan(),False),(self.dark_grid(),False),
(self.scan(),True),(np.full((300,220,3),255,np.uint8),False)]:
with self.subTest(colour_cleanup=colour,shape=image.shape):
full,light = {},{}
expected=detect_bounds(image,dpi=150,diagnostics=full,colour_cleanup=colour)
actual=detect_bounds(image,dpi=150,content_masks=light,colour_cleanup=colour)
self.assertEqual(actual,expected)
self.assertEqual(set(light),{'foreground','weak_candidates'})
for name in light:
np.testing.assert_array_equal(light[name],full['masks'][name])
def test_blank_detection_ignores_remote_specks(self):
image = np.full((1200,850,3),255,np.uint8)
for x,y in ((150,200),(400,600),(700,1000)):
cv2.rectangle(image,(x,y),(x+3,y+4),(30,30,30),-1)
mask,result = detect_page_content(image,dpi=150)
self.assertFalse(mask.any())
self.assertGreater(result['content_unsupported_pixels'],0)
def test_isolated_question_numbers_remain_content(self):
for text in ('1','7','16','x'):
with self.subTest(text=text):
image = np.full((1200,850,3),255,np.uint8)
cv2.putText(image,text,(300,600),cv2.FONT_HERSHEY_SIMPLEX,.7,(30,30,30),2)
self.assertTrue(detect_page_content(image,dpi=150)[0].any())
def test_faint_edge_question_number_remains_content(self):
image = np.full((1200,850,3),255,np.uint8)
cv2.putText(image,'1',(1,600),cv2.FONT_HERSHEY_SIMPLEX,.7,(190,190,190),2)
self.assertTrue(detect_page_content(image,dpi=150)[0].any())
def test_isolated_circle_diagram_remains_content(self):
image = np.full((1200,850,3),255,np.uint8)
cv2.circle(image,(400,600),20,(30,30,30),2)
self.assertTrue(detect_page_content(image,dpi=150)[0].any())
def test_solid_ink_dots_do_not_establish_content_at_multiple_resolutions(self):
for dpi in (150,200,300):
px=dpi/25.4
mask=np.zeros((round(100*px),round(70*px)),np.uint8)
cv2.ellipse(mask,(round(30*px),round(45*px)),
(round(.6*px),round(.8*px)),0,0,360,1,-1)
self.assertFalse(_blank_content_support(mask>0,dpi)[0].any())
def test_compact_dot_is_kept_with_neighbouring_question_number(self):
mask=np.zeros((400,300),np.uint8)
cv2.putText(mask,'7',(100,200),cv2.FONT_HERSHEY_SIMPLEX,.7,1,2)
mask[194:200,117:122]=1
retained,_=_blank_content_support(mask>0,150)
self.assertTrue(retained[194:200,117:122].all())
def test_large_isolated_numbers_and_diagram_near_bottom_remain_content(self):
for text in ('1','7','x','?','2','16'):
with self.subTest(text=text):
image=np.full((1200,850,3),255,np.uint8)
cv2.putText(image,text,(300,1150),cv2.FONT_HERSHEY_SIMPLEX,
1.5,(30,30,30),2)
self.assertTrue(detect_page_content(image,200)[0].any())
image=np.full((1200,850,3),255,np.uint8)
cv2.circle(image,(400,1140),20,(30,30,30),2)
self.assertTrue(detect_page_content(image,200)[0].any())
def test_content_analysis_recognizes_blank_grid_without_changing_crop_fallback(self):
image = np.full((1200,850,3),255,np.uint8)
for y in range(20,1200,20):
cv2.line(image,(0,y),(849,y),(65,65,65),1)
for x in range(10,850,20):
cv2.line(image,(x,0),(x,1199),(65,65,65),1)
for y in (110,370,630,890,1130):
cv2.ellipse(image,(25,y),(12,20),0,0,300,(35,35,35),2)
before = detect_bounds(image,dpi=150)
mask,result = detect_page_content(image,dpi=150)
self.assertTrue(result['content_ruling_confirmed'])
self.assertFalse(mask.any())
self.assertEqual(detect_bounds(image,dpi=150),before)
def test_content_analysis_preserves_pencil_only_writing(self):
image = np.full((1200,850,3),255,np.uint8)
cv2.putText(image,'pencil',(100,600),cv2.FONT_HERSHEY_SIMPLEX,
1,(195,195,195),2)
mask,result = detect_page_content(image,dpi=150)
self.assertTrue(mask[560:610,90:250].any())
self.assertEqual(result['content_state'],'content')
def test_faint_scanner_border_does_not_count_as_pencil(self):
image = np.full((1200,850,3),255,np.uint8)
cv2.rectangle(image,(1,1),(848,300),(195,195,195),1)
mask,_ = detect_page_content(image,dpi=150)
self.assertFalse(mask.any())
def test_faint_unruled_diagram_is_retained(self):
image = np.full((1200,850,3),255,np.uint8)
cv2.rectangle(image,(100,100),(750,1100),(195,195,195),2)
mask,_ = detect_page_content(image,dpi=150)
self.assertTrue(mask.any())
def test_content_analysis_preserves_faint_grid_note_and_fraction_bar(self):
image = self.dark_grid()
cv2.putText(image,'pencil',(100,1100),cv2.FONT_HERSHEY_SIMPLEX,
.7,(190,190,190),2)
cv2.line(image,(150,1020),(700,1020),(0,0,0),3)
mask,_ = detect_page_content(image,dpi=150)
self.assertTrue(mask[1070:1110,90:250].any())
self.assertTrue(mask[1018:1023,150:700].any())
def test_fast_background_blur_matches_opencv_pixel_for_pixel(self):
rng = np.random.default_rng(42)
for shape in ((97,131), (241,319)):