sebastien/Copies
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Margin detection update

Browse Source
This commit is contained in:
sebastien committed 2026-10-30 12:34:37 +01:00
1 parent 2c9dc65640
commit b295f871a7
6 files changed
+292 -68
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

copienator/commands/cutleft.py
+5 -1
View File
@@ -29,6 +29,7 @@ from copienator.filesystem import staged_files
from copienator.copy_errors import clear_copy_error, mark_copy_error, marked_copy_paths
from copienator.configuration import SHOW_PERSONAL_STEPS
from copienator.left_margin import analyse_margins
from copienator.name_grids import name_bottom_height_fraction
from copienator.annotation_layout import fingerprint
DELIMITER_WIDTH = 5
@@ -133,7 +134,10 @@ def process_single_pdf(
return None
reference_width = normal_page_width(pdf_path, images)
if automatic_margin and max_per_file != 1 and margin_analysis is None:
margin_analysis = analyse_margins(images, RENDER_DPI, reference_width)
name_bottom = name_bottom_height_fraction(
EvaluationWorkspace(pdf_path.parent.parent), pdf_path.stem)
margin_analysis = analyse_margins(
images, RENDER_DPI, reference_width, name_bottom_height_fraction=name_bottom)
full_width_pages = []
page_crops = []
for page_index, image in enumerate(images):
copienator/left_margin.py
+68 -43
View File
@@ -10,7 +10,8 @@ from copienator.ink_detection import detect_bounds
ANALYSIS_DPI = 100
WIDTH_TOLERANCE = .08
# Copie01's name/class block reaches about 27% of the full page height.
# Start below it with clearance, including on later pages with repeated names.
# Fallback when name localization is unavailable. A localized first-page
# name/class bottom replaces this cutoff on all pages, including repeated names.
BODY_TOP = .30
BODY_BOTTOM = .82
LABEL_BOTTOM = .92
@@ -20,7 +21,10 @@ LABEL_MARGIN_GAP_MM = 10
DISCARD_LEFT_FRACTION = .10
STANDARD_MARGIN_MM = 40
MARGIN_CLEARANCE_MM = 20
MIN_MARGIN_MM = MARGIN_CLEARANCE_MM
# Minimum usable margin is independent of the strip clearance.
MIN_MARGIN_MM = 33
STANDARD_LINE_HEIGHT_MM = 8
MAX_EXCEPTION_LINES = 4
def trimmed_left_edge(starts: list[float]) -> float | None:
@@ -51,38 +55,60 @@ def handwriting_mask(rgb: np.ndarray) -> np.ndarray:
return keep[labels] & diagnostics['masks']['foreground']
def occupied_band_starts(mask: np.ndarray, bottom_fraction: float) -> list[float]:
def occupied_bands(mask: np.ndarray, bottom_fraction: float, top_fraction: float = BODY_TOP) -> list[dict]:
"""Ignore empty bands and writing starting beyond the page midpoint."""
height, width = mask.shape
band_height = max(1, round(4 * ANALYSIS_DPI / 25.4))
bottom = round(bottom_fraction * height)
starts = []
for y in range(round(BODY_TOP * height), bottom, band_height):
for y in range(round(top_fraction * height), bottom, band_height):
band = mask[y:min(y + band_height, bottom)]
xs = np.flatnonzero(band.sum(axis=0) >= 2)
if band.sum() >= 40 and len(xs) >= 12 and xs[0] <= width / 2:
starts.append(float(xs[0]) * 25.4 / ANALYSIS_DPI)
starts.append({'start_mm': float(xs[0]) * 25.4 / ANALYSIS_DPI,
'top_mm': y * 25.4 / ANALYSIS_DPI,
'bottom_mm': min(y + band_height, bottom) * 25.4 / ANALYSIS_DPI})
return starts
def body_edge(mask: np.ndarray) -> dict:
"""Keep the body decile; collect a wider 30–92% label-search window."""
starts = occupied_band_starts(mask, BODY_BOTTOM)
def occupied_band_starts(mask: np.ndarray, bottom_fraction: float, top_fraction: float = BODY_TOP) -> list[float]:
return [band['start_mm'] for band in occupied_bands(mask, bottom_fraction, top_fraction)]
def short_margin_exception(estimate: dict, margin: float) -> bool:
"""One block of at most four 8 mm lines may depart from an established margin."""
bands = estimate.get('body_bands', [])
low = [band for band in bands if band['start_mm'] < MIN_MARGIN_MM]
if not low or low[-1]['bottom_mm'] - low[0]['top_mm'] > MAX_EXCEPTION_LINES * STANDARD_LINE_HEIGHT_MM:
return False
# Require a majority of occupied bands outside that entire block, and a
# consistent decile there. Coordinates retain blank gaps between bands.
rest = [band['start_mm'] for band in bands
if band['bottom_mm'] <= low[0]['top_mm'] or band['top_mm'] >= low[-1]['bottom_mm']]
return (len(rest) >= 6 and len(rest) > len(bands) / 2
and abs(trimmed_left_edge(rest) - margin) <= COHERENT_MARGIN_TOLERANCE_MM)
def body_edge(mask: np.ndarray, top_fraction: float = BODY_TOP) -> dict:
"""Collect body and label bands below the name or the fallback header cutoff."""
bands = occupied_bands(mask, BODY_BOTTOM, top_fraction)
starts = [band['start_mm'] for band in bands]
return {
'edge_mm': trimmed_left_edge(starts) if len(starts) >= 6 else None,
'bands': len(starts),
'body_bands': bands,
'band_starts_mm': starts,
'label_band_starts_mm': occupied_band_starts(mask, LABEL_BOTTOM),
'label_band_starts_mm': occupied_band_starts(mask, LABEL_BOTTOM, top_fraction),
}
def estimate_page(image: Image.Image, render_dpi: int) -> dict:
def estimate_page(image: Image.Image, render_dpi: int, *, top_fraction: float = BODY_TOP) -> dict:
scale = ANALYSIS_DPI / render_dpi
small = image.convert('RGB').resize(
(max(1, round(image.width * scale)), max(1, round(image.height * scale))),
Image.Resampling.LANCZOS,
)
return body_edge(handwriting_mask(np.asarray(small)))
return body_edge(handwriting_mask(np.asarray(small)), top_fraction)
def format_groups(images: list[Image.Image], reference_width: float) -> list[list[int]]:
@@ -100,36 +126,34 @@ def format_groups(images: list[Image.Image], reference_width: float) -> list[lis
return groups
def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width: float) -> dict:
def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width: float,
*, name_bottom_height_fraction: float | None = None) -> dict:
"""Estimate body margins, then retain left labels supported across coherent pages."""
name_localized = (type(name_bottom_height_fraction) in (int, float)
and 0 < name_bottom_height_fraction <= 1)
analysis_top = float(name_bottom_height_fraction) if name_localized else BODY_TOP
starts = [0] * len(images)
groups = []
page_estimates = {}
def page_estimate(index):
if index not in page_estimates:
page_estimates[index] = estimate_page(images[index], render_dpi)
page_estimates[index] = (estimate_page(images[index], render_dpi, top_fraction=analysis_top)
if name_localized else estimate_page(images[index], render_dpi))
return page_estimates[index]
for indices in format_groups(images, reference_width):
# Spread samples through the copy rather than relying on the header page.
sample = sorted({indices[0], indices[min(1, len(indices) - 1)],
indices[len(indices) // 2], indices[-1]})
estimates = {index: page_estimate(index) for index in sample}
estimates = {index: page_estimate(index) for index in indices}
values = [item['edge_mm'] for item in estimates.values() if item['edge_mm'] is not None]
# Trim across the copy's format group, not separately on each page:
# one marginal comment must not veto the other pages' shared margin.
band_starts = [start for item in estimates.values() for start in item['band_starts_mm']]
status = 'insufficient-ink'
edge = trimmed_left_edge(band_starts)
edge = round(float(np.median(values)), 1) if values else None
left_mm = 0.0
if len(values) >= min(2, len(indices)):
status = 'near-edge' if edge < MIN_MARGIN_MM else 'margin'
left_mm = (0.0 if edge < MIN_MARGIN_MM else
max(edge / 2, edge - MARGIN_CLEARANCE_MM))
# Use ALL coherent pages, not just the four body-margin samples.
# A sparse/low/outlying page cannot supply evidence for this format's
# established body margin. Its normal fallback behaviour remains below.
# Only independently coherent pages supply evidence for labels in the margin.
coherent = [index for index in indices
if edge is not None and page_estimate(index)['edge_mm'] is not None
and page_estimate(index)['edge_mm'] >= MIN_MARGIN_MM
@@ -151,8 +175,18 @@ def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width:
status = 'narrow-page'
left_mm = 0.0
start = round(left_mm * render_dpi / 25.4)
previous_coherent = 0
for index in indices:
starts[index] = start
estimate = page_estimate(index)
low = estimate['edge_mm'] is not None and estimate['edge_mm'] < MIN_MARGIN_MM
tolerated = (status == 'margin' and low and previous_coherent >= 2
and short_margin_exception(estimate, edge))
starts[index] = 0 if low and not tolerated else start
estimate['short_margin_exception'] = tolerated
estimate['decision'] = ('short-exception' if tolerated else
'page-near-edge' if low else status)
if index in coherent:
previous_coherent += 1
groups.append({
'pages': [index + 1 for index in indices],
'status': status,
@@ -164,22 +198,9 @@ def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width:
'label_edge_mm': label_edge,
'label_bands': len(label_starts),
'bands': len(band_starts),
'discarded_bands': int(len(band_starts) * DISCARD_LEFT_FRACTION),
'aggregation': 'median-of-page-deciles',
'samples': [{'page': index + 1, **item} for index, item in estimates.items()],
})
# Inspect actual neighbours, including pages outside the format samples.
# Blank/uncertain pages do not establish or extend a no-margin run.
no_margin_run = 0
no_margin_from_page = None
for index in range(len(images)):
edge = page_estimate(index)['edge_mm']
no_margin_run = no_margin_run + 1 if edge is not None and edge < MIN_MARGIN_MM else 0
if no_margin_run >= 2:
run_start = index - 1
no_margin_from_page = run_start + 1
starts[run_start:] = [0] * (len(images) - run_start)
break
pages = []
for index in range(len(images)):
estimate = page_estimate(index)
@@ -188,14 +209,18 @@ def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width:
'status': 'unknown' if edge is None else
('low' if edge < MIN_MARGIN_MM else 'normal')})
return {'version': 9, 'pages': pages, 'starts_px': starts, 'groups': groups,
'no_margin_from_page': no_margin_from_page,
return {'version': 13, 'pages': pages, 'starts_px': starts, 'groups': groups,
'no_margin_from_page': None,
'discard_left_fraction': DISCARD_LEFT_FRACTION, 'margin_clearance_mm': MARGIN_CLEARANCE_MM,
'cut_fraction_of_margin': .5, 'cut_rule': 'maximum',
'standard_margin_mm': STANDARD_MARGIN_MM, 'minimum_margin_mm': MIN_MARGIN_MM,
'body_height_fraction': [BODY_TOP, BODY_BOTTOM],
'label_height_fraction': [BODY_TOP, LABEL_BOTTOM],
'name_bottom_height_fraction': name_bottom_height_fraction if name_localized else None,
'top_cutoff_source': 'localized-name' if name_localized else 'fallback',
'body_height_fraction': [analysis_top, BODY_BOTTOM],
'label_height_fraction': [analysis_top, LABEL_BOTTOM],
'label_left_fraction': LABEL_LEFT_FRACTION,
'coherent_margin_tolerance_mm': COHERENT_MARGIN_TOLERANCE_MM,
'label_margin_gap_mm': LABEL_MARGIN_GAP_MM,
'maximum_band_start_width_fraction': .5}
'maximum_band_start_width_fraction': .5,
'standard_line_height_mm': STANDARD_LINE_HEIGHT_MM,
'max_exception_lines': MAX_EXCEPTION_LINES}
copienator/name_grids.py
+56 -7
View File
@@ -2,10 +2,11 @@
from __future__ import annotations
from pathlib import Path
import math
from google.genai import types
from PIL import Image, ImageDraw
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, field_validator
from copienator import atomic_write_json, read_json
from copienator.annotation_layout import fingerprint
@@ -26,7 +27,7 @@ def save_crop(page: Image.Image, pdf: Path, directory: Path, dpi: int) -> None:
crop.save(staging / f"{pdf.stem}.jpg", "JPEG", quality=95)
atomic_write_json(staging / f"{pdf.stem}.json", {
"source_sha256": fingerprint(pdf), "width_mm": MARGIN_MM,
"height_fraction": 0.25,
"height_fraction": crop.height / page.height,
})
@@ -77,12 +78,44 @@ def build_grids(directory: Path, copies_dir: Path) -> list[dict]:
class NameCell(BaseModel):
cell: int = Field(strict=True, ge=1, le=24)
name: str
box_2d: list[float] | None = Field(
default=None, description="[ymin, xmin, ymax, xmax], 0–1000 relative to the individual crop, excluding its header")
@field_validator('box_2d', mode='before')
@classmethod
def valid_box(cls, value):
# Bad localization must not discard an otherwise readable name.
if (not isinstance(value, list) or len(value) != 4
or any(type(v) not in (int, float) or not math.isfinite(v)
or not 0 <= v <= 1000 for v in value)):
return None
return value if value[0] < value[2] and value[1] < value[3] else None
class NameResponse(BaseModel):
cells: list[NameCell]
def name_bottom_height_fraction(workspace, stem: str) -> float | None:
"""Read the name/class bottom on page 1, in cutleft's 0–1 height units.
Returns None when localization is unavailable or the source/crop changed.
This is intended as a future lower boundary for margin analysis.
"""
directory = workspace.cutleft_dir / 'Names'
try:
cached = read_json(directory / f'{stem}_result.json')
value = cached['name_bottom_height_fraction']
if (cached['source_sha256'] == fingerprint(workspace.copies_dir / f'{stem}.pdf')
and cached['crop_sha256'] == fingerprint(directory / f'{stem}.jpg')
and cached['name'] != MISSING
and type(value) in (int, float) and 0 < value <= 1):
return float(value)
except (OSError, ValueError, KeyError, TypeError):
pass
return None
def identify_names(workspace, groups, *, client, model: str, names_text: str, overwrite: bool,
thinking_level: str | None = "low") -> dict[str, str]:
"""Identify pending copies before label requests; safely fall back on any failure."""
@@ -101,6 +134,7 @@ def identify_names(workspace, groups, *, client, model: str, names_text: str, ov
try:
cached = read_json(cache_path)
if (not overwrite and cached['crop_sha256'] == crop_hash
and 'name_bottom_height_fraction' in cached
and cached['names_text'] == names_text
and cached['name'] in roster | {MISSING}):
result[stem] = cached['name']
@@ -115,11 +149,17 @@ def identify_names(workspace, groups, *, client, model: str, names_text: str, ov
The image is a grid with exactly 6 columns and 4 rows. Read rows left to right,
then top to bottom. Thick black borders separate cells; headers show row, column,
and cell number. Each cell belongs to a DIFFERENT student. Never carry a name
from one cell into another. Ignore class names such as MPSI 3.
from one cell into another. Do not include class names such as MPSI 3 in the name.
Occupied cells are exactly:
{positions}
All other cells are EMPTY padding: do not return them.
Return JSON {{"cells": [{{"cell": 1, "name": "exact authorized name"}}]}}.
Return JSON {{"cells": [{{"cell": 1, "name": "exact authorized name", "box_2d": [100, 50, 400, 900]}}]}}.
For each identified student, box_2d must tightly enclose the handwritten name AND
any adjacent class designation, including a class written just below the name.
Use [ymin, xmin, ymax, xmax] coordinates normalized to 0–1000 relative to that
individual page crop, NOT the whole grid. Exclude the numbered header, black
borders and any padding outside the crop. Return box_2d=null for Missing names
or if the position cannot be determined reliably.
Return one entry per occupied cell, using its cell number. Match only confidently
readable handwriting to an exact authorized name below. Do not guess or assign
names by elimination. Return "Missing" for an unreadable, absent, or ambiguous name.
@@ -143,12 +183,21 @@ Authorized names:
entries = NameResponse.model_validate_json(response.text).cells
by_cell = {}
for entry in entries:
by_cell.setdefault(entry.cell, []).append(entry.name)
by_cell.setdefault(entry.cell, []).append(entry)
for stem, (cell, crop_hash, cache_path) in pending.items():
candidates = by_cell.get(cell['cell'], [])
name = candidates[0] if len(candidates) == 1 and candidates[0] in roster else MISSING
entry = candidates[0] if len(candidates) == 1 else None
name = entry.name if entry is not None and entry.name in roster else MISSING
box = entry.box_2d if name != MISSING else None
metadata = read_json(directory / f'{stem}.json')
bottom = box[2] / 1000 * metadata['height_fraction'] if box else None
result[stem] = name
atomic_write_json(cache_path, {'name': name, 'crop_sha256': crop_hash, 'names_text': names_text})
atomic_write_json(cache_path, {
'name': name, 'crop_sha256': crop_hash, 'names_text': names_text,
'source_sha256': metadata['source_sha256'],
'box_2d': box, # 0–1000 coordinates relative to the name crop.
'name_bottom_height_fraction': bottom, # Fraction of full page 1.
})
except Exception as exc: # Remote failures must preserve the existing fallback.
print(f"Name recognition failed for {grid['image']}: {type(exc).__name__}: {exc}; using label/name fallback.")
# Do not cache transport/schema failures: a resumed run should retry.
tests/test_left_margin.py
+113 -17
View File
@@ -53,23 +53,53 @@ class LeftMarginTests(unittest.TestCase):
group = result['groups'][0]
self.assertEqual(group['edge_mm'], 42)
self.assertEqual(group['left_mm'], 22)
self.assertEqual(group['discarded_bands'], 14)
self.assertEqual(result['starts_px'], [round(22 * 200 / 25.4)] * 4)
self.assertEqual(group['aggregation'], 'median-of-page-deciles')
self.assertEqual(result['starts_px'], [round(22 * 200 / 25.4)] * 3 + [0])
def test_short_exception_requires_prior_margin_and_one_small_block(self):
# At 4 mm per band, eight bands cover four standard 8 mm lines.
for prior, low_positions, expected in (
(2, list(range(8)), True),
(2, list(range(9)), False),
(1, list(range(8)), False),
(0, list(range(8)), False),
(2, [0, 2, 4, 6, 8], False)):
with self.subTest(prior=prior, low_positions=low_positions):
images = [Image.new('RGB', (1654, 200)) for _ in range(5)]
def estimate(image, _dpi):
index = next(i for i, candidate in enumerate(images) if candidate is image)
bands = [{'start_mm': 5 if index == prior and i in low_positions else 42,
'top_mm': i * 4, 'bottom_mm': (i + 1) * 4} for i in range(30)]
starts = [b['start_mm'] for b in bands]
return {'edge_mm': left_margin.trimmed_left_edge(starts),
'bands': len(bands), 'body_bands': bands, 'band_starts_mm': starts}
with patch.object(left_margin, 'estimate_page', side_effect=estimate):
result = left_margin.analyse_margins(images, 200, 1654)
self.assertEqual(result['pages'][prior]['short_margin_exception'], expected)
self.assertEqual(result['starts_px'][prior], round(22 * 200 / 25.4) if expected else 0)
self.assertGreater(result['starts_px'][-1], 0)
def test_short_exception_rejects_inconsistent_or_insufficient_remainder(self):
for remainder in ([65] * 22, [42] * 5):
starts = [5] * 8 + remainder
estimate = {'body_bands': [{'start_mm': x, 'top_mm': i * 4, 'bottom_mm': (i + 1) * 4}
for i, x in enumerate(starts)]}
self.assertFalse(left_margin.short_margin_exception(estimate, 42))
def test_repeated_left_writing_beyond_ten_percent_is_retained(self):
self.assertEqual(left_margin.trimmed_left_edge([10] * 20 + [42] * 80), 10)
def test_disagreeing_pages_use_half_margin_when_it_removes_more(self):
result = self.analyse([25, 31, 68, 49])
self.assertEqual(result['groups'][0]['left_mm'], 12.5)
self.assertEqual(result['starts_px'][1:], [round(12.5 * 200 / 25.4)] * 3)
result = self.analyse([33, 35, 41, 39])
self.assertEqual(result['groups'][0]['left_mm'], 18.5)
self.assertEqual(result['starts_px'][1:], [round(18.5 * 200 / 25.4)] * 3)
def test_small_margin_starts_at_left(self):
result = self.analyse([10, 15, 8, 13])
self.assertEqual(result['groups'][0]['left_mm'], 0)
self.assertEqual(result['starts_px'], [0] * 4)
def test_two_consecutive_no_margin_pages_enforce_left_edge_afterwards(self):
def test_low_pages_do_not_cancel_later_pages_margin(self):
images = [Image.new('RGB', (1654, 200)) for _ in range(8)]
# Low pages have fewer occupied bands, so pooling still proposes a margin.
edges = [42, 42, 10, 12, 42, 42, 42, 42]
@@ -81,8 +111,8 @@ class LeftMarginTests(unittest.TestCase):
with patch.object(left_margin, 'estimate_page', side_effect=estimate):
result = left_margin.analyse_margins(images, 200, 1654)
self.assertEqual(result['groups'][0]['left_mm'], 22)
self.assertEqual(result['no_margin_from_page'], 3)
self.assertEqual(result['starts_px'], [round(22 * 200 / 25.4)] * 2 + [0] * 6)
self.assertIsNone(result['no_margin_from_page'])
self.assertEqual(result['starts_px'], [round(22 * 200 / 25.4)] * 2 + [0] * 2 + [round(22 * 200 / 25.4)] * 4)
def test_isolated_low_or_blank_pages_do_not_trigger_no_margin_rule(self):
for edges in ([42, 10, 42, 12], [42, 10, None, 12]):
@@ -90,21 +120,24 @@ class LeftMarginTests(unittest.TestCase):
result = self.analyse(edges)
self.assertIsNone(result['no_margin_from_page'])
def test_two_no_margin_pages_enforce_rule_across_later_width_changes(self):
def test_low_pages_do_not_cancel_later_width_groups(self):
result = self.analyse([42, 42, 10, 12, 42, 42],
widths=[1654, 1654, 1654, 1654, 2000, 2000],
band_counts=[100, 100, 6, 6, 100, 100])
self.assertEqual(result['no_margin_from_page'], 3)
self.assertGreater(result['starts_px'][0], 0)
self.assertEqual(result['starts_px'][2:], [0] * 4)
self.assertIsNone(result['no_margin_from_page'])
# First group's median is 26 mm; the later width group still has 42 mm.
self.assertEqual(result['starts_px'][0], 0)
self.assertEqual(result['starts_px'][2:4], [0, 0])
self.assertTrue(all(result['starts_px'][4:]))
def test_larger_cut_distance_and_no_margin_threshold(self):
for edge, expected in ((0, 0), (19.9, 0), (20, 10), (20.1, 10.05), (28.4, 14.2), (40, 20), (60, 40)):
for edge, expected in ((0, 0), (20, 0), (25.4, 0), (26.4, 0), (27.9, 0), (28, 0), (31, 0), (32.5, 0), (32.9, 0), (33, 16.5), (33.1, 16.55), (40, 20), (60, 40)):
with self.subTest(edge=edge):
result = self.analyse([edge] * 4)
self.assertEqual(result['starts_px'][1:], [round(expected * 200 / 25.4)] * 3)
self.assertEqual(result['groups'][0]['status'], 'near-edge' if edge < 20 else 'margin')
self.assertEqual(result['groups'][0]['status'], 'near-edge' if edge < 33 else 'margin')
self.assertEqual(result['margin_clearance_mm'], 20)
self.assertEqual(result['minimum_margin_mm'], 33)
def test_body_bands_ignore_header_footer_and_abstain_on_sparse_ink(self):
mask = np.zeros((1169, 827), bool)
@@ -155,7 +188,7 @@ class LeftMarginTests(unittest.TestCase):
self.assertTrue(group['labels_in_margin'])
self.assertEqual(result['starts_px'], [0] * 6)
def test_unsampled_coherent_page_can_move_the_whole_group_left(self):
def test_any_coherent_page_can_move_the_whole_group_left(self):
images = [Image.new('RGB', (1654, 200)) for _ in range(6)]
def estimate(image, _dpi):
labels = [5.] * 20 if image is images[2] else []
@@ -164,7 +197,7 @@ class LeftMarginTests(unittest.TestCase):
with patch.object(left_margin, 'estimate_page', side_effect=estimate):
result = left_margin.analyse_margins(images, 200, 1654)
group = result['groups'][0]
self.assertNotIn(3, [p['page'] for p in group['samples']])
self.assertIn(3, [p['page'] for p in group['samples']])
self.assertIn(3, group['coherent_pages'])
self.assertEqual(result['starts_px'], [0] * 6)
@@ -193,6 +226,69 @@ class LeftMarginTests(unittest.TestCase):
self.assertFalse(result['groups'][0]['labels_in_margin'])
self.assertGreater(result['starts_px'][0], 0)
def test_localized_name_bottom_reveals_labels_above_old_cutoff(self):
mask = np.zeros((1169, 827), bool)
mask[50:150, 4:100] = True # Name above localized bottom.
mask[250:265, 30:110] = True # Heading between 20% and 30%.
for y in range(400, 920, 35):
mask[y:y + 5, 170:270] = True
old = left_margin.body_edge(mask)
localized = left_margin.body_edge(mask, top_fraction=.20)
self.assertGreater(min(old['label_band_starts_mm']), 40)
self.assertAlmostEqual(min(localized['label_band_starts_mm']), 7.62)
self.assertTrue(all(start > 2 for start in localized['label_band_starts_mm']))
def test_first_page_name_fraction_is_reused_on_every_page(self):
images = [Image.new('RGB', (1654, 200)) for _ in range(6)]
estimate = {'edge_mm': 42, 'bands': 12, 'band_starts_mm': [42.] * 12,
'label_band_starts_mm': [42.] * 12}
with patch.object(left_margin, 'estimate_page', return_value=estimate) as estimator:
result = left_margin.analyse_margins(images, 200, 1654,
name_bottom_height_fraction=.14)
self.assertEqual(estimator.call_count, 6)
self.assertTrue(all(call.kwargs == {'top_fraction': .14}
for call in estimator.call_args_list))
self.assertEqual(result['body_height_fraction'], [.14, .82])
self.assertEqual(result['label_height_fraction'], [.14, .92])
self.assertEqual(result['top_cutoff_source'], 'localized-name')
def test_unavailable_or_invalid_name_fraction_keeps_fallback(self):
images = [Image.new('RGB', (1654, 200))]
estimate = {'edge_mm': None, 'bands': 0, 'band_starts_mm': [],
'label_band_starts_mm': []}
for value in (None, 0, -1, 1.1, True, '0.1', float('nan')):
with self.subTest(value=value), patch.object(left_margin, 'estimate_page', return_value=estimate):
result = left_margin.analyse_margins(images, 200, 1654,
name_bottom_height_fraction=value)
self.assertEqual(result['body_height_fraction'][0], .30)
self.assertEqual(result['top_cutoff_source'], 'fallback')
self.assertIsNone(result['name_bottom_height_fraction'])
self.assertIsNone(left_margin.body_edge(np.zeros((100, 100), bool), .95)['edge_mm'])
def test_cutleft_reads_validated_name_cache_and_rejects_stale_source(self):
import json
import tempfile
from copienator.annotation_layout import fingerprint
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'Copies').mkdir()
names = root / 'Cutleft' / 'Names'
names.mkdir(parents=True)
pdf = root / 'Copies' / 'Copie01.pdf'
pdf.write_bytes(b'original-pdf')
crop = names / 'Copie01.jpg'
crop.write_bytes(b'name-crop')
(names / 'Copie01_result.json').write_text(json.dumps({
'name': 'Student', 'source_sha256': fingerprint(pdf),
'crop_sha256': fingerprint(crop), 'name_bottom_height_fraction': .14}))
images = [Image.new('RGB', (900, 600), 'white')]
for expected in (.14, None):
with patch.object(cutleft, 'get_pdf_pages', return_value=images), \
patch.object(cutleft, 'analyse_margins', return_value={'starts_px': [0], 'groups': []}) as analyse:
self.assertIsNotNone(cutleft.process_single_pdf(pdf))
self.assertEqual(analyse.call_args.kwargs['name_bottom_height_fraction'], expected)
pdf.write_bytes(b'changed-pdf')
def test_real_mask_rejects_ruling_and_preserves_handwriting(self):
image = Image.new('RGB', (827, 1169), 'white')
draw = ImageDraw.Draw(image)
@@ -224,7 +320,7 @@ class LeftMarginTests(unittest.TestCase):
def test_clear_margin_uses_same_conservative_edge_on_every_page(self):
result = self.analyse([43, 41, 46, 44])
self.assertEqual(result['groups'][0]['status'], 'margin')
self.assertEqual(result['starts_px'], [round(21 * 200 / 25.4)] * 4)
self.assertEqual(result['starts_px'], [round(23.5 * 200 / 25.4)] * 4)
self.assertNotIn('first_page_body_start_px', result)
def test_sparse_groups_stay_at_left(self):
tests/test_margin_benchmark.py
+5
View File
@@ -39,6 +39,11 @@ class MarginBenchmarkTests(unittest.TestCase):
case['sha256'])
self.assertEqual([p['page'] for p in case['pages']], list(range(1, case['page_count'] + 1)))
self.assertTrue(case['explanation'])
name_box = case['name_localization']['box_px']
self.assertEqual(case['name_localization']['page_height_px'],
case['recorded_schema']['page_crops'][0]['height_px'])
self.assertAlmostEqual(case['name_bottom_height_fraction'],
name_box[3] / case['name_localization']['page_height_px'])
self.assertTrue(any(p['labels'] for p in case['pages']))
for expected, recorded in zip(case['pages'], case['recorded_schema']['page_crops'], strict=True):
start = round(expected['suggested_start_mm'] * 200 / 25.4)
tests/test_name_grids.py
+45
View File
@@ -90,6 +90,51 @@ class NameGridTests(unittest.TestCase):
self.identify({'Copie01': []})
self.assertEqual(self.client.models.generate_content.call_count, 2)
def test_name_box_is_saved_in_full_page_height_units(self):
pdf = self.crop(1)
self.client.models.generate_content.return_value = Mock(text=json.dumps({'cells': [
{'cell': 1, 'name': 'Alice', 'box_2d': [100, 20, 800, 950]},
]}))
self.assertEqual(self.identify({'Copie01': []}), {'Copie01': 'Alice'})
cached = read_json(self.directory / 'Copie01_result.json')
self.assertEqual(cached['box_2d'], [100, 20, 800, 950])
self.assertAlmostEqual(cached['name_bottom_height_fraction'], .8 * 584 / 2338)
self.assertAlmostEqual(names.name_bottom_height_fraction(self.workspace, 'Copie01'),
.8 * 584 / 2338)
self.identify({'Copie01': []})
self.assertEqual(self.client.models.generate_content.call_count, 1)
pdf.write_bytes(b'changed')
self.assertIsNone(names.name_bottom_height_fraction(self.workspace, 'Copie01'))
def test_invalid_or_unmatched_boxes_do_not_produce_a_height(self):
self.crop(1)
for name, box in [('Alice', [800, 20, 100, 950]),
('Alice', [0, 0, 1001, 900]), ('Alice', [1, 2]),
('Alice', ['bad', 0, 800, 900]), ('Alice', None),
('Outsider', [100, 20, 800, 950]),
('Missing', [100, 20, 800, 950])]:
with self.subTest(name=name, box=box):
self.client.models.generate_content.return_value = Mock(text=json.dumps({'cells': [
{'cell': 1, 'name': name, 'box_2d': box},
]}))
self.assertEqual(self.identify({'Copie01': []}, overwrite=True),
{'Copie01': 'Alice' if name == 'Alice' else 'Missing'})
self.assertIsNone(names.name_bottom_height_fraction(self.workspace, 'Copie01'))
def test_legacy_name_cache_is_refreshed_to_request_position(self):
self.crop(1)
self.client.models.generate_content.return_value = Mock(text=json.dumps({'cells': [
{'cell': 1, 'name': 'Alice', 'box_2d': [100, 20, 800, 950]},
]}))
self.identify({'Copie01': []})
path = self.directory / 'Copie01_result.json'
cached = read_json(path)
del cached['name_bottom_height_fraction']
path.write_text(json.dumps(cached))
self.identify({'Copie01': []})
self.assertEqual(self.client.models.generate_content.call_count, 2)
self.assertIsNotNone(names.name_bottom_height_fraction(self.workspace, 'Copie01'))
def test_known_and_missing_names_choose_different_label_requests(self):
for number in (1, 2):
self.crop(number)