From b295f871a77926a1d0d73d0b3d46527ee986dbd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Miquel?= Date: Fri, 30 Oct 2026 12:34:37 +0100 Subject: [PATCH] Margin detection update --- copienator/commands/cutleft.py | 6 +- copienator/left_margin.py | 111 +++++++++++++++++----------- copienator/name_grids.py | 63 ++++++++++++++-- tests/test_left_margin.py | 130 ++++++++++++++++++++++++++++----- tests/test_margin_benchmark.py | 5 ++ tests/test_name_grids.py | 45 ++++++++++++ 6 files changed, 292 insertions(+), 68 deletions(-) diff --git a/copienator/commands/cutleft.py b/copienator/commands/cutleft.py index b9cc5dd..e92bb29 100644 --- a/copienator/commands/cutleft.py +++ b/copienator/commands/cutleft.py @@ -29,6 +29,7 @@ from copienator.filesystem import staged_files from copienator.copy_errors import clear_copy_error, mark_copy_error, marked_copy_paths from copienator.configuration import SHOW_PERSONAL_STEPS from copienator.left_margin import analyse_margins +from copienator.name_grids import name_bottom_height_fraction from copienator.annotation_layout import fingerprint DELIMITER_WIDTH = 5 @@ -133,7 +134,10 @@ def process_single_pdf( return None reference_width = normal_page_width(pdf_path, images) if automatic_margin and max_per_file != 1 and margin_analysis is None: - margin_analysis = analyse_margins(images, RENDER_DPI, reference_width) + name_bottom = name_bottom_height_fraction( + EvaluationWorkspace(pdf_path.parent.parent), pdf_path.stem) + margin_analysis = analyse_margins( + images, RENDER_DPI, reference_width, name_bottom_height_fraction=name_bottom) full_width_pages = [] page_crops = [] for page_index, image in enumerate(images): diff --git a/copienator/left_margin.py b/copienator/left_margin.py index 25282ea..bfbd32e 100644 --- a/copienator/left_margin.py +++ b/copienator/left_margin.py @@ -10,7 +10,8 @@ from copienator.ink_detection import detect_bounds ANALYSIS_DPI = 100 WIDTH_TOLERANCE = .08 # Copie01's name/class block reaches about 27% of the full page height. -# Start below it with clearance, including on later pages with repeated names. +# Fallback when name localization is unavailable. A localized first-page +# name/class bottom replaces this cutoff on all pages, including repeated names. BODY_TOP = .30 BODY_BOTTOM = .82 LABEL_BOTTOM = .92 @@ -20,7 +21,10 @@ LABEL_MARGIN_GAP_MM = 10 DISCARD_LEFT_FRACTION = .10 STANDARD_MARGIN_MM = 40 MARGIN_CLEARANCE_MM = 20 -MIN_MARGIN_MM = MARGIN_CLEARANCE_MM +# Minimum usable margin is independent of the strip clearance. +MIN_MARGIN_MM = 33 +STANDARD_LINE_HEIGHT_MM = 8 +MAX_EXCEPTION_LINES = 4 def trimmed_left_edge(starts: list[float]) -> float | None: @@ -51,38 +55,60 @@ def handwriting_mask(rgb: np.ndarray) -> np.ndarray: return keep[labels] & diagnostics['masks']['foreground'] -def occupied_band_starts(mask: np.ndarray, bottom_fraction: float) -> list[float]: +def occupied_bands(mask: np.ndarray, bottom_fraction: float, top_fraction: float = BODY_TOP) -> list[dict]: """Ignore empty bands and writing starting beyond the page midpoint.""" height, width = mask.shape band_height = max(1, round(4 * ANALYSIS_DPI / 25.4)) bottom = round(bottom_fraction * height) starts = [] - for y in range(round(BODY_TOP * height), bottom, band_height): + for y in range(round(top_fraction * height), bottom, band_height): band = mask[y:min(y + band_height, bottom)] xs = np.flatnonzero(band.sum(axis=0) >= 2) if band.sum() >= 40 and len(xs) >= 12 and xs[0] <= width / 2: - starts.append(float(xs[0]) * 25.4 / ANALYSIS_DPI) + starts.append({'start_mm': float(xs[0]) * 25.4 / ANALYSIS_DPI, + 'top_mm': y * 25.4 / ANALYSIS_DPI, + 'bottom_mm': min(y + band_height, bottom) * 25.4 / ANALYSIS_DPI}) return starts -def body_edge(mask: np.ndarray) -> dict: - """Keep the body decile; collect a wider 30–92% label-search window.""" - starts = occupied_band_starts(mask, BODY_BOTTOM) +def occupied_band_starts(mask: np.ndarray, bottom_fraction: float, top_fraction: float = BODY_TOP) -> list[float]: + return [band['start_mm'] for band in occupied_bands(mask, bottom_fraction, top_fraction)] + + +def short_margin_exception(estimate: dict, margin: float) -> bool: + """One block of at most four 8 mm lines may depart from an established margin.""" + bands = estimate.get('body_bands', []) + low = [band for band in bands if band['start_mm'] < MIN_MARGIN_MM] + if not low or low[-1]['bottom_mm'] - low[0]['top_mm'] > MAX_EXCEPTION_LINES * STANDARD_LINE_HEIGHT_MM: + return False + # Require a majority of occupied bands outside that entire block, and a + # consistent decile there. Coordinates retain blank gaps between bands. + rest = [band['start_mm'] for band in bands + if band['bottom_mm'] <= low[0]['top_mm'] or band['top_mm'] >= low[-1]['bottom_mm']] + return (len(rest) >= 6 and len(rest) > len(bands) / 2 + and abs(trimmed_left_edge(rest) - margin) <= COHERENT_MARGIN_TOLERANCE_MM) + + +def body_edge(mask: np.ndarray, top_fraction: float = BODY_TOP) -> dict: + """Collect body and label bands below the name or the fallback header cutoff.""" + bands = occupied_bands(mask, BODY_BOTTOM, top_fraction) + starts = [band['start_mm'] for band in bands] return { 'edge_mm': trimmed_left_edge(starts) if len(starts) >= 6 else None, 'bands': len(starts), + 'body_bands': bands, 'band_starts_mm': starts, - 'label_band_starts_mm': occupied_band_starts(mask, LABEL_BOTTOM), + 'label_band_starts_mm': occupied_band_starts(mask, LABEL_BOTTOM, top_fraction), } -def estimate_page(image: Image.Image, render_dpi: int) -> dict: +def estimate_page(image: Image.Image, render_dpi: int, *, top_fraction: float = BODY_TOP) -> dict: scale = ANALYSIS_DPI / render_dpi small = image.convert('RGB').resize( (max(1, round(image.width * scale)), max(1, round(image.height * scale))), Image.Resampling.LANCZOS, ) - return body_edge(handwriting_mask(np.asarray(small))) + return body_edge(handwriting_mask(np.asarray(small)), top_fraction) def format_groups(images: list[Image.Image], reference_width: float) -> list[list[int]]: @@ -100,36 +126,34 @@ def format_groups(images: list[Image.Image], reference_width: float) -> list[lis return groups -def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width: float) -> dict: +def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width: float, + *, name_bottom_height_fraction: float | None = None) -> dict: """Estimate body margins, then retain left labels supported across coherent pages.""" + name_localized = (type(name_bottom_height_fraction) in (int, float) + and 0 < name_bottom_height_fraction <= 1) + analysis_top = float(name_bottom_height_fraction) if name_localized else BODY_TOP starts = [0] * len(images) groups = [] page_estimates = {} def page_estimate(index): if index not in page_estimates: - page_estimates[index] = estimate_page(images[index], render_dpi) + page_estimates[index] = (estimate_page(images[index], render_dpi, top_fraction=analysis_top) + if name_localized else estimate_page(images[index], render_dpi)) return page_estimates[index] for indices in format_groups(images, reference_width): - # Spread samples through the copy rather than relying on the header page. - sample = sorted({indices[0], indices[min(1, len(indices) - 1)], - indices[len(indices) // 2], indices[-1]}) - estimates = {index: page_estimate(index) for index in sample} + estimates = {index: page_estimate(index) for index in indices} values = [item['edge_mm'] for item in estimates.values() if item['edge_mm'] is not None] - # Trim across the copy's format group, not separately on each page: - # one marginal comment must not veto the other pages' shared margin. band_starts = [start for item in estimates.values() for start in item['band_starts_mm']] status = 'insufficient-ink' - edge = trimmed_left_edge(band_starts) + edge = round(float(np.median(values)), 1) if values else None left_mm = 0.0 if len(values) >= min(2, len(indices)): status = 'near-edge' if edge < MIN_MARGIN_MM else 'margin' left_mm = (0.0 if edge < MIN_MARGIN_MM else max(edge / 2, edge - MARGIN_CLEARANCE_MM)) - # Use ALL coherent pages, not just the four body-margin samples. - # A sparse/low/outlying page cannot supply evidence for this format's - # established body margin. Its normal fallback behaviour remains below. + # Only independently coherent pages supply evidence for labels in the margin. coherent = [index for index in indices if edge is not None and page_estimate(index)['edge_mm'] is not None and page_estimate(index)['edge_mm'] >= MIN_MARGIN_MM @@ -151,8 +175,18 @@ def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width: status = 'narrow-page' left_mm = 0.0 start = round(left_mm * render_dpi / 25.4) + previous_coherent = 0 for index in indices: - starts[index] = start + estimate = page_estimate(index) + low = estimate['edge_mm'] is not None and estimate['edge_mm'] < MIN_MARGIN_MM + tolerated = (status == 'margin' and low and previous_coherent >= 2 + and short_margin_exception(estimate, edge)) + starts[index] = 0 if low and not tolerated else start + estimate['short_margin_exception'] = tolerated + estimate['decision'] = ('short-exception' if tolerated else + 'page-near-edge' if low else status) + if index in coherent: + previous_coherent += 1 groups.append({ 'pages': [index + 1 for index in indices], 'status': status, @@ -164,22 +198,9 @@ def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width: 'label_edge_mm': label_edge, 'label_bands': len(label_starts), 'bands': len(band_starts), - 'discarded_bands': int(len(band_starts) * DISCARD_LEFT_FRACTION), + 'aggregation': 'median-of-page-deciles', 'samples': [{'page': index + 1, **item} for index, item in estimates.items()], }) - # Inspect actual neighbours, including pages outside the format samples. - # Blank/uncertain pages do not establish or extend a no-margin run. - no_margin_run = 0 - no_margin_from_page = None - for index in range(len(images)): - edge = page_estimate(index)['edge_mm'] - no_margin_run = no_margin_run + 1 if edge is not None and edge < MIN_MARGIN_MM else 0 - if no_margin_run >= 2: - run_start = index - 1 - no_margin_from_page = run_start + 1 - starts[run_start:] = [0] * (len(images) - run_start) - break - pages = [] for index in range(len(images)): estimate = page_estimate(index) @@ -188,14 +209,18 @@ def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width: 'status': 'unknown' if edge is None else ('low' if edge < MIN_MARGIN_MM else 'normal')}) - return {'version': 9, 'pages': pages, 'starts_px': starts, 'groups': groups, - 'no_margin_from_page': no_margin_from_page, + return {'version': 13, 'pages': pages, 'starts_px': starts, 'groups': groups, + 'no_margin_from_page': None, 'discard_left_fraction': DISCARD_LEFT_FRACTION, 'margin_clearance_mm': MARGIN_CLEARANCE_MM, 'cut_fraction_of_margin': .5, 'cut_rule': 'maximum', 'standard_margin_mm': STANDARD_MARGIN_MM, 'minimum_margin_mm': MIN_MARGIN_MM, - 'body_height_fraction': [BODY_TOP, BODY_BOTTOM], - 'label_height_fraction': [BODY_TOP, LABEL_BOTTOM], + 'name_bottom_height_fraction': name_bottom_height_fraction if name_localized else None, + 'top_cutoff_source': 'localized-name' if name_localized else 'fallback', + 'body_height_fraction': [analysis_top, BODY_BOTTOM], + 'label_height_fraction': [analysis_top, LABEL_BOTTOM], 'label_left_fraction': LABEL_LEFT_FRACTION, 'coherent_margin_tolerance_mm': COHERENT_MARGIN_TOLERANCE_MM, 'label_margin_gap_mm': LABEL_MARGIN_GAP_MM, - 'maximum_band_start_width_fraction': .5} + 'maximum_band_start_width_fraction': .5, + 'standard_line_height_mm': STANDARD_LINE_HEIGHT_MM, + 'max_exception_lines': MAX_EXCEPTION_LINES} diff --git a/copienator/name_grids.py b/copienator/name_grids.py index 8e3e9c1..510c980 100644 --- a/copienator/name_grids.py +++ b/copienator/name_grids.py @@ -2,10 +2,11 @@ from __future__ import annotations from pathlib import Path +import math from google.genai import types from PIL import Image, ImageDraw -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from copienator import atomic_write_json, read_json from copienator.annotation_layout import fingerprint @@ -26,7 +27,7 @@ def save_crop(page: Image.Image, pdf: Path, directory: Path, dpi: int) -> None: crop.save(staging / f"{pdf.stem}.jpg", "JPEG", quality=95) atomic_write_json(staging / f"{pdf.stem}.json", { "source_sha256": fingerprint(pdf), "width_mm": MARGIN_MM, - "height_fraction": 0.25, + "height_fraction": crop.height / page.height, }) @@ -77,12 +78,44 @@ def build_grids(directory: Path, copies_dir: Path) -> list[dict]: class NameCell(BaseModel): cell: int = Field(strict=True, ge=1, le=24) name: str + box_2d: list[float] | None = Field( + default=None, description="[ymin, xmin, ymax, xmax], 0–1000 relative to the individual crop, excluding its header") + + @field_validator('box_2d', mode='before') + @classmethod + def valid_box(cls, value): + # Bad localization must not discard an otherwise readable name. + if (not isinstance(value, list) or len(value) != 4 + or any(type(v) not in (int, float) or not math.isfinite(v) + or not 0 <= v <= 1000 for v in value)): + return None + return value if value[0] < value[2] and value[1] < value[3] else None class NameResponse(BaseModel): cells: list[NameCell] +def name_bottom_height_fraction(workspace, stem: str) -> float | None: + """Read the name/class bottom on page 1, in cutleft's 0–1 height units. + + Returns None when localization is unavailable or the source/crop changed. + This is intended as a future lower boundary for margin analysis. + """ + directory = workspace.cutleft_dir / 'Names' + try: + cached = read_json(directory / f'{stem}_result.json') + value = cached['name_bottom_height_fraction'] + if (cached['source_sha256'] == fingerprint(workspace.copies_dir / f'{stem}.pdf') + and cached['crop_sha256'] == fingerprint(directory / f'{stem}.jpg') + and cached['name'] != MISSING + and type(value) in (int, float) and 0 < value <= 1): + return float(value) + except (OSError, ValueError, KeyError, TypeError): + pass + return None + + def identify_names(workspace, groups, *, client, model: str, names_text: str, overwrite: bool, thinking_level: str | None = "low") -> dict[str, str]: """Identify pending copies before label requests; safely fall back on any failure.""" @@ -101,6 +134,7 @@ def identify_names(workspace, groups, *, client, model: str, names_text: str, ov try: cached = read_json(cache_path) if (not overwrite and cached['crop_sha256'] == crop_hash + and 'name_bottom_height_fraction' in cached and cached['names_text'] == names_text and cached['name'] in roster | {MISSING}): result[stem] = cached['name'] @@ -115,11 +149,17 @@ def identify_names(workspace, groups, *, client, model: str, names_text: str, ov The image is a grid with exactly 6 columns and 4 rows. Read rows left to right, then top to bottom. Thick black borders separate cells; headers show row, column, and cell number. Each cell belongs to a DIFFERENT student. Never carry a name -from one cell into another. Ignore class names such as MPSI 3. +from one cell into another. Do not include class names such as MPSI 3 in the name. Occupied cells are exactly: {positions} All other cells are EMPTY padding: do not return them. -Return JSON {{"cells": [{{"cell": 1, "name": "exact authorized name"}}]}}. +Return JSON {{"cells": [{{"cell": 1, "name": "exact authorized name", "box_2d": [100, 50, 400, 900]}}]}}. +For each identified student, box_2d must tightly enclose the handwritten name AND +any adjacent class designation, including a class written just below the name. +Use [ymin, xmin, ymax, xmax] coordinates normalized to 0–1000 relative to that +individual page crop, NOT the whole grid. Exclude the numbered header, black +borders and any padding outside the crop. Return box_2d=null for Missing names +or if the position cannot be determined reliably. Return one entry per occupied cell, using its cell number. Match only confidently readable handwriting to an exact authorized name below. Do not guess or assign names by elimination. Return "Missing" for an unreadable, absent, or ambiguous name. @@ -143,12 +183,21 @@ Authorized names: entries = NameResponse.model_validate_json(response.text).cells by_cell = {} for entry in entries: - by_cell.setdefault(entry.cell, []).append(entry.name) + by_cell.setdefault(entry.cell, []).append(entry) for stem, (cell, crop_hash, cache_path) in pending.items(): candidates = by_cell.get(cell['cell'], []) - name = candidates[0] if len(candidates) == 1 and candidates[0] in roster else MISSING + entry = candidates[0] if len(candidates) == 1 else None + name = entry.name if entry is not None and entry.name in roster else MISSING + box = entry.box_2d if name != MISSING else None + metadata = read_json(directory / f'{stem}.json') + bottom = box[2] / 1000 * metadata['height_fraction'] if box else None result[stem] = name - atomic_write_json(cache_path, {'name': name, 'crop_sha256': crop_hash, 'names_text': names_text}) + atomic_write_json(cache_path, { + 'name': name, 'crop_sha256': crop_hash, 'names_text': names_text, + 'source_sha256': metadata['source_sha256'], + 'box_2d': box, # 0–1000 coordinates relative to the name crop. + 'name_bottom_height_fraction': bottom, # Fraction of full page 1. + }) except Exception as exc: # Remote failures must preserve the existing fallback. print(f"Name recognition failed for {grid['image']}: {type(exc).__name__}: {exc}; using label/name fallback.") # Do not cache transport/schema failures: a resumed run should retry. diff --git a/tests/test_left_margin.py b/tests/test_left_margin.py index ab1a65e..d6523b0 100644 --- a/tests/test_left_margin.py +++ b/tests/test_left_margin.py @@ -53,23 +53,53 @@ class LeftMarginTests(unittest.TestCase): group = result['groups'][0] self.assertEqual(group['edge_mm'], 42) self.assertEqual(group['left_mm'], 22) - self.assertEqual(group['discarded_bands'], 14) - self.assertEqual(result['starts_px'], [round(22 * 200 / 25.4)] * 4) + self.assertEqual(group['aggregation'], 'median-of-page-deciles') + self.assertEqual(result['starts_px'], [round(22 * 200 / 25.4)] * 3 + [0]) + + def test_short_exception_requires_prior_margin_and_one_small_block(self): + # At 4 mm per band, eight bands cover four standard 8 mm lines. + for prior, low_positions, expected in ( + (2, list(range(8)), True), + (2, list(range(9)), False), + (1, list(range(8)), False), + (0, list(range(8)), False), + (2, [0, 2, 4, 6, 8], False)): + with self.subTest(prior=prior, low_positions=low_positions): + images = [Image.new('RGB', (1654, 200)) for _ in range(5)] + def estimate(image, _dpi): + index = next(i for i, candidate in enumerate(images) if candidate is image) + bands = [{'start_mm': 5 if index == prior and i in low_positions else 42, + 'top_mm': i * 4, 'bottom_mm': (i + 1) * 4} for i in range(30)] + starts = [b['start_mm'] for b in bands] + return {'edge_mm': left_margin.trimmed_left_edge(starts), + 'bands': len(bands), 'body_bands': bands, 'band_starts_mm': starts} + with patch.object(left_margin, 'estimate_page', side_effect=estimate): + result = left_margin.analyse_margins(images, 200, 1654) + self.assertEqual(result['pages'][prior]['short_margin_exception'], expected) + self.assertEqual(result['starts_px'][prior], round(22 * 200 / 25.4) if expected else 0) + self.assertGreater(result['starts_px'][-1], 0) + + def test_short_exception_rejects_inconsistent_or_insufficient_remainder(self): + for remainder in ([65] * 22, [42] * 5): + starts = [5] * 8 + remainder + estimate = {'body_bands': [{'start_mm': x, 'top_mm': i * 4, 'bottom_mm': (i + 1) * 4} + for i, x in enumerate(starts)]} + self.assertFalse(left_margin.short_margin_exception(estimate, 42)) def test_repeated_left_writing_beyond_ten_percent_is_retained(self): self.assertEqual(left_margin.trimmed_left_edge([10] * 20 + [42] * 80), 10) def test_disagreeing_pages_use_half_margin_when_it_removes_more(self): - result = self.analyse([25, 31, 68, 49]) - self.assertEqual(result['groups'][0]['left_mm'], 12.5) - self.assertEqual(result['starts_px'][1:], [round(12.5 * 200 / 25.4)] * 3) + result = self.analyse([33, 35, 41, 39]) + self.assertEqual(result['groups'][0]['left_mm'], 18.5) + self.assertEqual(result['starts_px'][1:], [round(18.5 * 200 / 25.4)] * 3) def test_small_margin_starts_at_left(self): result = self.analyse([10, 15, 8, 13]) self.assertEqual(result['groups'][0]['left_mm'], 0) self.assertEqual(result['starts_px'], [0] * 4) - def test_two_consecutive_no_margin_pages_enforce_left_edge_afterwards(self): + def test_low_pages_do_not_cancel_later_pages_margin(self): images = [Image.new('RGB', (1654, 200)) for _ in range(8)] # Low pages have fewer occupied bands, so pooling still proposes a margin. edges = [42, 42, 10, 12, 42, 42, 42, 42] @@ -81,8 +111,8 @@ class LeftMarginTests(unittest.TestCase): with patch.object(left_margin, 'estimate_page', side_effect=estimate): result = left_margin.analyse_margins(images, 200, 1654) self.assertEqual(result['groups'][0]['left_mm'], 22) - self.assertEqual(result['no_margin_from_page'], 3) - self.assertEqual(result['starts_px'], [round(22 * 200 / 25.4)] * 2 + [0] * 6) + self.assertIsNone(result['no_margin_from_page']) + self.assertEqual(result['starts_px'], [round(22 * 200 / 25.4)] * 2 + [0] * 2 + [round(22 * 200 / 25.4)] * 4) def test_isolated_low_or_blank_pages_do_not_trigger_no_margin_rule(self): for edges in ([42, 10, 42, 12], [42, 10, None, 12]): @@ -90,21 +120,24 @@ class LeftMarginTests(unittest.TestCase): result = self.analyse(edges) self.assertIsNone(result['no_margin_from_page']) - def test_two_no_margin_pages_enforce_rule_across_later_width_changes(self): + def test_low_pages_do_not_cancel_later_width_groups(self): result = self.analyse([42, 42, 10, 12, 42, 42], widths=[1654, 1654, 1654, 1654, 2000, 2000], band_counts=[100, 100, 6, 6, 100, 100]) - self.assertEqual(result['no_margin_from_page'], 3) - self.assertGreater(result['starts_px'][0], 0) - self.assertEqual(result['starts_px'][2:], [0] * 4) + self.assertIsNone(result['no_margin_from_page']) + # First group's median is 26 mm; the later width group still has 42 mm. + self.assertEqual(result['starts_px'][0], 0) + self.assertEqual(result['starts_px'][2:4], [0, 0]) + self.assertTrue(all(result['starts_px'][4:])) def test_larger_cut_distance_and_no_margin_threshold(self): - for edge, expected in ((0, 0), (19.9, 0), (20, 10), (20.1, 10.05), (28.4, 14.2), (40, 20), (60, 40)): + for edge, expected in ((0, 0), (20, 0), (25.4, 0), (26.4, 0), (27.9, 0), (28, 0), (31, 0), (32.5, 0), (32.9, 0), (33, 16.5), (33.1, 16.55), (40, 20), (60, 40)): with self.subTest(edge=edge): result = self.analyse([edge] * 4) self.assertEqual(result['starts_px'][1:], [round(expected * 200 / 25.4)] * 3) - self.assertEqual(result['groups'][0]['status'], 'near-edge' if edge < 20 else 'margin') + self.assertEqual(result['groups'][0]['status'], 'near-edge' if edge < 33 else 'margin') self.assertEqual(result['margin_clearance_mm'], 20) + self.assertEqual(result['minimum_margin_mm'], 33) def test_body_bands_ignore_header_footer_and_abstain_on_sparse_ink(self): mask = np.zeros((1169, 827), bool) @@ -155,7 +188,7 @@ class LeftMarginTests(unittest.TestCase): self.assertTrue(group['labels_in_margin']) self.assertEqual(result['starts_px'], [0] * 6) - def test_unsampled_coherent_page_can_move_the_whole_group_left(self): + def test_any_coherent_page_can_move_the_whole_group_left(self): images = [Image.new('RGB', (1654, 200)) for _ in range(6)] def estimate(image, _dpi): labels = [5.] * 20 if image is images[2] else [] @@ -164,7 +197,7 @@ class LeftMarginTests(unittest.TestCase): with patch.object(left_margin, 'estimate_page', side_effect=estimate): result = left_margin.analyse_margins(images, 200, 1654) group = result['groups'][0] - self.assertNotIn(3, [p['page'] for p in group['samples']]) + self.assertIn(3, [p['page'] for p in group['samples']]) self.assertIn(3, group['coherent_pages']) self.assertEqual(result['starts_px'], [0] * 6) @@ -193,6 +226,69 @@ class LeftMarginTests(unittest.TestCase): self.assertFalse(result['groups'][0]['labels_in_margin']) self.assertGreater(result['starts_px'][0], 0) + def test_localized_name_bottom_reveals_labels_above_old_cutoff(self): + mask = np.zeros((1169, 827), bool) + mask[50:150, 4:100] = True # Name above localized bottom. + mask[250:265, 30:110] = True # Heading between 20% and 30%. + for y in range(400, 920, 35): + mask[y:y + 5, 170:270] = True + old = left_margin.body_edge(mask) + localized = left_margin.body_edge(mask, top_fraction=.20) + self.assertGreater(min(old['label_band_starts_mm']), 40) + self.assertAlmostEqual(min(localized['label_band_starts_mm']), 7.62) + self.assertTrue(all(start > 2 for start in localized['label_band_starts_mm'])) + + def test_first_page_name_fraction_is_reused_on_every_page(self): + images = [Image.new('RGB', (1654, 200)) for _ in range(6)] + estimate = {'edge_mm': 42, 'bands': 12, 'band_starts_mm': [42.] * 12, + 'label_band_starts_mm': [42.] * 12} + with patch.object(left_margin, 'estimate_page', return_value=estimate) as estimator: + result = left_margin.analyse_margins(images, 200, 1654, + name_bottom_height_fraction=.14) + self.assertEqual(estimator.call_count, 6) + self.assertTrue(all(call.kwargs == {'top_fraction': .14} + for call in estimator.call_args_list)) + self.assertEqual(result['body_height_fraction'], [.14, .82]) + self.assertEqual(result['label_height_fraction'], [.14, .92]) + self.assertEqual(result['top_cutoff_source'], 'localized-name') + + def test_unavailable_or_invalid_name_fraction_keeps_fallback(self): + images = [Image.new('RGB', (1654, 200))] + estimate = {'edge_mm': None, 'bands': 0, 'band_starts_mm': [], + 'label_band_starts_mm': []} + for value in (None, 0, -1, 1.1, True, '0.1', float('nan')): + with self.subTest(value=value), patch.object(left_margin, 'estimate_page', return_value=estimate): + result = left_margin.analyse_margins(images, 200, 1654, + name_bottom_height_fraction=value) + self.assertEqual(result['body_height_fraction'][0], .30) + self.assertEqual(result['top_cutoff_source'], 'fallback') + self.assertIsNone(result['name_bottom_height_fraction']) + self.assertIsNone(left_margin.body_edge(np.zeros((100, 100), bool), .95)['edge_mm']) + + def test_cutleft_reads_validated_name_cache_and_rejects_stale_source(self): + import json + import tempfile + from copienator.annotation_layout import fingerprint + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'Copies').mkdir() + names = root / 'Cutleft' / 'Names' + names.mkdir(parents=True) + pdf = root / 'Copies' / 'Copie01.pdf' + pdf.write_bytes(b'original-pdf') + crop = names / 'Copie01.jpg' + crop.write_bytes(b'name-crop') + (names / 'Copie01_result.json').write_text(json.dumps({ + 'name': 'Student', 'source_sha256': fingerprint(pdf), + 'crop_sha256': fingerprint(crop), 'name_bottom_height_fraction': .14})) + images = [Image.new('RGB', (900, 600), 'white')] + for expected in (.14, None): + with patch.object(cutleft, 'get_pdf_pages', return_value=images), \ + patch.object(cutleft, 'analyse_margins', return_value={'starts_px': [0], 'groups': []}) as analyse: + self.assertIsNotNone(cutleft.process_single_pdf(pdf)) + self.assertEqual(analyse.call_args.kwargs['name_bottom_height_fraction'], expected) + pdf.write_bytes(b'changed-pdf') + def test_real_mask_rejects_ruling_and_preserves_handwriting(self): image = Image.new('RGB', (827, 1169), 'white') draw = ImageDraw.Draw(image) @@ -224,7 +320,7 @@ class LeftMarginTests(unittest.TestCase): def test_clear_margin_uses_same_conservative_edge_on_every_page(self): result = self.analyse([43, 41, 46, 44]) self.assertEqual(result['groups'][0]['status'], 'margin') - self.assertEqual(result['starts_px'], [round(21 * 200 / 25.4)] * 4) + self.assertEqual(result['starts_px'], [round(23.5 * 200 / 25.4)] * 4) self.assertNotIn('first_page_body_start_px', result) def test_sparse_groups_stay_at_left(self): diff --git a/tests/test_margin_benchmark.py b/tests/test_margin_benchmark.py index aea1fe3..d383695 100644 --- a/tests/test_margin_benchmark.py +++ b/tests/test_margin_benchmark.py @@ -39,6 +39,11 @@ class MarginBenchmarkTests(unittest.TestCase): case['sha256']) self.assertEqual([p['page'] for p in case['pages']], list(range(1, case['page_count'] + 1))) self.assertTrue(case['explanation']) + name_box = case['name_localization']['box_px'] + self.assertEqual(case['name_localization']['page_height_px'], + case['recorded_schema']['page_crops'][0]['height_px']) + self.assertAlmostEqual(case['name_bottom_height_fraction'], + name_box[3] / case['name_localization']['page_height_px']) self.assertTrue(any(p['labels'] for p in case['pages'])) for expected, recorded in zip(case['pages'], case['recorded_schema']['page_crops'], strict=True): start = round(expected['suggested_start_mm'] * 200 / 25.4) diff --git a/tests/test_name_grids.py b/tests/test_name_grids.py index 74440e8..def8a58 100644 --- a/tests/test_name_grids.py +++ b/tests/test_name_grids.py @@ -90,6 +90,51 @@ class NameGridTests(unittest.TestCase): self.identify({'Copie01': []}) self.assertEqual(self.client.models.generate_content.call_count, 2) + def test_name_box_is_saved_in_full_page_height_units(self): + pdf = self.crop(1) + self.client.models.generate_content.return_value = Mock(text=json.dumps({'cells': [ + {'cell': 1, 'name': 'Alice', 'box_2d': [100, 20, 800, 950]}, + ]})) + self.assertEqual(self.identify({'Copie01': []}), {'Copie01': 'Alice'}) + cached = read_json(self.directory / 'Copie01_result.json') + self.assertEqual(cached['box_2d'], [100, 20, 800, 950]) + self.assertAlmostEqual(cached['name_bottom_height_fraction'], .8 * 584 / 2338) + self.assertAlmostEqual(names.name_bottom_height_fraction(self.workspace, 'Copie01'), + .8 * 584 / 2338) + self.identify({'Copie01': []}) + self.assertEqual(self.client.models.generate_content.call_count, 1) + pdf.write_bytes(b'changed') + self.assertIsNone(names.name_bottom_height_fraction(self.workspace, 'Copie01')) + + def test_invalid_or_unmatched_boxes_do_not_produce_a_height(self): + self.crop(1) + for name, box in [('Alice', [800, 20, 100, 950]), + ('Alice', [0, 0, 1001, 900]), ('Alice', [1, 2]), + ('Alice', ['bad', 0, 800, 900]), ('Alice', None), + ('Outsider', [100, 20, 800, 950]), + ('Missing', [100, 20, 800, 950])]: + with self.subTest(name=name, box=box): + self.client.models.generate_content.return_value = Mock(text=json.dumps({'cells': [ + {'cell': 1, 'name': name, 'box_2d': box}, + ]})) + self.assertEqual(self.identify({'Copie01': []}, overwrite=True), + {'Copie01': 'Alice' if name == 'Alice' else 'Missing'}) + self.assertIsNone(names.name_bottom_height_fraction(self.workspace, 'Copie01')) + + def test_legacy_name_cache_is_refreshed_to_request_position(self): + self.crop(1) + self.client.models.generate_content.return_value = Mock(text=json.dumps({'cells': [ + {'cell': 1, 'name': 'Alice', 'box_2d': [100, 20, 800, 950]}, + ]})) + self.identify({'Copie01': []}) + path = self.directory / 'Copie01_result.json' + cached = read_json(path) + del cached['name_bottom_height_fraction'] + path.write_text(json.dumps(cached)) + self.identify({'Copie01': []}) + self.assertEqual(self.client.models.generate_content.call_count, 2) + self.assertIsNotNone(names.name_bottom_height_fraction(self.workspace, 'Copie01')) + def test_known_and_missing_names_choose_different_label_requests(self): for number in (1, 2): self.crop(number)