Margin detection update

copienator/commands/cutleft.py

@@ -29,6 +29,7 @@ from copienator.filesystem import staged_files
 from copienator.copy_errors import clear_copy_error, mark_copy_error, marked_copy_paths
 from copienator.configuration import SHOW_PERSONAL_STEPS
 from copienator.left_margin import analyse_margins
+from copienator.name_grids import name_bottom_height_fraction
 from copienator.annotation_layout import fingerprint
 
 DELIMITER_WIDTH = 5
@@ -133,7 +134,10 @@ def process_single_pdf(
             return None
         reference_width = normal_page_width(pdf_path, images)
         if automatic_margin and max_per_file != 1 and margin_analysis is None:
-            margin_analysis = analyse_margins(images, RENDER_DPI, reference_width)
+            name_bottom = name_bottom_height_fraction(
+                EvaluationWorkspace(pdf_path.parent.parent), pdf_path.stem)
+            margin_analysis = analyse_margins(
+                images, RENDER_DPI, reference_width, name_bottom_height_fraction=name_bottom)
         full_width_pages = []
         page_crops = []
         for page_index, image in enumerate(images):

copienator/left_margin.py

@@ -10,7 +10,8 @@ from copienator.ink_detection import detect_bounds
 ANALYSIS_DPI = 100
 WIDTH_TOLERANCE = .08
 # Copie01's name/class block reaches about 27% of the full page height.
-# Start below it with clearance, including on later pages with repeated names.
+# Fallback when name localization is unavailable. A localized first-page
+# name/class bottom replaces this cutoff on all pages, including repeated names.
 BODY_TOP = .30
 BODY_BOTTOM = .82
 LABEL_BOTTOM = .92
@@ -20,7 +21,10 @@ LABEL_MARGIN_GAP_MM = 10
 DISCARD_LEFT_FRACTION = .10
 STANDARD_MARGIN_MM = 40
 MARGIN_CLEARANCE_MM = 20
-MIN_MARGIN_MM = MARGIN_CLEARANCE_MM
+# Minimum usable margin is independent of the strip clearance.
+MIN_MARGIN_MM = 33
+STANDARD_LINE_HEIGHT_MM = 8
+MAX_EXCEPTION_LINES = 4
 
 
 def trimmed_left_edge(starts: list[float]) -> float | None:
@@ -51,38 +55,60 @@ def handwriting_mask(rgb: np.ndarray) -> np.ndarray:
     return keep[labels] & diagnostics['masks']['foreground']
 
 
-def occupied_band_starts(mask: np.ndarray, bottom_fraction: float) -> list[float]:
+def occupied_bands(mask: np.ndarray, bottom_fraction: float, top_fraction: float = BODY_TOP) -> list[dict]:
     """Ignore empty bands and writing starting beyond the page midpoint."""
     height, width = mask.shape
     band_height = max(1, round(4 * ANALYSIS_DPI / 25.4))
     bottom = round(bottom_fraction * height)
     starts = []
-    for y in range(round(BODY_TOP * height), bottom, band_height):
+    for y in range(round(top_fraction * height), bottom, band_height):
         band = mask[y:min(y + band_height, bottom)]
         xs = np.flatnonzero(band.sum(axis=0) >= 2)
         if band.sum() >= 40 and len(xs) >= 12 and xs[0] <= width / 2:
-            starts.append(float(xs[0]) * 25.4 / ANALYSIS_DPI)
+            starts.append({'start_mm': float(xs[0]) * 25.4 / ANALYSIS_DPI,
+                           'top_mm': y * 25.4 / ANALYSIS_DPI,
+                           'bottom_mm': min(y + band_height, bottom) * 25.4 / ANALYSIS_DPI})
     return starts
 
 
-def body_edge(mask: np.ndarray) -> dict:
-    """Keep the body decile; collect a wider 30–92% label-search window."""
-    starts = occupied_band_starts(mask, BODY_BOTTOM)
+def occupied_band_starts(mask: np.ndarray, bottom_fraction: float, top_fraction: float = BODY_TOP) -> list[float]:
+    return [band['start_mm'] for band in occupied_bands(mask, bottom_fraction, top_fraction)]
+
+
+def short_margin_exception(estimate: dict, margin: float) -> bool:
+    """One block of at most four 8 mm lines may depart from an established margin."""
+    bands = estimate.get('body_bands', [])
+    low = [band for band in bands if band['start_mm'] < MIN_MARGIN_MM]
+    if not low or low[-1]['bottom_mm'] - low[0]['top_mm'] > MAX_EXCEPTION_LINES * STANDARD_LINE_HEIGHT_MM:
+        return False
+    # Require a majority of occupied bands outside that entire block, and a
+    # consistent decile there. Coordinates retain blank gaps between bands.
+    rest = [band['start_mm'] for band in bands
+            if band['bottom_mm'] <= low[0]['top_mm'] or band['top_mm'] >= low[-1]['bottom_mm']]
+    return (len(rest) >= 6 and len(rest) > len(bands) / 2
+            and abs(trimmed_left_edge(rest) - margin) <= COHERENT_MARGIN_TOLERANCE_MM)
+
+
+def body_edge(mask: np.ndarray, top_fraction: float = BODY_TOP) -> dict:
+    """Collect body and label bands below the name or the fallback header cutoff."""
+    bands = occupied_bands(mask, BODY_BOTTOM, top_fraction)
+    starts = [band['start_mm'] for band in bands]
     return {
         'edge_mm': trimmed_left_edge(starts) if len(starts) >= 6 else None,
         'bands': len(starts),
+        'body_bands': bands,
         'band_starts_mm': starts,
-        'label_band_starts_mm': occupied_band_starts(mask, LABEL_BOTTOM),
+        'label_band_starts_mm': occupied_band_starts(mask, LABEL_BOTTOM, top_fraction),
     }
 
 
-def estimate_page(image: Image.Image, render_dpi: int) -> dict:
+def estimate_page(image: Image.Image, render_dpi: int, *, top_fraction: float = BODY_TOP) -> dict:
     scale = ANALYSIS_DPI / render_dpi
     small = image.convert('RGB').resize(
         (max(1, round(image.width * scale)), max(1, round(image.height * scale))),
         Image.Resampling.LANCZOS,
     )
-    return body_edge(handwriting_mask(np.asarray(small)))
+    return body_edge(handwriting_mask(np.asarray(small)), top_fraction)
 
 
 def format_groups(images: list[Image.Image], reference_width: float) -> list[list[int]]:
@@ -100,36 +126,34 @@ def format_groups(images: list[Image.Image], reference_width: float) -> list[lis
     return groups
 
 
-def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width: float) -> dict:
+def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width: float,
+                    *, name_bottom_height_fraction: float | None = None) -> dict:
     """Estimate body margins, then retain left labels supported across coherent pages."""
+    name_localized = (type(name_bottom_height_fraction) in (int, float)
+                      and 0 < name_bottom_height_fraction <= 1)
+    analysis_top = float(name_bottom_height_fraction) if name_localized else BODY_TOP
     starts = [0] * len(images)
     groups = []
     page_estimates = {}
 
     def page_estimate(index):
         if index not in page_estimates:
-            page_estimates[index] = estimate_page(images[index], render_dpi)
+            page_estimates[index] = (estimate_page(images[index], render_dpi, top_fraction=analysis_top)
+                                     if name_localized else estimate_page(images[index], render_dpi))
         return page_estimates[index]
 
     for indices in format_groups(images, reference_width):
-        # Spread samples through the copy rather than relying on the header page.
-        sample = sorted({indices[0], indices[min(1, len(indices) - 1)],
-                         indices[len(indices) // 2], indices[-1]})
-        estimates = {index: page_estimate(index) for index in sample}
+        estimates = {index: page_estimate(index) for index in indices}
         values = [item['edge_mm'] for item in estimates.values() if item['edge_mm'] is not None]
-        # Trim across the copy's format group, not separately on each page:
-        # one marginal comment must not veto the other pages' shared margin.
         band_starts = [start for item in estimates.values() for start in item['band_starts_mm']]
         status = 'insufficient-ink'
-        edge = trimmed_left_edge(band_starts)
+        edge = round(float(np.median(values)), 1) if values else None
         left_mm = 0.0
         if len(values) >= min(2, len(indices)):
             status = 'near-edge' if edge < MIN_MARGIN_MM else 'margin'
             left_mm = (0.0 if edge < MIN_MARGIN_MM else
                        max(edge / 2, edge - MARGIN_CLEARANCE_MM))
-        # Use ALL coherent pages, not just the four body-margin samples.
-        # A sparse/low/outlying page cannot supply evidence for this format's
-        # established body margin. Its normal fallback behaviour remains below.
+        # Only independently coherent pages supply evidence for labels in the margin.
         coherent = [index for index in indices
                     if edge is not None and page_estimate(index)['edge_mm'] is not None
                     and page_estimate(index)['edge_mm'] >= MIN_MARGIN_MM
@@ -151,8 +175,18 @@ def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width:
             status = 'narrow-page'
             left_mm = 0.0
         start = round(left_mm * render_dpi / 25.4)
+        previous_coherent = 0
         for index in indices:
-            starts[index] = start
+            estimate = page_estimate(index)
+            low = estimate['edge_mm'] is not None and estimate['edge_mm'] < MIN_MARGIN_MM
+            tolerated = (status == 'margin' and low and previous_coherent >= 2
+                         and short_margin_exception(estimate, edge))
+            starts[index] = 0 if low and not tolerated else start
+            estimate['short_margin_exception'] = tolerated
+            estimate['decision'] = ('short-exception' if tolerated else
+                                    'page-near-edge' if low else status)
+            if index in coherent:
+                previous_coherent += 1
         groups.append({
             'pages': [index + 1 for index in indices],
             'status': status,
@@ -164,22 +198,9 @@ def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width:
             'label_edge_mm': label_edge,
             'label_bands': len(label_starts),
             'bands': len(band_starts),
-            'discarded_bands': int(len(band_starts) * DISCARD_LEFT_FRACTION),
+            'aggregation': 'median-of-page-deciles',
             'samples': [{'page': index + 1, **item} for index, item in estimates.items()],
         })
-    # Inspect actual neighbours, including pages outside the format samples.
-    # Blank/uncertain pages do not establish or extend a no-margin run.
-    no_margin_run = 0
-    no_margin_from_page = None
-    for index in range(len(images)):
-        edge = page_estimate(index)['edge_mm']
-        no_margin_run = no_margin_run + 1 if edge is not None and edge < MIN_MARGIN_MM else 0
-        if no_margin_run >= 2:
-            run_start = index - 1
-            no_margin_from_page = run_start + 1
-            starts[run_start:] = [0] * (len(images) - run_start)
-            break
-
     pages = []
     for index in range(len(images)):
         estimate = page_estimate(index)
@@ -188,14 +209,18 @@ def analyse_margins(images: list[Image.Image], render_dpi: int, reference_width:
                       'status': 'unknown' if edge is None else
                       ('low' if edge < MIN_MARGIN_MM else 'normal')})
 
-    return {'version': 9, 'pages': pages, 'starts_px': starts, 'groups': groups,
-            'no_margin_from_page': no_margin_from_page,
+    return {'version': 13, 'pages': pages, 'starts_px': starts, 'groups': groups,
+            'no_margin_from_page': None,
             'discard_left_fraction': DISCARD_LEFT_FRACTION, 'margin_clearance_mm': MARGIN_CLEARANCE_MM,
             'cut_fraction_of_margin': .5, 'cut_rule': 'maximum',
             'standard_margin_mm': STANDARD_MARGIN_MM, 'minimum_margin_mm': MIN_MARGIN_MM,
-            'body_height_fraction': [BODY_TOP, BODY_BOTTOM],
-            'label_height_fraction': [BODY_TOP, LABEL_BOTTOM],
+            'name_bottom_height_fraction': name_bottom_height_fraction if name_localized else None,
+            'top_cutoff_source': 'localized-name' if name_localized else 'fallback',
+            'body_height_fraction': [analysis_top, BODY_BOTTOM],
+            'label_height_fraction': [analysis_top, LABEL_BOTTOM],
             'label_left_fraction': LABEL_LEFT_FRACTION,
             'coherent_margin_tolerance_mm': COHERENT_MARGIN_TOLERANCE_MM,
             'label_margin_gap_mm': LABEL_MARGIN_GAP_MM,
-            'maximum_band_start_width_fraction': .5}
+            'maximum_band_start_width_fraction': .5,
+            'standard_line_height_mm': STANDARD_LINE_HEIGHT_MM,
+            'max_exception_lines': MAX_EXCEPTION_LINES}

copienator/name_grids.py

@@ -2,10 +2,11 @@
 from __future__ import annotations
 
 from pathlib import Path
+import math
 
 from google.genai import types
 from PIL import Image, ImageDraw
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 from copienator import atomic_write_json, read_json
 from copienator.annotation_layout import fingerprint
@@ -26,7 +27,7 @@ def save_crop(page: Image.Image, pdf: Path, directory: Path, dpi: int) -> None:
         crop.save(staging / f"{pdf.stem}.jpg", "JPEG", quality=95)
         atomic_write_json(staging / f"{pdf.stem}.json", {
             "source_sha256": fingerprint(pdf), "width_mm": MARGIN_MM,
-            "height_fraction": 0.25,
+            "height_fraction": crop.height / page.height,
         })
 
 
@@ -77,12 +78,44 @@ def build_grids(directory: Path, copies_dir: Path) -> list[dict]:
 class NameCell(BaseModel):
     cell: int = Field(strict=True, ge=1, le=24)
     name: str
+    box_2d: list[float] | None = Field(
+        default=None, description="[ymin, xmin, ymax, xmax], 0–1000 relative to the individual crop, excluding its header")
+
+    @field_validator('box_2d', mode='before')
+    @classmethod
+    def valid_box(cls, value):
+        # Bad localization must not discard an otherwise readable name.
+        if (not isinstance(value, list) or len(value) != 4
+                or any(type(v) not in (int, float) or not math.isfinite(v)
+                       or not 0 <= v <= 1000 for v in value)):
+            return None
+        return value if value[0] < value[2] and value[1] < value[3] else None
 
 
 class NameResponse(BaseModel):
     cells: list[NameCell]
 
 
+def name_bottom_height_fraction(workspace, stem: str) -> float | None:
+    """Read the name/class bottom on page 1, in cutleft's 0–1 height units.
+
+    Returns None when localization is unavailable or the source/crop changed.
+    This is intended as a future lower boundary for margin analysis.
+    """
+    directory = workspace.cutleft_dir / 'Names'
+    try:
+        cached = read_json(directory / f'{stem}_result.json')
+        value = cached['name_bottom_height_fraction']
+        if (cached['source_sha256'] == fingerprint(workspace.copies_dir / f'{stem}.pdf')
+                and cached['crop_sha256'] == fingerprint(directory / f'{stem}.jpg')
+                and cached['name'] != MISSING
+                and type(value) in (int, float) and 0 < value <= 1):
+            return float(value)
+    except (OSError, ValueError, KeyError, TypeError):
+        pass
+    return None
+
+
 def identify_names(workspace, groups, *, client, model: str, names_text: str, overwrite: bool,
                    thinking_level: str | None = "low") -> dict[str, str]:
     """Identify pending copies before label requests; safely fall back on any failure."""
@@ -101,6 +134,7 @@ def identify_names(workspace, groups, *, client, model: str, names_text: str, ov
             try:
                 cached = read_json(cache_path)
                 if (not overwrite and cached['crop_sha256'] == crop_hash
+                        and 'name_bottom_height_fraction' in cached
                         and cached['names_text'] == names_text
                         and cached['name'] in roster | {MISSING}):
                     result[stem] = cached['name']
@@ -115,11 +149,17 @@ def identify_names(workspace, groups, *, client, model: str, names_text: str, ov
 The image is a grid with exactly 6 columns and 4 rows. Read rows left to right,
 then top to bottom. Thick black borders separate cells; headers show row, column,
 and cell number. Each cell belongs to a DIFFERENT student. Never carry a name
-from one cell into another. Ignore class names such as MPSI 3.
+from one cell into another. Do not include class names such as MPSI 3 in the name.
 Occupied cells are exactly:
 {positions}
 All other cells are EMPTY padding: do not return them.
-Return JSON {{"cells": [{{"cell": 1, "name": "exact authorized name"}}]}}.
+Return JSON {{"cells": [{{"cell": 1, "name": "exact authorized name", "box_2d": [100, 50, 400, 900]}}]}}.
+For each identified student, box_2d must tightly enclose the handwritten name AND
+any adjacent class designation, including a class written just below the name.
+Use [ymin, xmin, ymax, xmax] coordinates normalized to 0–1000 relative to that
+individual page crop, NOT the whole grid. Exclude the numbered header, black
+borders and any padding outside the crop. Return box_2d=null for Missing names
+or if the position cannot be determined reliably.
 Return one entry per occupied cell, using its cell number. Match only confidently
 readable handwriting to an exact authorized name below. Do not guess or assign
 names by elimination. Return "Missing" for an unreadable, absent, or ambiguous name.
@@ -143,12 +183,21 @@ Authorized names:
             entries = NameResponse.model_validate_json(response.text).cells
             by_cell = {}
             for entry in entries:
-                by_cell.setdefault(entry.cell, []).append(entry.name)
+                by_cell.setdefault(entry.cell, []).append(entry)
             for stem, (cell, crop_hash, cache_path) in pending.items():
                 candidates = by_cell.get(cell['cell'], [])
-                name = candidates[0] if len(candidates) == 1 and candidates[0] in roster else MISSING
+                entry = candidates[0] if len(candidates) == 1 else None
+                name = entry.name if entry is not None and entry.name in roster else MISSING
+                box = entry.box_2d if name != MISSING else None
+                metadata = read_json(directory / f'{stem}.json')
+                bottom = box[2] / 1000 * metadata['height_fraction'] if box else None
                 result[stem] = name
-                atomic_write_json(cache_path, {'name': name, 'crop_sha256': crop_hash, 'names_text': names_text})
+                atomic_write_json(cache_path, {
+                    'name': name, 'crop_sha256': crop_hash, 'names_text': names_text,
+                    'source_sha256': metadata['source_sha256'],
+                    'box_2d': box,  # 0–1000 coordinates relative to the name crop.
+                    'name_bottom_height_fraction': bottom,  # Fraction of full page 1.
+                })
         except Exception as exc:  # Remote failures must preserve the existing fallback.
             print(f"Name recognition failed for {grid['image']}: {type(exc).__name__}: {exc}; using label/name fallback.")
             # Do not cache transport/schema failures: a resumed run should retry.