modest improvement to cropping (safer)

copienator/commands/splitting_int.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import math
 import shutil
 import tempfile
 from collections import defaultdict
@@ -23,6 +24,7 @@ from copienator import (
 from copienator.filesystem import staged_directory
 
 SQUARE = 1000 // 38
+ANSWER_TOP_PADDING_POINTS = 4 * 72 / 25.4
 Coordinate = tuple[str, int, int, int, int, int]
 ParsedCoordinate = tuple[str, str, int, int, int, int, int]
 
@@ -171,7 +173,15 @@ def _render_split_outputs(
 
                 for page_number in range(start_page, end_page + 1):
                     page = document[page_number]
-                    y0 = (y_start / 1000) * page.rect.height if page_number == start_page else 0
+                    y0 = (
+                        math.floor(max(
+                            0,
+                            (y_start / 1000) * page.rect.height
+                            - ANSWER_TOP_PADDING_POINTS,
+                        ))
+                        if page_number == start_page
+                        else 0
+                    )
                     y1 = (end_y / 1000) * page.rect.height if page_number == end_page else page.rect.height
                     clip = pymupdf.Rect(
                         fraction_x0 * page.rect.width, y0,

copienator/crop_blank_margins.py

@@ -110,7 +110,8 @@ def main() -> None:
               flush=True)
         (args.output/'report.json').write_text(json.dumps(rows, indent=2)+'\n')
     with (args.output/'report.csv').open('w') as stream:
-        writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
+        fieldnames = list(dict.fromkeys(key for row in rows for key in row))
+        writer = csv.DictWriter(stream, fieldnames=fieldnames)
         writer.writeheader()
         writer.writerows(rows)
     cards = []

copienator/ink_detection.py

@@ -65,13 +65,26 @@ def _large_blank_ink(gray: np.ndarray, clean: np.ndarray, dpi: float):
     n, labels, stats, centers = cv2.connectedComponentsWithStats(
         (residual > 35).astype(np.uint8), 8)
     keep = np.zeros(n, bool)
+    xx = stats[1:, 0]
     ww, hh, area = stats[1:, 2], stats[1:, 3], stats[1:, 4]
-    keep[1:] = ((area >= .8*px*px) & (np.minimum(ww, hh) >= .6*px)
+    density = area / (ww*hh)
-                & (area / (ww*hh) > .3)
+    shortest, longest = np.minimum(ww, hh), np.maximum(ww, hh)
-                & (np.maximum(ww, hh) < 4*np.minimum(ww, hh)))
+    compact_ink = ((area >= .8*px*px) & (shortest >= .6*px)
+                   & (density > .3) & (longest < 4*shortest))
+    # Ruling suppression can fragment faint pencil handwriting into sparse,
+    # elongated components. Admit those moderately more readily in the central
+    # 80% of the sheet. The outermost 9% deliberately uses a stricter filter:
+    # punched holes, torn binding edges, and page numbers usually occur there.
+    center_x = xx + ww/2
+    central = (center_x > width*.1) & (center_x < width*.9)
+    central_ink = (central & (area >= .55*px*px) & (shortest >= .45*px)
+                   & (density > .18) & (longest < 6*shortest))
+    outer = (center_x < width*.09) | (center_x > width*.91)
+    outer_ink = (outer & (area >= 1.2*px*px) & (shortest >= .8*px)
+                 & (density >= .4) & (longest < 3*shortest))
+    keep[1:] = np.where(outer, outer_ink, compact_ink | central_ink)
     # Use side columns as a prior, then require repeated size and alignment. An
     # isolated note in the same column remains eligible to protect the margin.
-    xx = stats[1:, 0]
     candidates = np.flatnonzero(
         ((xx < 15*px) | (xx+ww > width-15*px))
         & (ww > px) & (ww < 9*px) & (hh > px) & (hh < 12*px)) + 1

docs/crop_blank_margins.md

@@ -46,6 +46,11 @@ alignment in the outer 15 mm are treated as punched holes only when at least
 three span a substantial part of the page. Writing in the same side column still
 protects its margin.
 
+On confirmed ruled paper, faint sparse strokes in the central 80% of the page
+use a moderately more sensitive component filter. The outermost 9% on each side
+uses a stricter filter because punched holes, torn binding edges, and page
+numbers normally appear there.
+
 The large-blank refinement changes an edge only when it finds at least 30 mm of
 additional empty paper. A 2 mm recovery neighbourhood is applied before the
 normal padding. Apparently blank pages and pages without reliable ink seeds are

tests/test_ink_detection.py

@@ -108,6 +108,14 @@ class InkDetectionTests(unittest.TestCase):
                     .7,(190,190,190),2)
         self.assertGreater(detect_bounds(image,dpi=150)['bottom_px'],1100)
 
+    def test_sparse_central_pencil_on_dark_grid_survives(self):
+        image = self.dark_grid()
+        # Thin, pale handwriting crossing the ruling is split into sparse
+        # components. Its central position distinguishes it from page holes.
+        cv2.putText(image,'result',(330,1090),cv2.FONT_HERSHEY_SCRIPT_SIMPLEX,
+                    .85,(155,155,155),1)
+        self.assertGreater(detect_bounds(image,dpi=150)['bottom_px'],1090)
+
     def test_large_unruled_diagram_is_not_paper(self):
         image = np.full((1200,850,3),255,np.uint8)
         cv2.rectangle(image,(100,100),(750,1100),(20,20,20),3)
@@ -151,6 +159,12 @@ class InkDetectionTests(unittest.TestCase):
                 r = detect_bounds(image,dpi=150)
                 self.assertGreater(r['bottom_px'],1060)
 
+    def test_faint_page_number_in_outer_band_does_not_block_crop(self):
+        image = self.dark_grid()
+        cv2.putText(image,'4/',(3,1160),cv2.FONT_HERSHEY_SIMPLEX,
+                    .55,(130,130,130),1)
+        self.assertLess(detect_bounds(image,dpi=150)['bottom_px'],850)
+
     def test_faded_neutral_grid(self):
         image = self.dark_grid()
         image[np.all(image == 65,axis=2)] = 145

tests/test_splitting_geometry.py

@@ -8,7 +8,10 @@ import numpy as np
 import pymupdf
 from pdf2image import convert_from_path
 
-from copienator.commands.splitting_int import _render_split_outputs
+from copienator.commands.splitting_int import (
+    ANSWER_TOP_PADDING_POINTS,
+    _render_split_outputs,
+)
 
 
 class SplittingGeometryTests(unittest.TestCase):
@@ -60,7 +63,8 @@ class SplittingGeometryTests(unittest.TestCase):
                             else:
                                 coordinates = [("A", 0, 250, 260, 0, 100),
                                                ("_", 0, 711, 721, 0, 100)]
-                                bounds = (bounds[0], max(bounds[1], height // 4),
+                                answer_top = int(height // 4 - ANSWER_TOP_PADDING_POINTS)
+                                bounds = (bounds[0], max(bounds[1], answer_top),
                                           bounds[2], min(bounds[3], height * 3 // 4))
                             _render_split_outputs(source, coordinates, root)
                             self.assert_matches_preview(root / "A.pdf", preview, bounds)
@@ -85,7 +89,7 @@ class SplittingGeometryTests(unittest.TestCase):
             rendered = convert_from_path(root / "A.pdf", dpi=72)
             self.assertEqual(len(rendered), 2)
             for actual, preview, bounds in zip(rendered, previews,
-                                               [(40, 700, 780, 920), (20, 80, 760, 250)]):
+                                               [(40, 688, 780, 920), (20, 80, 760, 250)]):
                 expected = np.asarray(preview.crop(bounds)).astype(float)
                 actual = np.asarray(actual).astype(float)
                 self.assertEqual(actual.shape, expected.shape)