sebastien/Copies
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Miscs

Browse Source
This commit is contained in:
sebastien
2026-09-08 16:41:04 +02:00
parent db4ed2ef31
commit bf05272797
14 changed files with 725 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
copienator/commands/cutleft.py
+58 -16
View File
@@ -2,8 +2,8 @@ from __future__ import annotations
import argparse
import threading
import time
import tkinter as tk
from tkinter import messagebox
from collections.abc import Sequence
from functools import lru_cache
from pathlib import Path
@@ -23,6 +23,7 @@ from copienator import (
workspace_from_target,
)
from copienator.filesystem import staged_files
from copienator.copy_errors import clear_copy_error, mark_copy_error, marked_copy_paths
DELIMITER_WIDTH = 5
DELIMITER_COLOR = (0, 0, 0)
@@ -156,6 +157,11 @@ class ImageReviewer:
) -> None:
self.files = files
self.output_dir = output_dir
self.workspace = EvaluationWorkspace(output_dir.parent)
self.completed = False
self.had_errors = False
self.stop_prefetch = threading.Event()
self.current_result = None
self.index = 0
self.current_shift = 0
self.default_max_per_file = default_max_per_file
@@ -174,6 +180,8 @@ class ImageReviewer:
self.label_info = tk.Label(self.root, text="", font=("Arial", 12, "bold"))
self.label_info.pack(pady=5)
self.root.bind("<Return>", self.on_next)
self.root.bind("s", self.on_skip)
self.root.protocol("WM_DELETE_WINDOW", self.on_close)
self.root.bind("n", lambda _event: self.on_shift(50))
self.root.bind("N", lambda _event: self.on_shift(100))
self.root.bind("t", lambda _event: self.on_shift(-50))
@@ -194,20 +202,25 @@ class ImageReviewer:
def prefetch_worker(self) -> None:
processed_index = -1
while True:
while not self.stop_prefetch.is_set():
target = self.index + 1
if target < len(self.files) and target != processed_index:
get_pdf_pages(self.files[target])
try:
get_pdf_pages(self.files[target])
except Exception:
pass # The foreground review reports and flags conversion errors.
processed_index = target
time.sleep(0.05)
self.stop_prefetch.wait(0.05)
def load_current_image(self) -> None:
if self.index >= len(self.files):
print("All files processed.")
self.root.destroy()
self.completed = True
self.on_close()
return
self.is_processing = False
self.current_shift = 0
self.current_result = None
self.trigger_processing(self.files[self.index], self.current_shift)
def trigger_processing(self, pdf_path: Path, shift: int) -> None:
@@ -228,13 +241,13 @@ class ImageReviewer:
def check_manual_queue(self, pdf_path: Path) -> None:
try:
result = self.manual_queue.get_nowait()
self.is_processing = False
if result is None:
print(f"Failed to process {pdf_path.name}, skipping.")
self.index += 1
self.load_current_image()
self._mark_error(pdf_path, "Échec de la conversion pour la découpe des marges")
self._advance()
else:
self.handle_processing_result(result, pdf_path)
self.is_processing = False
except Empty:
self.root.after(100, lambda: self.check_manual_queue(pdf_path))
@@ -244,7 +257,7 @@ class ImageReviewer:
pdf_path: Path,
) -> None:
self.current_preview = result[0]
save_results(result, pdf_path, self.output_dir)
self.current_result = result
self.update_display(pdf_path.name, result[2])
def update_display(self, filename: str, schema: dict[str, object]) -> None:
@@ -258,7 +271,7 @@ class ImageReviewer:
f"[{self.index + 1}/{len(self.files)}] {filename} | "
f"Shift: {self.current_shift}px\nFiles: {schema['number_of_files']} | "
f"Cols: {schema['columns_per_file']}\n"
"Enter: Next | n: +50 | N: +100 | t: -50 | "
"Enter: Save and next | s: flag error and skip | n: +50 | N: +100 | t: -50 | "
"1: use single column"
),
fg="black",
@@ -272,8 +285,34 @@ class ImageReviewer:
self.trigger_processing(self.files[self.index], self.current_shift)
def on_next(self, _event: object) -> None:
if self.is_processing:
if self.is_processing or self.current_result is None:
return
pdf_path = self.files[self.index]
try:
save_results(self.current_result, pdf_path, self.output_dir)
clear_copy_error(self.workspace, pdf_path)
except Exception as exc:
self._mark_error(pdf_path, f"Échec de l’enregistrement : {exc}")
messagebox.showerror("Enregistrement impossible", str(exc), parent=self.root)
return
self._advance()
def _mark_error(self, pdf_path: Path, reason: str) -> None:
mark_copy_error(self.workspace, pdf_path, reason)
self.had_errors = True
print(f"[Copie signalée] {pdf_path.name}: {reason}")
def on_skip(self, _event=None) -> None:
if self.is_processing or self.index >= len(self.files):
return
self._mark_error(self.files[self.index], "Problème repéré pendant la découpe des marges")
self._advance()
def on_close(self) -> None:
self.stop_prefetch.set()
self.root.destroy()
def _advance(self) -> None:
self.index += 1
self.current_shift = 0
self.current_max_per_file = self.default_max_per_file
@@ -304,23 +343,27 @@ def run(
target: Path,
*,
fullpage: bool = False,
marked: bool = False,
) -> ExitCode:
files = _selected_files(workspace, target)
files = marked_copy_paths(workspace) if marked else _selected_files(workspace, target)
if not files:
print("No PDF files found.")
return ExitCode.SUCCESS
workspace.cutleft_dir.mkdir(parents=True, exist_ok=True)
_get_pdf_pages_cached.cache_clear()
ImageReviewer(
reviewer = ImageReviewer(
files,
workspace.cutleft_dir,
default_max_per_file=1 if fullpage else 5,
)
return ExitCode.SUCCESS
if not reviewer.completed:
return ExitCode.INTERRUPTED
return ExitCode.PARTIAL if reviewer.had_errors else ExitCode.SUCCESS
def build_parser() -> argparse.ArgumentParser:
parser = target_parser("Interactively crop the label margin from PDF copies")
parser.add_argument("--marked", action="store_true", help="Review flagged copies and clear each flag after saving with Enter")
parser.add_argument(
"--fullpage",
action="store_true",
@@ -334,11 +377,10 @@ def main(argv: Sequence[str] | None = None) -> int:
def handle(args: argparse.Namespace) -> ExitCode:
workspace, target = workspace_from_target(args)
return run(workspace, target, fullpage=args.fullpage)
return run(workspace, target, fullpage=args.fullpage, marked=args.marked)
return execute(parser, argv, handle)
if __name__ == "__main__":
raise SystemExit(main())
copienator/commands/gemini_for_enonce.py
+5
View File
@@ -190,6 +190,7 @@ def generate_rubrics(client, group_context_text: str) -> dict[str, str]:
types.Part.from_text(text=f"--- CONTENU DU GROUPE ---\n{group_context_text}"),
])],
config=types.GenerateContentConfig(
automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
system_instruction="Rédige tous les barèmes et consignes de notation en français. Conserve les clés JSON, les labels et les formules mathématiques.",
temperature=0.2,
response_mime_type="application/json",
@@ -243,6 +244,7 @@ def refine_existing(workspace: EvaluationWorkspace, mode: str, *, api_client=Non
+ context + "\n\n" + "\n\n".join(questions.values())
))])],
config=types.GenerateContentConfig(
automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
temperature=0.1, response_mime_type="application/json",
response_json_schema=LabelGroups.model_json_schema(),
),
@@ -337,6 +339,7 @@ def process_exam(
]
config_1 = types.GenerateContentConfig(
automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
temperature=0.1,
response_mime_type="application/json",
response_json_schema=ExamQuestions.model_json_schema(),
@@ -375,6 +378,7 @@ def process_exam(
]
config_2 = types.GenerateContentConfig(
automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
temperature=0.1,
response_mime_type="application/json",
response_json_schema=ExamSolutions.model_json_schema(),
@@ -411,6 +415,7 @@ def process_exam(
]
config_3 = types.GenerateContentConfig(
automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
temperature=0.1,
response_mime_type="application/json",
response_json_schema=ExamContext.model_json_schema(),
copienator/commands/gemini_for_labels.py
+125 -7
View File
@@ -76,6 +76,8 @@ be missing.
##wrong_labels##
##wrong_label_text_context##
Here's a list of the names of the students, pick the one that matches
the best or `\"Unknown\"` if you cannot read the name
@@ -133,6 +135,8 @@ be missing.
##wrong_labels##
##wrong_label_text_context##
Since this copy isn't the first part of a sequence, simply set the
name to `\"Continued\"`."""
@@ -147,7 +151,66 @@ class AnnotationData(BaseModel):
)
def generate_request(file, labels, names, context_labels, wrong_labels):
TEXT_CONTEXT_MAX_CHARS = 4000
def _label_filename(path: Path) -> str:
return path.stem if path.suffix.casefold() in {".tex", ".txt"} else path.name
def _common_prefix_length(left: str, right: str) -> int:
left_folded = left.casefold()
right_folded = right.casefold()
limit = min(len(left_folded), len(right_folded))
for index in range(limit):
if left_folded[index] != right_folded[index]:
return index
return limit
def closest_text_context(
workspace: EvaluationWorkspace, wrong_labels: list[str]
) -> tuple[Path | None, str]:
"""Return a bounded excerpt from the Text file closest to an invalid label."""
text_dir = workspace.root / "Text"
if not wrong_labels or not text_dir.is_dir():
return None, ""
ranked: list[tuple[int, str, Path]] = []
for path in text_dir.iterdir():
if not path.is_file() or path.suffix.casefold() == ".pdf":
continue
filename = _label_filename(path)
prefix_length = max(
_common_prefix_length(filename, wrong_label)
for wrong_label in wrong_labels
)
if prefix_length:
ranked.append((prefix_length, filename.casefold(), path))
for _prefix_length, _filename, path in sorted(
ranked, key=lambda item: (-item[0], item[1])
):
try:
content = path.read_text(encoding="utf-8")
except (OSError, UnicodeError):
continue
if len(content) > TEXT_CONTEXT_MAX_CHARS:
content = content[:TEXT_CONTEXT_MAX_CHARS] + "\n[excerpt truncated]"
return path, content
return None, ""
def generate_request(
file,
labels,
names,
context_labels,
wrong_labels,
wrong_label_text_context="",
wrong_label_text_file: Path | None = None,
seed: int = 0,
):
"""Generates request for Gemini with context."""
image_path = Path(file)
@@ -162,9 +225,36 @@ def generate_request(file, labels, names, context_labels, wrong_labels):
text = my_prompt2.replace("##labels##", labels)\
.replace("##prev_context##", context_str)
if wrong_labels:
text= text.replace("##wrong_labels##\n\n", f"On a previous request, you answered with the following wrong labels : {wrong_labels}. These are wrong, since they do not exactly match any of the labels in the previous list.")
formatted_wrong_labels = "\n".join(f'- "{label}"' for label in wrong_labels)
text = text.replace(
"##wrong_labels##",
"On the previous request for this image, you answered with these "
"invalid labels:\n"
f"{formatted_wrong_labels}\n"
"They are wrong because they do not exactly match any label in the "
"valid list above.\n\n"
"CRITICAL RETRY CONSTRAINT: NEVER return any of the invalid labels "
"listed above again. Your answer must use only exact labels copied "
"verbatim from the valid list. If the handwriting resembles an "
"invalid label, choose the closest exact valid label instead.",
)
else:
text = text.replace("##wrong_labels##\n\n", "")
text = text.replace("##wrong_labels##", "")
if wrong_label_text_context and wrong_label_text_file:
text = text.replace(
"##wrong_label_text_context##",
"Here is an excerpt from the exam text file whose name has the "
"longest prefix in common with the invalid label(s), "
f"`{wrong_label_text_file.name}`:\n\n"
"<exam_text_excerpt>\n"
f"{wrong_label_text_context}\n"
"</exam_text_excerpt>\n\n"
"Use this excerpt as extra context for identifying the handwritten "
"label, but return only an exact label from the valid list above.",
)
else:
text = text.replace("##wrong_label_text_context##", "")
contents = [
@@ -181,9 +271,10 @@ def generate_request(file, labels, names, context_labels, wrong_labels):
]
generate_content_config = types.GenerateContentConfig(
automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
temperature=1.0,
top_p=0.95,
seed=0,
seed=seed,
max_output_tokens=65535,
response_mime_type= "application/json",
response_json_schema= AnnotationData.model_json_schema(),
@@ -293,17 +384,30 @@ def process_copy_group(
f"{len(accumulated_labels)} accumulated labels..."
)
attempt = 0
label_retry_count = 0
wrong_labels: list[str] = []
while True:
if attempt > 0:
sleep(10 * attempt)
try:
text_context_file, text_context = closest_text_context(
workspace, wrong_labels
)
if text_context_file:
print(
f"[{group_key}] Retry context for {image_file.name}: "
f"{text_context_file.relative_to(workspace.root)}"
)
request_seed = max(0, label_retry_count - 1)
contents, request_config = generate_request(
image_file,
labels_text,
names_text,
accumulated_labels,
wrong_labels,
text_context,
text_context_file,
seed=request_seed,
)
response = client.models.generate_content(
model=MODEL_ID,
@@ -321,9 +425,23 @@ def process_copy_group(
f"Error: {image_file.name} contained unknown labels: "
f"{unknown}"
)
wrong_labels.extend(unknown)
attempt += 1
continue
unique_unknown = list(dict.fromkeys(unknown))
if (
label_retry_count >= 2
and set(unique_unknown) == set(wrong_labels)
):
for item in annotation.list:
if item.label in unique_unknown:
item.label = f"??{item.label}"
print(
f"Warning: {image_file.name} repeated the same unknown "
"label(s) on the third try; keeping them with a ?? prefix."
)
else:
wrong_labels = unique_unknown
label_retry_count += 1
attempt += 1
continue
if annotation.name not in valid_names:
print(
f"Error: {image_file.name} returned unknown name: "
copienator/commands/page_splitter.py
+7 -4
View File
@@ -26,6 +26,7 @@ from copienator import (
workspace_from_target,
)
from copienator.platform import launch_pdf_arranger
from copienator.copy_errors import marked_copy_paths
# Keep the new shortcut available with older personal configuration files.
PAGE_SPLITTER_KB = {"reverse_pages": "i", **PAGE_SPLITTER_KB}
@@ -647,8 +648,8 @@ def _selected_inputs(
return list(reversed(candidates))
def run(workspace: EvaluationWorkspace, target: Path) -> ExitCode:
inputs = _selected_inputs(workspace, target)
def run(workspace: EvaluationWorkspace, target: Path, *, marked: bool = False) -> ExitCode:
inputs = list(reversed(marked_copy_paths(workspace, originals=True))) if marked else _selected_inputs(workspace, target)
if not inputs:
print(f"No PDF files found in {target}")
return ExitCode.SUCCESS
@@ -659,7 +660,9 @@ def run(workspace: EvaluationWorkspace, target: Path) -> ExitCode:
def build_parser() -> argparse.ArgumentParser:
return target_parser("Interactively split and reorder scanned PDF pages")
parser = target_parser("Interactively split and reorder scanned PDF pages")
parser.add_argument("--marked", action="store_true", help="Process only copies flagged during margin review")
return parser
def main(argv: Sequence[str] | None = None) -> int:
@@ -667,7 +670,7 @@ def main(argv: Sequence[str] | None = None) -> int:
def handle(args: argparse.Namespace) -> ExitCode:
workspace, target = workspace_from_target(args)
return run(workspace, target)
return run(workspace, target, marked=args.marked)
return execute(parser, argv, handle)