From b6c2a634cac92ee447b79fb3f9ecce80a23dfdc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Miquel?= Date: Mon, 12 Oct 2026 11:24:13 +0200 Subject: [PATCH] Use different seeds for retries --- copienator/commands/correction.py | 12 +++++++- copienator/commands/gemini_for_labels.py | 12 ++++---- copienator/gemini_config.py | 10 +++++++ copienator/prompting.py | 4 +-- tests/test_gemini_config.py | 37 +++++++++++++++++++++++- tests/test_gui_core.py | 4 ++- tests/test_label_timeouts.py | 22 ++++++++++++++ 7 files changed, 91 insertions(+), 10 deletions(-) diff --git a/copienator/commands/correction.py b/copienator/commands/correction.py index cc0cad4..d8ebdaf 100644 --- a/copienator/commands/correction.py +++ b/copienator/commands/correction.py @@ -15,12 +15,13 @@ from collections.abc import Sequence from pathlib import Path from google import genai +from google.genai import types from copienator import configuration as config from copienator.commands import grouping from copienator.answer_origins import save_origins from copienator import prompting -from copienator.gemini_config import thinking_config +from copienator.gemini_config import fresh_seed, thinking_config from copienator import configuration as settings from copienator.feedback_boxes import valid_feedback_box from copienator.latex_json import repair_latex_json @@ -274,10 +275,19 @@ def call_gemini_with_retries(model_id, contents, config, """Handles requests to Gemini with a 1min and 5min retry mechanism, and quota fallback.""" global pro_quota_exhausted delays = [60, 300] + request_seed = None + # Own the retries here so the SDK cannot silently resend a fixed seed. + http_options = config.http_options or types.HttpOptions() + config = config.model_copy(update={"http_options": http_options.model_copy( + update={"retry_options": types.HttpRetryOptions(attempts=1)} + )}) for attempt in range(3): if stop_requested.is_set(): raise CorrectionStopRequested + request_seed = fresh_seed(request_seed) + config = config.model_copy(update={"seed": request_seed}) + tprint(f"\tGemini attempt {attempt + 1}/3: seed={request_seed}") # Switch to fallback immediately if quota was exhausted by another thread if model_id == MODEL_ID_pro and pro_quota_exhausted and fallback_model_id: model_id = fallback_model_id diff --git a/copienator/commands/gemini_for_labels.py b/copienator/commands/gemini_for_labels.py index 40b3ef2..9801294 100644 --- a/copienator/commands/gemini_for_labels.py +++ b/copienator/commands/gemini_for_labels.py @@ -15,7 +15,7 @@ from google.genai import types from pydantic import BaseModel, Field from copienator import configuration as config -from copienator.gemini_config import thinking_config +from copienator.gemini_config import fresh_seed, thinking_config from copienator.copy_errors import marked_copy_paths from copienator import ( CliError, @@ -213,7 +213,7 @@ def generate_request( wrong_labels, wrong_label_text_context="", wrong_label_text_file: Path | None = None, - seed: int = 0, + seed: int | None = None, *, retry: bool = False, ): @@ -283,7 +283,7 @@ def generate_request( ), automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True), thinking_config=thinking_config("low" if retry else config.THINKING_LEVEL_FOR_LABEL), - seed=seed, + seed=fresh_seed() if seed is None else seed, max_output_tokens=65535, response_mime_type= "application/json", response_json_schema= AnnotationData.model_json_schema(), @@ -432,6 +432,7 @@ def process_copy_group( attempt = 0 label_retry_count = 0 wrong_labels: list[str] = [] + request_seed = None while attempt < MAX_ATTEMPTS: if progress: progress(group_key, f"{image_file.name}, attempt {attempt + 1}/{MAX_ATTEMPTS}", started) @@ -439,9 +440,11 @@ def process_copy_group( print(f"[{group_key}] Retrying {image_file.name} in {10 * attempt}s...") sleep(10 * attempt) level = "low" if attempt > 0 else config.THINKING_LEVEL_FOR_LABEL + request_seed = fresh_seed(request_seed) print( f"[{group_key}] {image_file.name}: attempt {attempt + 1}/{MAX_ATTEMPTS} " - f"({REQUEST_TIMEOUT_SECONDS}s timeout, thinking={level or 'default'})." + f"({REQUEST_TIMEOUT_SECONDS}s timeout, thinking={level or 'default'}, " + f"seed={request_seed})." ) try: text_context_file, text_context = closest_text_context( @@ -452,7 +455,6 @@ def process_copy_group( f"[{group_key}] Retry context for {image_file.name}: " f"{text_context_file.relative_to(workspace.root)}" ) - request_seed = max(0, label_retry_count - 1) contents, request_config = generate_request( image_file, labels_text, diff --git a/copienator/gemini_config.py b/copienator/gemini_config.py index ced7e09..7b873e3 100644 --- a/copienator/gemini_config.py +++ b/copienator/gemini_config.py @@ -1,6 +1,16 @@ +import secrets + from google.genai import types +def fresh_seed(previous: int | None = None) -> int: + """Choose a random int32 seed, excluding the preceding attempt's seed.""" + limit = 2**31 + if previous is None: + return secrets.randbelow(limit) + return (previous + 1 + secrets.randbelow(limit - 1)) % limit + + def thinking_config(level: str | None) -> types.ThinkingConfig | None: """Use a supported reasoning level, or omit it for the model default.""" if level is None: diff --git a/copienator/prompting.py b/copienator/prompting.py index 07f1732..19f963d 100644 --- a/copienator/prompting.py +++ b/copienator/prompting.py @@ -4,7 +4,7 @@ import json from . import utils from . import read_json from . import configuration as settings -from .gemini_config import thinking_config +from .gemini_config import fresh_seed, thinking_config PERSPECTIVE_GUIDANCE = ( "Ce barème est indicatif, si une réponse est entièrement correcte mais utilise " @@ -223,7 +223,7 @@ def generate_request(input_dir, file, full_label, *, thinking_level=settings.THI generate_content_config = types.GenerateContentConfig( automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True), thinking_config=thinking_config(thinking_level), - seed=0, + seed=fresh_seed(), max_output_tokens=65535, response_mime_type= "application/json", response_json_schema= TypeAdapter(List[EvaluationEntry]).json_schema() diff --git a/tests/test_gemini_config.py b/tests/test_gemini_config.py index f5259e0..5319260 100644 --- a/tests/test_gemini_config.py +++ b/tests/test_gemini_config.py @@ -8,10 +8,40 @@ from google.genai import types from copienator import configuration, prompting from copienator.commands import correction, gemini_for_enonce, gemini_for_labels -from copienator.gemini_config import thinking_config +from copienator.gemini_config import fresh_seed, thinking_config class GeminiConfigTests(unittest.TestCase): + def test_fresh_seed_excludes_previous_even_at_int32_boundary(self): + for previous in (0, 1234, 2**31 - 1): + for random_value in (0, 2**31 - 2): + with patch("copienator.gemini_config.secrets.randbelow", return_value=random_value): + seed = fresh_seed(previous) + self.assertNotEqual(seed, previous) + self.assertTrue(0 <= seed < 2**31) + + def test_correction_retries_change_seed_without_mutating_config(self): + client = Mock() + client.models.generate_content_stream.side_effect = [ + TimeoutError(), RuntimeError("429 minute limit retry in 1s"), + [SimpleNamespace(text="result")], + ] + config = types.GenerateContentConfig(seed=0, thinking_config=thinking_config("medium")) + with patch.object(correction, "client", client), \ + patch.object(correction, "fresh_seed", side_effect=[10, 20, 30]), \ + patch.object(correction, "tprint"), \ + patch.object(correction, "stop_requested") as stop: + stop.is_set.return_value = False + stop.wait.return_value = False + self.assertEqual(correction.call_gemini_with_retries("model", [], config), "result") + configs = [call.kwargs["config"] for call in client.models.generate_content_stream.call_args_list] + self.assertEqual([cfg.seed for cfg in configs], [10, 20, 30]) + self.assertEqual(config.seed, 0) + self.assertIsNone(config.http_options) + for cfg in configs: + self.assert_request_config(cfg, "medium") + self.assertEqual(cfg.http_options.retry_options.attempts, 1) + def assert_request_config(self, config, level): data = config.model_dump(mode="json", by_alias=True, exclude_none=True) for key in ("temperature", "topP", "topK", "thinkingBudget"): @@ -34,6 +64,8 @@ class GeminiConfigTests(unittest.TestCase): batch = prompting.batch_generation_config(level) self.assert_request_config(types.GenerateContentConfig.model_validate(batch), level) self.assertEqual(batch["responseSchema"], prompting.UNROLLED_SCHEMA) + # Let Gemini randomize each execution, including resubmission. + self.assertNotIn("seed", batch) def test_labels_and_statement_use_their_own_settings(self): with tempfile.TemporaryDirectory() as directory: @@ -46,6 +78,7 @@ class GeminiConfigTests(unittest.TestCase): with patch.object(configuration, "THINKING_LEVEL_LITE", "high"): gemini_for_enonce._request_rubrics(client, "Question") self.assert_request_config(client.models.generate_content.call_args.kwargs["config"], "high") + self.assertIsNone(client.models.generate_content.call_args.kwargs["config"].seed) def test_correction_helpers_use_flash_level(self): with patch.object(configuration, "THINKING_LEVEL_FLASH", "low"), patch.object( @@ -77,6 +110,8 @@ class GeminiConfigTests(unittest.TestCase): self.assert_request_config(calls[0].kwargs["config"], "high") self.assert_request_config(calls[1].kwargs["config"], "low") self.assertEqual(calls[1].kwargs["model"], "flash-model") + self.assertNotEqual(calls[0].kwargs["config"].seed, calls[1].kwargs["config"].seed) + self.assertIsNone(config.seed) self.assert_request_config(config, "high") def test_invalid_level_fails_before_submission(self): diff --git a/tests/test_gui_core.py b/tests/test_gui_core.py index 0558ee1..3841181 100644 --- a/tests/test_gui_core.py +++ b/tests/test_gui_core.py @@ -1380,7 +1380,9 @@ class StandardCliTests(unittest.TestCase): call.kwargs["config"].seed for call in client.models.generate_content.call_args_list ] - self.assertEqual(seeds, [0, 0, 1]) + self.assertTrue(all(isinstance(seed, int) for seed in seeds)) + self.assertNotEqual(seeds[0], seeds[1]) + self.assertNotEqual(seeds[1], seeds[2]) result = read_json(evaluation / "Copies" / "Copie42_03.json") self.assertEqual(result["list"][0]["label"], "??Ex 10 : 1)a)") self.assertIn("keeping them with a ?? prefix", output.getvalue()) diff --git a/tests/test_label_timeouts.py b/tests/test_label_timeouts.py index f688489..a5204f3 100644 --- a/tests/test_label_timeouts.py +++ b/tests/test_label_timeouts.py @@ -9,6 +9,7 @@ from unittest.mock import Mock, patch from copienator import CliError, EvaluationWorkspace, ExitCode from copienator.commands import gemini_for_labels as labels +from copienator.copy_errors import mark_copy_error class LabelTimeoutTests(unittest.TestCase): @@ -73,6 +74,25 @@ class LabelTimeoutTests(unittest.TestCase): self.assertEqual(options.retry_options.attempts, 1) self.assertTrue((self.workspace.copies_dir / "Copie01_01.json").exists()) + def test_flagged_reruns_get_fresh_seeds_and_replace_existing_results(self): + pdf = self.workspace.copies_dir / "Copie01.pdf" + pdf.touch() + mark_copy_error(self.workspace, pdf, "Review", source="plotting") + self.client.models.generate_content.return_value = self.response + with patch.object(labels, "selected_images", return_value=(self.images[:1], [])), \ + patch.object(labels, "fresh_seed", side_effect=[101, 202]), \ + patch.object(labels.config, "THINKING_LEVEL_FOR_LABEL", "medium"): + for _ in range(2): + self.assertEqual(labels.run( + self.workspace, [], marked=True, client=self.client, + sleep=lambda _: None, max_workers=1, + ), ExitCode.SUCCESS) + configs = [call.kwargs["config"] + for call in self.client.models.generate_content.call_args_list] + self.assertEqual([cfg.seed for cfg in configs], [101, 202]) + self.assertEqual([cfg.thinking_config.thinking_level.value for cfg in configs], + ["MEDIUM", "MEDIUM"]) + def test_retries_use_low_thinking_and_next_image_restores_configured_level(self): self.client.models.generate_content.side_effect = [ TimeoutError(), TimeoutError(), self.response, self.response, @@ -87,6 +107,8 @@ class LabelTimeoutTests(unittest.TestCase): ["MEDIUM", "LOW", "LOW", "MEDIUM"], ) self.assertTrue(all(cfg.http_options.timeout == 80_000 for cfg in configs)) + self.assertNotEqual(configs[0].seed, configs[1].seed) + self.assertNotEqual(configs[1].seed, configs[2].seed) def test_exhaustion_defers_dependent_pages_but_other_copies_finish_and_resume(self): self.client.models.generate_content.side_effect = [