Use different seeds for retries

This commit is contained in:
sebastien committed 2026-10-12 11:24:13 +02:00
1 parent c58bbc3672
commit b6c2a634ca
7 files changed
+91 -10

No files matched your search

+36 -1
View File
@@ -8,10 +8,40 @@ from google.genai import types
from copienator import configuration, prompting
from copienator.commands import correction, gemini_for_enonce, gemini_for_labels
from copienator.gemini_config import thinking_config
from copienator.gemini_config import fresh_seed, thinking_config
class GeminiConfigTests(unittest.TestCase):
def test_fresh_seed_excludes_previous_even_at_int32_boundary(self):
for previous in (0, 1234, 2**31 - 1):
for random_value in (0, 2**31 - 2):
with patch("copienator.gemini_config.secrets.randbelow", return_value=random_value):
seed = fresh_seed(previous)
self.assertNotEqual(seed, previous)
self.assertTrue(0 <= seed < 2**31)
def test_correction_retries_change_seed_without_mutating_config(self):
client = Mock()
client.models.generate_content_stream.side_effect = [
TimeoutError(), RuntimeError("429 minute limit retry in 1s"),
[SimpleNamespace(text="result")],
]
config = types.GenerateContentConfig(seed=0, thinking_config=thinking_config("medium"))
with patch.object(correction, "client", client), \
patch.object(correction, "fresh_seed", side_effect=[10, 20, 30]), \
patch.object(correction, "tprint"), \
patch.object(correction, "stop_requested") as stop:
stop.is_set.return_value = False
stop.wait.return_value = False
self.assertEqual(correction.call_gemini_with_retries("model", [], config), "result")
configs = [call.kwargs["config"] for call in client.models.generate_content_stream.call_args_list]
self.assertEqual([cfg.seed for cfg in configs], [10, 20, 30])
self.assertEqual(config.seed, 0)
self.assertIsNone(config.http_options)
for cfg in configs:
self.assert_request_config(cfg, "medium")
self.assertEqual(cfg.http_options.retry_options.attempts, 1)
def assert_request_config(self, config, level):
data = config.model_dump(mode="json", by_alias=True, exclude_none=True)
for key in ("temperature", "topP", "topK", "thinkingBudget"):
@@ -34,6 +64,8 @@ class GeminiConfigTests(unittest.TestCase):
batch = prompting.batch_generation_config(level)
self.assert_request_config(types.GenerateContentConfig.model_validate(batch), level)
self.assertEqual(batch["responseSchema"], prompting.UNROLLED_SCHEMA)
# Let Gemini randomize each execution, including resubmission.
self.assertNotIn("seed", batch)
def test_labels_and_statement_use_their_own_settings(self):
with tempfile.TemporaryDirectory() as directory:
@@ -46,6 +78,7 @@ class GeminiConfigTests(unittest.TestCase):
with patch.object(configuration, "THINKING_LEVEL_LITE", "high"):
gemini_for_enonce._request_rubrics(client, "Question")
self.assert_request_config(client.models.generate_content.call_args.kwargs["config"], "high")
self.assertIsNone(client.models.generate_content.call_args.kwargs["config"].seed)
def test_correction_helpers_use_flash_level(self):
with patch.object(configuration, "THINKING_LEVEL_FLASH", "low"), patch.object(
@@ -77,6 +110,8 @@ class GeminiConfigTests(unittest.TestCase):
self.assert_request_config(calls[0].kwargs["config"], "high")
self.assert_request_config(calls[1].kwargs["config"], "low")
self.assertEqual(calls[1].kwargs["model"], "flash-model")
self.assertNotEqual(calls[0].kwargs["config"].seed, calls[1].kwargs["config"].seed)
self.assertIsNone(config.seed)
self.assert_request_config(config, "high")
def test_invalid_level_fails_before_submission(self):
+3 -1
View File
@@ -1380,7 +1380,9 @@ class StandardCliTests(unittest.TestCase):
call.kwargs["config"].seed
for call in client.models.generate_content.call_args_list
]
self.assertEqual(seeds, [0, 0, 1])
self.assertTrue(all(isinstance(seed, int) for seed in seeds))
self.assertNotEqual(seeds[0], seeds[1])
self.assertNotEqual(seeds[1], seeds[2])
result = read_json(evaluation / "Copies" / "Copie42_03.json")
self.assertEqual(result["list"][0]["label"], "??Ex 10 : 1)a)")
self.assertIn("keeping them with a ?? prefix", output.getvalue())
+22
View File
@@ -9,6 +9,7 @@ from unittest.mock import Mock, patch
from copienator import CliError, EvaluationWorkspace, ExitCode
from copienator.commands import gemini_for_labels as labels
from copienator.copy_errors import mark_copy_error
class LabelTimeoutTests(unittest.TestCase):
@@ -73,6 +74,25 @@ class LabelTimeoutTests(unittest.TestCase):
self.assertEqual(options.retry_options.attempts, 1)
self.assertTrue((self.workspace.copies_dir / "Copie01_01.json").exists())
def test_flagged_reruns_get_fresh_seeds_and_replace_existing_results(self):
pdf = self.workspace.copies_dir / "Copie01.pdf"
pdf.touch()
mark_copy_error(self.workspace, pdf, "Review", source="plotting")
self.client.models.generate_content.return_value = self.response
with patch.object(labels, "selected_images", return_value=(self.images[:1], [])), \
patch.object(labels, "fresh_seed", side_effect=[101, 202]), \
patch.object(labels.config, "THINKING_LEVEL_FOR_LABEL", "medium"):
for _ in range(2):
self.assertEqual(labels.run(
self.workspace, [], marked=True, client=self.client,
sleep=lambda _: None, max_workers=1,
), ExitCode.SUCCESS)
configs = [call.kwargs["config"]
for call in self.client.models.generate_content.call_args_list]
self.assertEqual([cfg.seed for cfg in configs], [101, 202])
self.assertEqual([cfg.thinking_config.thinking_level.value for cfg in configs],
["MEDIUM", "MEDIUM"])
def test_retries_use_low_thinking_and_next_image_restores_configured_level(self):
self.client.models.generate_content.side_effect = [
TimeoutError(), TimeoutError(), self.response, self.response,
@@ -87,6 +107,8 @@ class LabelTimeoutTests(unittest.TestCase):
["MEDIUM", "LOW", "LOW", "MEDIUM"],
)
self.assertTrue(all(cfg.http_options.timeout == 80_000 for cfg in configs))
self.assertNotEqual(configs[0].seed, configs[1].seed)
self.assertNotEqual(configs[1].seed, configs[2].seed)
def test_exhaustion_defers_dependent_pages_but_other_copies_finish_and_resume(self):
self.client.models.generate_content.side_effect = [