Use different seeds for retries
This commit is contained in:
1 parent
c58bbc3672
commit
b6c2a634ca
7 files changed
+91
-10
No files matched your search
@@ -8,10 +8,40 @@ from google.genai import types
|
||||
|
||||
from copienator import configuration, prompting
|
||||
from copienator.commands import correction, gemini_for_enonce, gemini_for_labels
|
||||
from copienator.gemini_config import thinking_config
|
||||
from copienator.gemini_config import fresh_seed, thinking_config
|
||||
|
||||
|
||||
class GeminiConfigTests(unittest.TestCase):
|
||||
def test_fresh_seed_excludes_previous_even_at_int32_boundary(self):
|
||||
for previous in (0, 1234, 2**31 - 1):
|
||||
for random_value in (0, 2**31 - 2):
|
||||
with patch("copienator.gemini_config.secrets.randbelow", return_value=random_value):
|
||||
seed = fresh_seed(previous)
|
||||
self.assertNotEqual(seed, previous)
|
||||
self.assertTrue(0 <= seed < 2**31)
|
||||
|
||||
def test_correction_retries_change_seed_without_mutating_config(self):
|
||||
client = Mock()
|
||||
client.models.generate_content_stream.side_effect = [
|
||||
TimeoutError(), RuntimeError("429 minute limit retry in 1s"),
|
||||
[SimpleNamespace(text="result")],
|
||||
]
|
||||
config = types.GenerateContentConfig(seed=0, thinking_config=thinking_config("medium"))
|
||||
with patch.object(correction, "client", client), \
|
||||
patch.object(correction, "fresh_seed", side_effect=[10, 20, 30]), \
|
||||
patch.object(correction, "tprint"), \
|
||||
patch.object(correction, "stop_requested") as stop:
|
||||
stop.is_set.return_value = False
|
||||
stop.wait.return_value = False
|
||||
self.assertEqual(correction.call_gemini_with_retries("model", [], config), "result")
|
||||
configs = [call.kwargs["config"] for call in client.models.generate_content_stream.call_args_list]
|
||||
self.assertEqual([cfg.seed for cfg in configs], [10, 20, 30])
|
||||
self.assertEqual(config.seed, 0)
|
||||
self.assertIsNone(config.http_options)
|
||||
for cfg in configs:
|
||||
self.assert_request_config(cfg, "medium")
|
||||
self.assertEqual(cfg.http_options.retry_options.attempts, 1)
|
||||
|
||||
def assert_request_config(self, config, level):
|
||||
data = config.model_dump(mode="json", by_alias=True, exclude_none=True)
|
||||
for key in ("temperature", "topP", "topK", "thinkingBudget"):
|
||||
@@ -34,6 +64,8 @@ class GeminiConfigTests(unittest.TestCase):
|
||||
batch = prompting.batch_generation_config(level)
|
||||
self.assert_request_config(types.GenerateContentConfig.model_validate(batch), level)
|
||||
self.assertEqual(batch["responseSchema"], prompting.UNROLLED_SCHEMA)
|
||||
# Let Gemini randomize each execution, including resubmission.
|
||||
self.assertNotIn("seed", batch)
|
||||
|
||||
def test_labels_and_statement_use_their_own_settings(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
@@ -46,6 +78,7 @@ class GeminiConfigTests(unittest.TestCase):
|
||||
with patch.object(configuration, "THINKING_LEVEL_LITE", "high"):
|
||||
gemini_for_enonce._request_rubrics(client, "Question")
|
||||
self.assert_request_config(client.models.generate_content.call_args.kwargs["config"], "high")
|
||||
self.assertIsNone(client.models.generate_content.call_args.kwargs["config"].seed)
|
||||
|
||||
def test_correction_helpers_use_flash_level(self):
|
||||
with patch.object(configuration, "THINKING_LEVEL_FLASH", "low"), patch.object(
|
||||
@@ -77,6 +110,8 @@ class GeminiConfigTests(unittest.TestCase):
|
||||
self.assert_request_config(calls[0].kwargs["config"], "high")
|
||||
self.assert_request_config(calls[1].kwargs["config"], "low")
|
||||
self.assertEqual(calls[1].kwargs["model"], "flash-model")
|
||||
self.assertNotEqual(calls[0].kwargs["config"].seed, calls[1].kwargs["config"].seed)
|
||||
self.assertIsNone(config.seed)
|
||||
self.assert_request_config(config, "high")
|
||||
|
||||
def test_invalid_level_fails_before_submission(self):
|
||||
|
||||
Reference in new issue
Block a user