121 lines
6.1 KiB
Python
121 lines
6.1 KiB
Python
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import Mock, patch
|
|
|
|
from copienator import EvaluationWorkspace, atomic_write_json, read_json
|
|
from copienator.commands import correction
|
|
|
|
|
|
class ConfirmedLabelRegradingTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.addCleanup(correction.stop_requested.clear)
|
|
self.root = Path(temporary.name)
|
|
self.workspace = EvaluationWorkspace(self.root)
|
|
(self.root / 'labels').write_text('Ex 1\nEx 2\n')
|
|
for mode in ('Text', 'Sol'):
|
|
(self.root / mode).mkdir()
|
|
(self.root / mode / 'Ex 1').write_text(f'{mode} material')
|
|
self.group = self.root / 'Par label/Ex 1/Group_1.jpg'
|
|
self.group.parent.mkdir(parents=True)
|
|
self.group.touch()
|
|
atomic_write_json(self.group.with_suffix('.json'), [
|
|
['02', 0, 400, 1], ['01', 450, 950, .5],
|
|
])
|
|
self.pdf = self.root / 'Copies/Copie01/Ex 1.pdf'
|
|
self.pdf.parent.mkdir(parents=True)
|
|
self.pdf.write_bytes(b'original answer')
|
|
args = correction.build_parser().parse_args([str(self.root)])
|
|
correction.configure_runtime(self.workspace, [(str(self.group), 'Ex 1')], args, api_client=Mock())
|
|
self.original = [
|
|
{'id': '02', 'result': {'score': 4, 'feedback': [], 'error': ''}},
|
|
{'id': '01', 'result': {'score': 0, 'error': 'wrong-label',
|
|
'feedback': [{'text': 'Wrong question', 'box_2d': None}]}},
|
|
]
|
|
self.regraded = [{'id': '01', 'result': {
|
|
'score': 2.5, 'error': '', 'feedback': [
|
|
{'text': 'Monotonicity is missing', 'box_2d': [100, 200, 300, 400]},
|
|
],
|
|
}}]
|
|
|
|
def process(self, responses):
|
|
with (
|
|
patch.object(correction.prompting, 'generate_request', return_value=([], {})),
|
|
patch.object(correction.prompting, 'get_single_image_bytes', return_value=b'image'),
|
|
patch.object(correction, 'call_gemini_with_retries', side_effect=responses) as call,
|
|
):
|
|
tasks = correction.process_single_task((str(self.group), 'Ex 1'), json.dumps(self.original))
|
|
return tasks, call
|
|
|
|
def test_confirmed_label_replaces_rejection_and_maps_boxes_to_original_group(self):
|
|
tasks, calls = self.process(['Ex 1', json.dumps(self.regraded)])
|
|
self.assertEqual(tasks, [])
|
|
self.assertEqual(calls.call_count, 2)
|
|
self.assertEqual(correction.errors_summary, [])
|
|
saved = read_json(self.workspace.correction_file)['Ex 1'][0]
|
|
self.assertEqual(saved[0], self.original[0])
|
|
result = saved[1]['result']
|
|
self.assertEqual(result['score'], 2.5)
|
|
self.assertEqual(result['error'], '')
|
|
self.assertEqual(result['feedback'][0]['text'], 'Monotonicity is missing')
|
|
box = result['feedback'][0]['box_2d']
|
|
self.assertAlmostEqual(box[0], 500 * 1000 / 950)
|
|
self.assertAlmostEqual(box[2], 600 * 1000 / 950)
|
|
self.assertEqual([box[1], box[3]], [100, 200])
|
|
self.assertEqual(self.pdf.read_bytes(), b'original answer')
|
|
self.assertEqual(correction.completed_tasks, [(str(self.group), 'Ex 1')])
|
|
|
|
def test_invalid_regrading_never_publishes_stale_zero_or_marks_group_complete(self):
|
|
for response in (
|
|
[],
|
|
[{'id': '02', 'result': self.regraded[0]['result']}],
|
|
[self.regraded[0], self.regraded[0]],
|
|
[self.original[1]],
|
|
[{'id': '01', 'result': {'error': ''}}],
|
|
):
|
|
with self.subTest(response=response):
|
|
self.process(['Ex 1', json.dumps(response)])
|
|
self.assertFalse(self.workspace.correction_file.exists())
|
|
self.assertEqual(correction.completed_tasks, [])
|
|
self.assertEqual(correction.results['Ex 1'], [])
|
|
self.assertEqual(self.pdf.read_bytes(), b'original answer')
|
|
|
|
def test_interrupted_regrading_keeps_wrong_label_in_pending_response(self):
|
|
with self.assertRaises(correction.CorrectionStopRequested):
|
|
self.process(['Ex 1', correction.CorrectionStopRequested()])
|
|
pending = read_json(self.workspace.correction_pending_responses_file)
|
|
self.assertEqual(json.loads(pending[str(self.group)]), self.original)
|
|
self.assertFalse(self.workspace.correction_file.exists())
|
|
self.process(['Ex 1', json.dumps(self.regraded)])
|
|
self.assertEqual(read_json(self.workspace.correction_pending_responses_file), {})
|
|
self.assertEqual(read_json(self.workspace.correction_file)['Ex 1'][0][1]['result']['score'], 2.5)
|
|
|
|
def test_empty_answer_from_regrading_uses_normal_empty_answer_handling(self):
|
|
response = [{'id': '01', 'result': {'score': 0, 'feedback': [], 'error': 'empty-answer'}}]
|
|
self.process(['Ex 1', json.dumps(response)])
|
|
result = read_json(self.workspace.correction_file)['Ex 1'][0][1]['result']
|
|
self.assertEqual(result['error'], 'empty-answer')
|
|
self.assertEqual(result['suffix'], '_old')
|
|
self.assertFalse(self.pdf.exists())
|
|
self.assertEqual(self.pdf.with_name('Ex 1_old.pdf').read_bytes(), b'original answer')
|
|
|
|
def test_different_label_keeps_existing_move_behavior(self):
|
|
(self.pdf.parent / 'Ex 2.pdf').touch()
|
|
_, calls = self.process(['Ex 2'])
|
|
self.assertEqual(calls.call_count, 1)
|
|
result = read_json(self.workspace.correction_file)['Ex 1'][0][1]['result']
|
|
self.assertEqual(result['error'], 'wrg-lbl:Ex 2?')
|
|
self.assertEqual(result['delayed'], [['wrong-label', 'Ex 2']])
|
|
|
|
def test_additional_answer_from_regrading_still_gets_resolved(self):
|
|
(self.pdf.parent / 'Ex 2.pdf').touch()
|
|
self.regraded[0]['result']['error'] = 'additional-answer'
|
|
_, calls = self.process(['Ex 1', json.dumps(self.regraded), '["Ex 1", "Ex 2"]'])
|
|
self.assertEqual(calls.call_count, 3)
|
|
result = read_json(self.workspace.correction_file)['Ex 1'][0][1]['result']
|
|
self.assertEqual(result['score'], 2.5)
|
|
self.assertEqual(result['delayed'], [['add-label', 'Ex 2']])
|