better faulty encoding detection

tests/test_latex_json.py

@@ -1,7 +1,11 @@
 import json
 import unittest
 
-from copienator.latex_json import repair_latex_json, normalize_literal_newlines
+from copienator.latex_json import (
+    normalize_literal_newlines,
+    repair_decoded_latex_controls,
+    repair_latex_json,
+)
 from copienator.commands.gemini_for_enonce import ExamQuestions, ExamSolutions, ExamContext, GroupRubrics
 
 
@@ -44,6 +48,50 @@ class LatexJsonTests(unittest.TestCase):
         self.assertEqual(repair_latex_json(raw),
                          r'{"label":"\not", "other":"\frac", "rubric_content":"\\frac"}')
 
+    def test_feedback_text_can_be_repaired_before_json_decoding(self):
+        raw = (
+            r'{"text":"$\forall x, f \big(x\big) \neq x \to \text{false}$",'
+            r'"other":"\forall"}'
+        )
+        repaired = repair_latex_json(
+            raw,
+            additional_content_fields={"text"},
+        )
+        parsed = json.loads(repaired)
+        self.assertEqual(
+            parsed["text"],
+            r"$\forall x, f \big(x\big) \neq x \to \text{false}$",
+        )
+        self.assertEqual(parsed["other"], "\x0corall")
+
+        valid_prose = json.dumps({"text": "First line\ntext outside math"})
+        self.assertEqual(
+            repair_latex_json(
+                valid_prose,
+                additional_content_fields={"text"},
+            ),
+            valid_prose,
+        )
+
+    def test_decoded_control_repair_is_limited_to_known_commands_in_math(self):
+        corrupted = (
+            "$"
+            + "\x0corall x, f "
+            + "\x08ig(x\x08ig) "
+            + "\neq x "
+            + "\to "
+            + "\text{false}$"
+        )
+        self.assertEqual(
+            repair_decoded_latex_controls(corrupted),
+            r"$\forall x, f \big(x\big) \neq x \to \text{false}$",
+        )
+        ordinary_whitespace = "First paragraph\nnext line\tindented"
+        self.assertEqual(
+            repair_decoded_latex_controls(ordinary_whitespace),
+            ordinary_whitespace,
+        )
+
     def test_does_not_repair_structural_errors(self):
         with self.assertRaises(json.JSONDecodeError):
             json.loads(repair_latex_json(r'{"question_content":"\frac", broken}'))