Miscs improvements (Interro02)

copienator/commands/post_correction.py

@@ -19,6 +19,10 @@ from copienator import (
 
 WORD_LIST_FILE = Path(__file__).resolve().parents[1] / "data" / "liste_francais.txt"
 ACCENT_PATTERN = re.compile(r"[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]")
+MATH_PATTERN = re.compile(
+    r"(\$\$.*?\$\$|\$.*?\$|\\\(.*?\\\)|\\\[.*?\\\])",
+    re.DOTALL,
+)
 
 
 def build_parser() -> argparse.ArgumentParser:
@@ -26,22 +30,37 @@ def build_parser() -> argparse.ArgumentParser:
 
 
 def escape_latex_underscores(text: str) -> str:
-    r"""Escape underscores outside LaTeX math environments."""
-    math_pattern = re.compile(
-        r"(\$\$.*?\$\$|\$.*?\$|\\\(.*?\\\)|\\\[.*?\\\])",
-        re.DOTALL,
-    )
+    r"""Escape underscores outside math without double-escaping existing ones."""
+
+    def escape_plain(value: str) -> str:
+        # Collapse any existing escape run as well, making cleanup idempotent.
+        return re.sub(r"\\*_", lambda _match: r"\_", value)
+
     parts: list[str] = []
     last_end = 0
-    for match in math_pattern.finditer(text):
+    for match in MATH_PATTERN.finditer(text):
         start, end = match.span()
-        parts.append(text[last_end:start].replace("_", r"\_"))
+        parts.append(escape_plain(text[last_end:start]))
         parts.append(match.group(0))
         last_end = end
-    parts.append(text[last_end:].replace("_", r"\_"))
+    parts.append(escape_plain(text[last_end:]))
     return "".join(parts)
 
 
+def normalize_overescaped_latex_commands(text: str) -> str:
+    r"""Collapse doubled command escapes inside LaTeX math environments.
+
+    Model responses occasionally contain ``\\mathbb`` after JSON decoding where
+    LaTeX requires ``\mathbb``.  A doubled backslash followed by whitespace is a
+    legitimate row break (for example in ``cases``), so it must be preserved.
+    """
+
+    def normalize_math(match: re.Match[str]) -> str:
+        return re.sub(r"\\\\(?=[A-Za-z{}])", r"\\", match.group(0))
+
+    return MATH_PATTERN.sub(normalize_math, text)
+
+
 def build_lookup_map(word_list_path: Path = WORD_LIST_FILE) -> dict[str, str]:
     words = word_list_path.read_text(encoding="utf-8").splitlines()
     lookup: dict[str, str] = {}
@@ -68,8 +87,23 @@ def fix_hex_corruption_safe(text: str) -> str:
     )
 
 
-def some_other_replacements(text: str) -> str:
-    return text.replace("\neq", "\\neq").replace("\not", "\\not")
+def repair_json_escape_corruption(text: str) -> str:
+    r"""Restore observed LaTeX commands consumed as JSON control escapes."""
+    replacements = (
+        ("\x0crac", r"\frac"),
+        ("\x0ceuille", r"\equiv"),
+        ("\theta", r"\theta"),
+        ("\times", r"\times"),
+        ("\textbackslash ", "\\"),
+        ("\negthinspace", r"\negthinspace"),
+        ("\neq", r"\neq"),
+        ("\not", r"\not"),
+        ("∈", r"\ensuremath{\in}"),
+        ("⊂", r"\ensuremath{\subset}"),
+    )
+    for broken, repaired in replacements:
+        text = text.replace(broken, repaired)
+    return text
 
 
 def clean_string(text: str, lookup: dict[str, str]) -> str:
@@ -80,7 +114,9 @@ def clean_string(text: str, lookup: dict[str, str]) -> str:
     text = re.sub(r" \x00{1,2} ", " à ", text)
     if "\x00" in text:
         text = fast_fix(text, lookup).replace("\x00", "")
-    return escape_latex_underscores(some_other_replacements(text))
+    text = repair_json_escape_corruption(text)
+    text = normalize_overescaped_latex_commands(text)
+    return escape_latex_underscores(text)
 
 
 def clean_obj(value: Any, lookup: dict[str, str]) -> Any: