from __future__ import annotations import argparse import re from collections.abc import Sequence from pathlib import Path from typing import Any from copienator import ( EvaluationWorkspace, ExitCode, atomic_write_json, evaluation_parser, execute, read_json, workspace_from_args, ) WORD_LIST_FILE = Path(__file__).with_name("liste_francais.txt") ACCENT_PATTERN = re.compile(r"[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]") def build_parser() -> argparse.ArgumentParser: return evaluation_parser("Clean encoding and LaTeX issues in correction.json.") def escape_latex_underscores(text: str) -> str: r"""Escape underscores outside LaTeX math environments.""" math_pattern = re.compile( r"(\$\$.*?\$\$|\$.*?\$|\\\(.*?\\\)|\\\[.*?\\\])", re.DOTALL, ) parts: list[str] = [] last_end = 0 for match in math_pattern.finditer(text): start, end = match.span() parts.append(text[last_end:start].replace("_", r"\_")) parts.append(match.group(0)) last_end = end parts.append(text[last_end:].replace("_", r"\_")) return "".join(parts) def build_lookup_map(word_list_path: Path = WORD_LIST_FILE) -> dict[str, str]: words = word_list_path.read_text(encoding="utf-8").splitlines() lookup: dict[str, str] = {} for word in words: broken_key = ACCENT_PATTERN.sub("\x00", word) if "\x00" in broken_key: lookup[broken_key.lower()] = word return lookup def fast_fix(text: str, lookup: dict[str, str]) -> str: def replacer(match: re.Match[str]) -> str: broken_word = match.group(0) return lookup.get(broken_word.lower(), broken_word) return re.sub(r"[a-zA-Z\x00]+", replacer, text) def fix_hex_corruption_safe(text: str) -> str: return re.sub( r"\x00([eEfF][0-9a-fA-F])", lambda match: chr(int(match.group(1), 16)), text, ) def some_other_replacements(text: str) -> str: return text.replace("\neq", "\\neq").replace("\not", "\\not") def clean_string(text: str, lookup: dict[str, str]) -> str: text = fix_hex_corruption_safe(text) text = text.replace("\x19", "\x00") text = text.replace("\x18", "\x00") text = text.replace("\x00\x00", "\x00") text = re.sub(r" \x00{1,2} ", " à ", text) if "\x00" in text: text = fast_fix(text, lookup).replace("\x00", "") return escape_latex_underscores(some_other_replacements(text)) def clean_obj(value: Any, lookup: dict[str, str]) -> Any: if isinstance(value, str): return clean_string(value, lookup) if isinstance(value, list): return [clean_obj(item, lookup) for item in value] if isinstance(value, dict): return { key: item if key == "suffix" else clean_obj(item, lookup) for key, item in value.items() } return value def run( workspace: EvaluationWorkspace, *, word_list_path: Path = WORD_LIST_FILE, ) -> ExitCode: workspace.require_files("correction.json") lookup = build_lookup_map(word_list_path) data = read_json(workspace.correction_file) cleaned = clean_obj(data, lookup) atomic_write_json(workspace.correction_file, cleaned) print(f"Fixed JSON saved to {workspace.correction_file}") return ExitCode.SUCCESS def main(argv: Sequence[str] | None = None) -> int: parser = build_parser() return execute(parser, argv, lambda args: run(workspace_from_args(args))) if __name__ == "__main__": raise SystemExit(main())