119 lines
3.5 KiB
Python
119 lines
3.5 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import re
|
|
from collections.abc import Sequence
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from copienator import (
|
|
EvaluationWorkspace,
|
|
ExitCode,
|
|
atomic_write_json,
|
|
evaluation_parser,
|
|
execute,
|
|
read_json,
|
|
workspace_from_args,
|
|
)
|
|
|
|
WORD_LIST_FILE = Path(__file__).resolve().parents[1] / "data" / "liste_francais.txt"
|
|
ACCENT_PATTERN = re.compile(r"[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]")
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
return evaluation_parser("Clean encoding and LaTeX issues in correction.json.")
|
|
|
|
|
|
def escape_latex_underscores(text: str) -> str:
|
|
r"""Escape underscores outside LaTeX math environments."""
|
|
math_pattern = re.compile(
|
|
r"(\$\$.*?\$\$|\$.*?\$|\\\(.*?\\\)|\\\[.*?\\\])",
|
|
re.DOTALL,
|
|
)
|
|
parts: list[str] = []
|
|
last_end = 0
|
|
for match in math_pattern.finditer(text):
|
|
start, end = match.span()
|
|
parts.append(text[last_end:start].replace("_", r"\_"))
|
|
parts.append(match.group(0))
|
|
last_end = end
|
|
parts.append(text[last_end:].replace("_", r"\_"))
|
|
return "".join(parts)
|
|
|
|
|
|
def build_lookup_map(word_list_path: Path = WORD_LIST_FILE) -> dict[str, str]:
|
|
words = word_list_path.read_text(encoding="utf-8").splitlines()
|
|
lookup: dict[str, str] = {}
|
|
for word in words:
|
|
broken_key = ACCENT_PATTERN.sub("\x00", word)
|
|
if "\x00" in broken_key:
|
|
lookup[broken_key.lower()] = word
|
|
return lookup
|
|
|
|
|
|
def fast_fix(text: str, lookup: dict[str, str]) -> str:
|
|
def replacer(match: re.Match[str]) -> str:
|
|
broken_word = match.group(0)
|
|
return lookup.get(broken_word.lower(), broken_word)
|
|
|
|
return re.sub(r"[a-zA-Z\x00]+", replacer, text)
|
|
|
|
|
|
def fix_hex_corruption_safe(text: str) -> str:
|
|
return re.sub(
|
|
r"\x00([eEfF][0-9a-fA-F])",
|
|
lambda match: chr(int(match.group(1), 16)),
|
|
text,
|
|
)
|
|
|
|
|
|
def some_other_replacements(text: str) -> str:
|
|
return text.replace("\neq", "\\neq").replace("\not", "\\not")
|
|
|
|
|
|
def clean_string(text: str, lookup: dict[str, str]) -> str:
|
|
text = fix_hex_corruption_safe(text)
|
|
text = text.replace("\x19", "\x00")
|
|
text = text.replace("\x18", "\x00")
|
|
text = text.replace("\x00\x00", "\x00")
|
|
text = re.sub(r" \x00{1,2} ", " à ", text)
|
|
if "\x00" in text:
|
|
text = fast_fix(text, lookup).replace("\x00", "")
|
|
return escape_latex_underscores(some_other_replacements(text))
|
|
|
|
|
|
def clean_obj(value: Any, lookup: dict[str, str]) -> Any:
|
|
if isinstance(value, str):
|
|
return clean_string(value, lookup)
|
|
if isinstance(value, list):
|
|
return [clean_obj(item, lookup) for item in value]
|
|
if isinstance(value, dict):
|
|
return {
|
|
key: item if key == "suffix" else clean_obj(item, lookup)
|
|
for key, item in value.items()
|
|
}
|
|
return value
|
|
|
|
|
|
def run(
|
|
workspace: EvaluationWorkspace,
|
|
*,
|
|
word_list_path: Path = WORD_LIST_FILE,
|
|
) -> ExitCode:
|
|
workspace.require_files("correction.json")
|
|
lookup = build_lookup_map(word_list_path)
|
|
data = read_json(workspace.correction_file)
|
|
cleaned = clean_obj(data, lookup)
|
|
atomic_write_json(workspace.correction_file, cleaned)
|
|
print(f"Fixed JSON saved to {workspace.correction_file}")
|
|
return ExitCode.SUCCESS
|
|
|
|
|
|
def main(argv: Sequence[str] | None = None) -> int:
|
|
parser = build_parser()
|
|
return execute(parser, argv, lambda args: run(workspace_from_args(args)))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|