Standardisation 2
This commit is contained in:
+89
-126
@@ -1,155 +1,118 @@
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from copienator import atomic_write_json
|
||||
from copienator import (
|
||||
EvaluationWorkspace,
|
||||
ExitCode,
|
||||
atomic_write_json,
|
||||
evaluation_parser,
|
||||
execute,
|
||||
read_json,
|
||||
workspace_from_args,
|
||||
)
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
sys.exit("Usage: python script.py <InputDir>")
|
||||
WORD_LIST_FILE = Path(__file__).with_name("liste_francais.txt")
|
||||
ACCENT_PATTERN = re.compile(r"[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]")
|
||||
|
||||
def escape_latex_underscores(text):
|
||||
r"""
|
||||
Escape '_' outside LaTeX math environments.
|
||||
Supports:
|
||||
- $...$
|
||||
- $$...$$
|
||||
- \( ... \)
|
||||
- \[ ... \]
|
||||
"""
|
||||
|
||||
# Regex matching LaTeX math blocks
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
return evaluation_parser("Clean encoding and LaTeX issues in correction.json.")
|
||||
|
||||
|
||||
def escape_latex_underscores(text: str) -> str:
|
||||
r"""Escape underscores outside LaTeX math environments."""
|
||||
math_pattern = re.compile(
|
||||
r'(\$\$.*?\$\$|' # $$...$$
|
||||
r'\$.*?\$|' # $...$
|
||||
r'\\\(.*?\\\)|' # \( ... \)
|
||||
r'\\\[.*?\\\])', # \[ ... \]
|
||||
re.DOTALL
|
||||
r"(\$\$.*?\$\$|\$.*?\$|\\\(.*?\\\)|\\\[.*?\\\])",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
parts = []
|
||||
parts: list[str] = []
|
||||
last_end = 0
|
||||
|
||||
for match in math_pattern.finditer(text):
|
||||
start, end = match.span()
|
||||
|
||||
# Escape underscores outside math
|
||||
outside = text[last_end:start].replace('_', r'\_')
|
||||
parts.append(outside)
|
||||
|
||||
# Keep math block unchanged
|
||||
parts.append(text[last_end:start].replace("_", r"\_"))
|
||||
parts.append(match.group(0))
|
||||
|
||||
last_end = end
|
||||
parts.append(text[last_end:].replace("_", r"\_"))
|
||||
return "".join(parts)
|
||||
|
||||
# Remaining text after last math block
|
||||
outside = text[last_end:].replace('_', r'\_')
|
||||
parts.append(outside)
|
||||
|
||||
return ''.join(parts)
|
||||
def build_lookup_map(word_list_path: Path = WORD_LIST_FILE) -> dict[str, str]:
|
||||
words = word_list_path.read_text(encoding="utf-8").splitlines()
|
||||
lookup: dict[str, str] = {}
|
||||
for word in words:
|
||||
broken_key = ACCENT_PATTERN.sub("\x00", word)
|
||||
if "\x00" in broken_key:
|
||||
lookup[broken_key.lower()] = word
|
||||
return lookup
|
||||
|
||||
arg_path = Path(sys.argv[1])
|
||||
tasks = [] # List of tuples: (filepath_str, label_str)
|
||||
results = {}
|
||||
|
||||
INPUT_DIR = str(arg_path)
|
||||
if not arg_path.exists():
|
||||
sys.exit(f"Directory {INPUT_DIR} not found.")
|
||||
|
||||
import json
|
||||
import ftfy
|
||||
import re
|
||||
import urllib.request
|
||||
|
||||
with open('liste_francais.txt', 'r') as f:
|
||||
french_words = f.read().splitlines()
|
||||
|
||||
# 2. Pre-compute an O(1) lookup dictionary
|
||||
# We simulate the corruption by replacing accents with null bytes (\x00)
|
||||
lookup_map = {}
|
||||
for word in french_words:
|
||||
# Replace all French accents with \x00 to create the "broken" key
|
||||
broken_key = re.sub(r'[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]', '\x00', word)
|
||||
if '\x00' in broken_key:
|
||||
lookup_map[broken_key] = word # e.g., "\x00cole" -> "école"
|
||||
|
||||
# 3. Fast replace function
|
||||
def fast_fix(text):
|
||||
# Find words containing regular letters and null bytes
|
||||
def replacer(match):
|
||||
def fast_fix(text: str, lookup: dict[str, str]) -> str:
|
||||
def replacer(match: re.Match[str]) -> str:
|
||||
broken_word = match.group(0)
|
||||
# Return the fixed word from our map, or leave it if not found
|
||||
# (Handles case-insensitivity by falling back to lowercase map)
|
||||
fixed = lookup_map.get(broken_word.lower())
|
||||
# if not fixed:
|
||||
# print(f"No match found for: {repr(broken_word)}")
|
||||
return fixed or broken_word
|
||||
return lookup.get(broken_word.lower(), broken_word)
|
||||
|
||||
return re.sub(r'[a-zA-Z\x00]+', replacer, text)
|
||||
# return text
|
||||
return re.sub(r"[a-zA-Z\x00]+", replacer, text)
|
||||
|
||||
|
||||
INPUT_FILE = Path(INPUT_DIR) / "correction.json"
|
||||
OUTPUT_FILE = Path(INPUT_DIR) / "correction.json"
|
||||
|
||||
def fix_hex_corruption_safe(text):
|
||||
# Only matches \x00 followed by hex if it results in an accented character
|
||||
# or common Latin-1 symbols
|
||||
return re.sub(r'\x00([eEfF][0-9a-fA-F])',
|
||||
lambda m: chr(int(m.group(1), 16)),
|
||||
text)
|
||||
|
||||
def some_other_replacements(s):
|
||||
s = s.replace("\neq", "\\neq")
|
||||
s = s.replace("\not", "\\not")
|
||||
return s
|
||||
def fix_hex_corruption_safe(text: str) -> str:
|
||||
return re.sub(
|
||||
r"\x00([eEfF][0-9a-fA-F])",
|
||||
lambda match: chr(int(match.group(1), 16)),
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
def clean_string(s: str) -> str:
|
||||
# fix encoding issues
|
||||
# s = ftfy.fix_text(s)
|
||||
# print(s)
|
||||
s = fix_hex_corruption_safe(s)
|
||||
|
||||
s = s.replace('\x19', '\x00')
|
||||
s = s.replace('\x18', '\x00')
|
||||
s = s.replace('\x00\x00', '\x00')
|
||||
|
||||
s = re.sub(r' \x00{1,2} ', ' à ', s)
|
||||
|
||||
if '\x00' in s:
|
||||
s = fast_fix(s)
|
||||
s = s.replace('\x00', '')
|
||||
s = some_other_replacements(s)
|
||||
return escape_latex_underscores(s)
|
||||
def some_other_replacements(text: str) -> str:
|
||||
return text.replace("\neq", "\\neq").replace("\not", "\\not")
|
||||
|
||||
|
||||
def clean_obj(obj):
|
||||
if isinstance(obj, str):
|
||||
return clean_string(obj)
|
||||
|
||||
elif isinstance(obj, list):
|
||||
return [clean_obj(x) for x in obj]
|
||||
|
||||
elif isinstance(obj, dict):
|
||||
r = {}
|
||||
for k, v in obj.items():
|
||||
if k != "suffix":
|
||||
r[k] = clean_obj(v)
|
||||
else:
|
||||
r[k] = v
|
||||
return r
|
||||
|
||||
else:
|
||||
return obj
|
||||
def clean_string(text: str, lookup: dict[str, str]) -> str:
|
||||
text = fix_hex_corruption_safe(text)
|
||||
text = text.replace("\x19", "\x00")
|
||||
text = text.replace("\x18", "\x00")
|
||||
text = text.replace("\x00\x00", "\x00")
|
||||
text = re.sub(r" \x00{1,2} ", " à ", text)
|
||||
if "\x00" in text:
|
||||
text = fast_fix(text, lookup).replace("\x00", "")
|
||||
return escape_latex_underscores(some_other_replacements(text))
|
||||
|
||||
|
||||
with open(INPUT_FILE, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
def clean_obj(value: Any, lookup: dict[str, str]) -> Any:
|
||||
if isinstance(value, str):
|
||||
return clean_string(value, lookup)
|
||||
if isinstance(value, list):
|
||||
return [clean_obj(item, lookup) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: item if key == "suffix" else clean_obj(item, lookup)
|
||||
for key, item in value.items()
|
||||
}
|
||||
return value
|
||||
|
||||
data = clean_obj(data)
|
||||
|
||||
atomic_write_json(OUTPUT_FILE, data)
|
||||
def run(
|
||||
workspace: EvaluationWorkspace,
|
||||
*,
|
||||
word_list_path: Path = WORD_LIST_FILE,
|
||||
) -> ExitCode:
|
||||
workspace.require_files("correction.json")
|
||||
lookup = build_lookup_map(word_list_path)
|
||||
data = read_json(workspace.correction_file)
|
||||
cleaned = clean_obj(data, lookup)
|
||||
atomic_write_json(workspace.correction_file, cleaned)
|
||||
print(f"Fixed JSON saved to {workspace.correction_file}")
|
||||
return ExitCode.SUCCESS
|
||||
|
||||
print("Fixed JSON saved to", OUTPUT_FILE)
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
return execute(parser, argv, lambda args: run(workspace_from_args(args)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
Reference in New Issue
Block a user