miscs

post-correction.py

@@ -39,23 +39,47 @@ def fast_fix(text):
         broken_word = match.group(0)
         # Return the fixed word from our map, or leave it if not found
         # (Handles case-insensitivity by falling back to lowercase map)
-        return lookup_map.get(broken_word.lower(), broken_word)
+        fixed = lookup_map.get(broken_word.lower())
+        # if not fixed:
+            # print(f"No match found for: {repr(broken_word)}")
+        return fixed or broken_word
 
     return re.sub(r'[a-zA-Z\x00]+', replacer, text)
-    return text
+    # return text
 
 
 INPUT_FILE = Path(INPUT_DIR) / "correction.json"
 OUTPUT_FILE = Path(INPUT_DIR) / "correction.json"
 
+def fix_hex_corruption_safe(text):
+    # Only matches \x00 followed by hex if it results in an accented character
+    # or common Latin-1 symbols
+    return re.sub(r'\x00([eEfF][0-9a-fA-F])',
+                  lambda m: chr(int(m.group(1), 16)),
+                  text)
+
+def some_other_replacements(s):
+    s = s.replace("\neq", "\\neq")
+    s = s.replace("\not", "\\not")
+    return s
+
+
 def clean_string(s: str) -> str:
     # fix encoding issues
-    s = ftfy.fix_text(s)
+    # s = ftfy.fix_text(s)
+    # print(s)
+    s = fix_hex_corruption_safe(s)
+
     s = s.replace('\x19', '\x00')
     s = s.replace('\x18', '\x00')
-    s = fast_fix(s)
-    s = s.replace('\x00', '')
-    return s
+    s = s.replace('\x00\x00', '\x00')
+
+    s = re.sub(r' \x00{1,2} ', ' à ', s)
+
+    if '\x00' in s:
+        s = fast_fix(s)
+        s = s.replace('\x00', '')
+    return some_other_replacements(s)
 
 
 def clean_obj(obj):