Miscs

post-correction.py

@@ -20,28 +20,28 @@ import ftfy
 import re
 import urllib.request
 
-# url = "https://raw.githubusercontent.com/hbenbel/French-Dictionary/master/dictionary/dictionary.txt"
-# french_words = urllib.request.urlopen(url).read().decode('utf-8').splitlines()
+with open('liste_francais.txt', 'r') as f:
+    french_words = f.read().splitlines()
 
 # 2. Pre-compute an O(1) lookup dictionary
 # We simulate the corruption by replacing accents with null bytes (\x00)
-# lookup_map = {}
-# for word in french_words:
-#     # Replace all French accents with \x00 to create the "broken" key
-#     broken_key = re.sub(r'[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]', '\x00', word)
-#     if '\x00' in broken_key:
-#         lookup_map[broken_key] = word # e.g., "\x00cole" -> "école"
+lookup_map = {}
+for word in french_words:
+    # Replace all French accents with \x00 to create the "broken" key
+    broken_key = re.sub(r'[éèêëàâäîïôöùûüçœÉÈÊËÀÂÄÎÏÔÖÙÛÜÇŒ]', '\x00', word)
+    if '\x00' in broken_key:
+        lookup_map[broken_key] = word # e.g., "\x00cole" -> "école"
 
 # 3. Fast replace function
 def fast_fix(text):
     # Find words containing regular letters and null bytes
-    # def replacer(match):
-    #     broken_word = match.group(0)
-    #     # Return the fixed word from our map, or leave it if not found
-    #     # (Handles case-insensitivity by falling back to lowercase map)
-    #     return lookup_map.get(broken_word.lower(), broken_word)
+    def replacer(match):
+        broken_word = match.group(0)
+        # Return the fixed word from our map, or leave it if not found
+        # (Handles case-insensitivity by falling back to lowercase map)
+        return lookup_map.get(broken_word.lower(), broken_word)
 
-    # return re.sub(r'[a-zA-Z\x00]+', replacer, text)
+    return re.sub(r'[a-zA-Z\x00]+', replacer, text)
     return text