Gemini_for_enonce : améliorations

Readme.org

@@ -1,7 +1,7 @@
 #+title:  Script
 #+author: Sébastien Miquel
 #+date:   14-03-2026
-# Time-stamp: <07-08-26 18:06>
+# Time-stamp: <08-08-26 12:28>
 #+OPTIONS:
 
 * Méta
@@ -82,39 +82,52 @@ Copier `default_config.py` en `config.py`. Éventuellement le modifier.
     suite)
  3. Mettre l'énoncé, au format pdf, et l'énoncé et le corrigé au
     format .tex dans le dossier.
- 4. Mettre les fichiers pdfs scannés dans =Interro=.
- 5. Dans le dossier =Interro= créer un fichier =labels= avec les labels
-    des questions, comme =Ex 1 : 1)a)=, un par ligne.
- 6. Il faudra créer des dossiers =Text=, =Sol= et =Persp=, et dans ces
-    dossiers créer, pour chaque label (ou groupe de labels : par
-    exemple un seul fichier =Ex 1= peut être utilisé pour toutes les
-    questions de l'exercice 1) un fichier texte qui contient
-    respectivement l'énoncé, un corrigé, et des indications de comment
-    corriger (Gemini met une note sur 4, on peut demander 2 points
-    pour tel truc, etc)
- 7. Suivre les étapes plus bas.
 
 * Étapes et Script
 ** Prétraitement de l'énoncé
 
- - Dans le dossier de l'évaluation, mettre :
+Dans le dossier de l'évaluation, mettre : `enonce.pdf`, `enonce.tex`,
-   + `enonce.pdf`
+`correction.tex`.
-   + `enonce.tex`
+
-   + `correction.tex`.
+ - `python gemini_for_enonce.py Interro` or
- - `python gemini_for_enonce.py Interro`
+   `python gemini_for_enonce.py Interro --restart`
-   Se charge de créer
+
+   À partir des trois fichiers précédents, se charge de détecter les
+   labels des questions et leur contenu.
+
+   Les questions vont également être regroupées. Par la suite, quand
+   des requêtes de corrections seront effectuées sur une question,
+   seulement les énoncés des questions du groupe seront envoyés (et le
+   corrigé de la question). Il faut donc que chaque groupe contienne
+   si possible le contexte nécessaire pour comprendre la question.
+
+   Une fenêtre s'ouvre pour permettre d'éditer le résultat. Ne pas
+   hésiter à faire des groupes plus gros que les groupes par défaut.
+
+   Après relecture le script génère :
+
    + un fichier `labels` avec les labels des questions
    + Un dossier `Text` avec le contenu textuel des questions,
      regroupées.
    + Un dossier `Sol` avec le contenu textuel du corrigé, question par
-     question
+     question.
    + Un dossier `Text2`, qui compile un fichier `.tex` pour chaque
-     question
+     question (utilisé pour compiler un rendu pdf du corrigé pour
+     chaque question)
    + Un dossier `Sol2`, qui compile un fichier `.tex` pour chaque
      correction de chaque question.
+   + Un dossier `Persp` avec des instruction de barème pour chaque
+     question.
+
+   Éventuellement : vérifier et modifier les barèmes dans `Persp`.
+
+ - Alternative personnelle : `python enonce_info.py Interro`
+
 
 ** Prétraitement des copies
 
+Mettre les copies scannées au format pdf dans =Interro=.
+
  1. =./rotate_all.sh Interro= (facultatif)
 
     Retourne tous les pdf de 180°, si la photocopie a été faite à
@@ -138,15 +151,6 @@ Copier `default_config.py` en `config.py`. Éventuellement le modifier.
 
     Rerun on a single file with =python cutleft.py Interro/Copies/Copie01.pdf=
 
-** Génération d'information sur l'énoncé
-
- 1. =python enonce_info.py Interro= (gestion perso)
-OU
- 1. =python gemini_for_enonce.py Interro=
-    + Nécessite `enonce.pdf`, =enonce.tex/org= et `correction.tex/org`
-    + Génère : `Text` and `Sol` folders. Todo : `Text2`.
-    `python gemini_for_enonce.py Interro --reread` will regenerate the
-
 ** Labelisation et regroupement
 
 Set proxy with ~export HTTPS_PROXY="http://10.0.0.1:3128"~

annotating.py

@@ -280,25 +280,19 @@ import subprocess
 import PIL.ImageOps
 
 
+from config import LATEX_ANOT_AFTER, LATEX_ANOT_BEFORE
+
+
 def render_real_latex_text(text, width_px, bg_color=(255, 255, 255, 255), max_lines=None, fontsize=19):
     dpi = 100
     width_in = width_px / dpi
     line_spacing = int(fontsize * 1.2)
 
     # Use the 'standalone' class with 'varwidth' to auto-crop height while restricting width
-    latex_template = f"""\\documentclass[varwidth={width_in}in,margin=0.2cm]{{standalone}}
+    header = LATEX_ANOT_BEFORE.format(
-\\usepackage[utf8]{{inputenc}}
+        width_in=width_in, fontsize=fontsize, line_spacing=line_spacing
-\\usepackage[T1]{{fontenc}}
+    )
-\\usepackage{{lmodern}}  % Enables arbitrary font scaling
+    latex_template = f"{header}{text}{LATEX_ANOT_AFTER}"
-\\usepackage{{amsmath, amssymb}}
-\\usepackage{{mathabx}} % larger inline operators.
-\\usepackage{{commands}}
-%\\usepackage{{anyfontsize}} % replaced by lmodern
-\\begin{{document}}
-\\fontsize{{{fontsize}}}{{{line_spacing}}}\\selectfont
-{text}
-\\end{{document}}
-"""
 
     with tempfile.TemporaryDirectory() as temp_dir:
         tex_path = os.path.join(temp_dir, 'text.tex')

default_config.py

@@ -39,3 +39,45 @@ PLOTTING_KB = {
     "open original pdf": "O",
     "open eval": "i",
 }
+
+### Latex templates
+
+    ### Pour la génération d'énoncés 
+
+LATEX_BEFORE = r"""\documentclass[varwidth=24.8cm,margin=0.4cm]{standalone}
+\usepackage[utf8]{inputenc}
+\usepackage[T1]{fontenc}
+\usepackage{lmodern}
+\usepackage{amsmath, amssymb}
+\usepackage{commands}
+\usepackage{minted}
+\usepackage{graphicx}
+\usepackage{enumitem}
+\begin{document}
+    \begin{minipage}{24.8cm}
+"""
+
+LATEX_AFTER = r"""
+    \end{minipage}
+\end{document}
+"""
+
+
+    ### Pour les annotations sur copies
+
+
+LATEX_ANOT_BEFORE = r"""\documentclass[varwidth={width_in}in,margin=0.2cm]{standalone}
+\usepackage[utf8]{inputenc}
+\usepackage[T1]{fontenc}
+\usepackage{lmodern}  % Enables arbitrary font scaling
+\usepackage{amsmath, amssymb}
+\usepackage{mathabx} % larger inline operators.
+\usepackage{commands}
+%\usepackage{anyfontsize} % replaced by lmodern
+\begin{document}
+\fontsize{{{fontsize}}}{{{line_spacing}}}\selectfont
+"""
+
+LATEX_ANOT_AFTER = r"""
+\end{document}
+"""

gemini_for_enonce.py

@@ -16,7 +16,12 @@ def get_lcp(s1: str, s2: str) -> str:
     i = 0
     while i < len(s1) and i < len(s2) and s1[i] == s2[i]:
         i += 1
-    return s1[:i]
+    lcp = s1[:i]
+    if ')' in s1 or ')' in s2:
+        last_paren = lcp.rfind(')')
+        if last_paren != -1:
+            return lcp[:last_paren + 1]
+    return lcp
 
 
 import config
@@ -114,7 +119,18 @@ PROMPT_3 = """I am providing:
 2. The source code of the exam questions (`enonce` file).
 
 Your task:
- Extract important information necessary to understand the questions (e.g., definitions of objects, global notations, hypotheses, context) that are NOT part of the question texts themselves.
+ Extract important information necessary to understand the questions (e.g., definitions of objects, global notations, hypotheses, context) that are NOT part of the question texts themselves. Often, this information can be in a previous \\item that is not itself a question, but contains the question items. 
+
+For example, given LaTeX code like
+
+\\item Let N, M be two commutating matrices
+\\begin{itemize}
+  \\item Prove that N, M have a common eigenvector
+  \\item Prove that N, M are co-trigonalizable.
+\\end{itemize}
+
+the `Let N, M be two commutating matrices` part is not a question itself, and is important information to understand the next two questions.
+
  For each extracted piece of information, identify:
  1. The label of the FIRST question that comes immediately AFTER this information in the exam.
  2. The label of the LAST question that uses or relies on this information.
@@ -128,9 +144,16 @@ def find_file(folder: Path, base_name: str) -> Path:
             return path
     return None
 
-def process_exam(folder_path: str):
+def process_exam(folder_path: str, restart: bool = False):
     folder = Path(folder_path)
 
+    cache_dir = folder / "Cache"
+    cache_dir.mkdir(exist_ok=True)
+
+    cache_q_file = cache_dir / "gemini_questions.json"
+    cache_s_file = cache_dir / "gemini_solutions.json"
+    cache_c_file = cache_dir / "gemini_context.json"
+
     # 1. Resolve files
     pdf_path = folder / "enonce.pdf"
     enonce_path = find_file(folder, "enonce")
@@ -172,10 +195,8 @@ def process_exam(folder_path: str):
         response_json_schema=ExamQuestions.model_json_schema(),
     )
 
-    cache_q_file = folder / "gemini_questions.json"
+    if cache_q_file.is_file() and not restart:
-
+        print("Loading cached questions from Cache/gemini_questions.json...")
-    if cache_q_file.is_file():
-        print("Loading cached questions from gemini_questions.json...")
         response_q_text = cache_q_file.read_text(encoding="utf-8")
     else:
         print("Sending request 1 (Questions) to Gemini...")
@@ -212,10 +233,8 @@ def process_exam(folder_path: str):
         response_json_schema=ExamSolutions.model_json_schema(),
     )
 
-    cache_s_file = folder / "gemini_solutions.json"
+    if cache_s_file.is_file() and not restart:
-
+        print("Loading cached solutions from Cache/gemini_solutions.json...")
-    if cache_s_file.is_file():
-        print("Loading cached solutions from gemini_solutions.json...")
         response_s_text = cache_s_file.read_text(encoding="utf-8")
     else:
         print("Sending request 2 (Solutions) to Gemini...")
@@ -250,10 +269,8 @@ def process_exam(folder_path: str):
         response_json_schema=ExamContext.model_json_schema(),
     )
 
-    cache_c_file = folder / "gemini_context.json"
+    if cache_c_file.is_file() and not restart:
-
+        print("Loading cached context from Cache/gemini_context.json...")
-    if cache_c_file.is_file():
-        print("Loading cached context from gemini_context.json...")
         response_c_text = cache_c_file.read_text(encoding="utf-8")
     else:
         print("Sending request 3 (Context) to Gemini...")
@@ -329,51 +346,70 @@ def process_exam(folder_path: str):
     trunc_map = {}
 
     # --- INITIAL GROUPING COMPUTATION ---
+    # 1. Normalize labels first
+    for item in extracted_data.items:
+        if isinstance(item, QuestionItem):
+            item.label = item.label.replace("Exercice", "Ex").replace(".", ")")
+
+    # 2. Extract questions and compute grouping indices
     questions_only = [item for item in extracted_data.items if isinstance(item, QuestionItem)]
+    # questions_only = [item for item in extracted_data.items if isinstance(item, QuestionItem)]
 
     q_group_indices = []
     if questions_only:
-        current_g = [0]
+        n = len(questions_only)
-        for i in range(1, len(questions_only)):
+        if n == 1:
-            p = get_lcp(questions_only[current_g[0]].label, questions_only[i].label)
+            q_group_indices = [[0]]
-            proposed = current_g + [i]
+        else:
-            valid = True
+            adj_lcp = [get_lcp(questions_only[i].label, questions_only[i+1].label) for i in range(n - 1)]
-            for k in range(len(proposed) - 1):
-                if get_lcp(questions_only[proposed[k]].label, questions_only[proposed[k+1]].label) != p:
-                    valid = False
-                    break
-            if valid:
-                current_g.append(i)
-            else:
-                q_group_indices.append(current_g)
-                current_g = [i]
-        q_group_indices.append(current_g)
 
-    group_starter_labels = {questions_only[g[0]].label for g in q_group_indices[1:]} if q_group_indices else set()
+            current_g = [0]
+            for i in range(n - 1):
+                p = adj_lcp[i]
+                prev_p = adj_lcp[i - 1] if i > 0 else ""
+                next_p = adj_lcp[i + 1] if i < n - 2 else ""
+
+                # Group i and i+1 together if p is non-empty and at least as specific as adjacent LCPs
+                if p and len(p) >= len(prev_p) and len(p) >= len(next_p):
+                    current_g.append(i + 1)
+                else:
+                    q_group_indices.append(current_g)
+                    current_g = [i + 1]
+            q_group_indices.append(current_g)
+
+
+    # Build list of unique ContextItems from extracted data
+    all_contexts = [item for item in extracted_data.items if isinstance(item, ContextItem)]
 
     initial_groups = []
-    current_group = []
+    for g_indices in q_group_indices:
+        group_items = []
+        first_q_idx = g_indices[0]
+        last_q_idx = g_indices[-1]
 
-    for i, item in enumerate(extracted_data.items):
+        for q_idx in g_indices:
-        is_new_group = False
+            q_item = questions_only[q_idx]
-        if isinstance(item, QuestionItem):
-            if item.label in group_starter_labels:
-                if not (len(current_group) > 0 and isinstance(current_group[-1], ContextItem)):
-                    is_new_group = True
-        elif isinstance(item, ContextItem):
-            if i + 1 < len(extracted_data.items):
-                next_item = extracted_data.items[i+1]
-                if isinstance(next_item, QuestionItem) and next_item.label in group_starter_labels:
-                    is_new_group = True
 
-        if is_new_group and current_group:
+            # 1. Collect contexts targeting this specific question
-            initial_groups.append(current_group)
+            # 2. Or contexts carried over from an earlier group (only added at the start of the group)
-            current_group = []
+            for ctx in all_contexts:
+                target_idx = label_to_idx.get(ctx.target_question_label, -1)
+                last_idx = label_to_idx.get(ctx.last_question_label, -1)
 
-        current_group.append(item)
+                if target_idx != -1 and last_idx != -1:
+                    is_exact_target = (target_idx == q_idx)
+                    is_carried_over = (q_idx == first_q_idx and target_idx < first_q_idx and last_idx >= first_q_idx)
 
-    if current_group:
+                    if is_exact_target or is_carried_over:
-        initial_groups.append(current_group)
+                        group_items.append(ContextItem(
+                            target_question_label=ctx.target_question_label,
+                            last_question_label=ctx.last_question_label,
+                            content=ctx.content
+                        ))
+
+            group_items.append(q_item)
+
+        initial_groups.append(group_items)
 
     # ---- Transform labels, and check uniqueness
 
@@ -383,9 +419,6 @@ def process_exam(folder_path: str):
         for item in group:
             if isinstance(item, QuestionItem):
                 orig_label = item.label
-                # 1. Transform label
-                item.label = item.label.replace("Exercice", "Ex")
-                item.label = item.label.replace(".", ")")
 
                 # 2. Ensure uniqueness (prefix with XX)
                 while item.label in seen_labels:
@@ -469,11 +502,28 @@ def process_exam(folder_path: str):
             input("Press ENTER to return to the editor...")
             continue
 
+        # Map original contexts by normalized content
+        orig_contexts = {c.context_content.strip(): c for c in context_data.contexts}
+
         # 2. Actual Parsing
         grouped_items = []
+        current_raw_group = []  # Stores (is_context, label_or_flag, content)
+        all_new_q_labels = []
+
+        # Pass 1: Read all edited lines and collect question labels in sequence
+        for line in edited_lines:
+            if line == "---" or " ### " not in line:
+                continue
+            lbl, content_raw = line.split(" ### ", 1)
+            lbl = lbl.strip()
+            if lbl != "CONTEXT":
+                all_new_q_labels.append(lbl)
+
+        # Mapping from original question index to new label
+        idx_to_new_label = {i: all_new_q_labels[i] for i in range(min(len(questions_only), len(all_new_q_labels)))}
+
+        orig_q_idx = 0
         current_group = []
-        labels_list = []
-        orig_idx = 0
 
         for line in edited_lines:
             if line == "---":
@@ -488,37 +538,82 @@ def process_exam(folder_path: str):
             new_label, edited_content_raw = line.split(" ### ", 1)
             new_label = new_label.strip()
 
-            if new_label != "CONTEXT":
-                labels_list.append(new_label)
-
             if "…" in edited_content_raw and edited_content_raw in trunc_map:
                 edited_content_raw = trunc_map[edited_content_raw]
 
             edited_content = edited_content_raw.replace(' \\n ', '\n')
 
-            if orig_idx < len(extracted_data.items):
+            if new_label == "CONTEXT":
-                orig_item = extracted_data.items[orig_idx] # <-- Add this line
+                current_group.append(('CONTEXT', edited_content))
-
+            else:
-                if isinstance(orig_item, QuestionItem):
+                sol_content = questions_only[orig_q_idx].solution_content if orig_q_idx < len(questions_only) else ""
-                    current_group.append(QuestionItem(
+                current_group.append(QuestionItem(
-                        label=new_label,
+                    label=new_label,
-                        question_content=edited_content,
+                    question_content=edited_content,
-                        solution_content=orig_item.solution_content
+                    solution_content=sol_content
-                    ))
+                ))
-                elif isinstance(orig_item, ContextItem):
+                orig_q_idx += 1
-                    current_group.append(ContextItem(
-                        target_question_label=orig_item.target_question_label,
-                        last_question_label=orig_item.last_question_label,
-                        content=edited_content
-                    ))
-                orig_idx += 1
 
         if current_group:
             grouped_items.append(current_group)
 
-        # If we reached here without 'continue', the data is valid
+        # Pass 2: Resolve ContextItem target/last labels per group
+        final_grouped_items = []
+        for group in grouped_items:
+            final_group = []
+            q_in_group = [item for item in group if isinstance(item, QuestionItem)]
+            g_first_label = q_in_group[0].label if q_in_group else ""
+            g_last_label = q_in_group[-1].label if q_in_group else ""
+
+            for i, item in enumerate(group):
+                if isinstance(item, tuple) and item[0] == 'CONTEXT':
+                    c_text = item[1]
+                    norm_text = c_text.strip()
+
+                    # Find next question label in group following this context
+                    next_q_label = g_first_label
+                    for successor in group[i+1:]:
+                        if isinstance(successor, QuestionItem):
+                            next_q_label = successor.label
+                            break
+
+                    if norm_text in orig_contexts:
+                        orig_c = orig_contexts[norm_text]
+                        orig_target_idx = label_to_idx.get(orig_c.target_question_label, -1)
+                        orig_last_idx = label_to_idx.get(orig_c.last_question_label, -1)
+
+                        mapped_target = idx_to_new_label.get(orig_target_idx, next_q_label)
+                        mapped_last = idx_to_new_label.get(orig_last_idx, g_last_label)
+
+                        # Check if context's last question is BEFORE the first question of this group
+                        first_q_idx_in_exam = all_new_q_labels.index(g_first_label) if g_first_label in all_new_q_labels else -1
+                        last_q_idx_in_exam = all_new_q_labels.index(mapped_last) if mapped_last in all_new_q_labels else -1
+
+                        if last_q_idx_in_exam != -1 and first_q_idx_in_exam != -1 and last_q_idx_in_exam < first_q_idx_in_exam:
+                            mapped_last = g_last_label
+
+                        final_group.append(ContextItem(
+                            target_question_label=mapped_target,
+                            last_question_label=mapped_last,
+                            content=c_text
+                        ))
+                    else:
+                        # New context created by user
+                        final_group.append(ContextItem(
+                            target_question_label=next_q_label,
+                            last_question_label=g_last_label,
+                            content=c_text
+                        ))
+                else:
+                    final_group.append(item)
+
+            final_grouped_items.append(final_group)
+
+        grouped_items = final_grouped_items
         break
 
+    labels_list = [item.label for group in grouped_items for item in group if isinstance(item, QuestionItem)]
+
     # Save labels and proceed
     with open(folder / "labels", 'w', encoding='utf-8') as f_labels:
         for label in labels_list:
@@ -697,6 +792,8 @@ if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description="Extract exam and solution code via Gemini.")
     parser.add_argument("folder", help="Directory containing the exam files")
+    parser.add_argument("--restart", action="store_true", help="Ignore cache files and re-run extraction requests.")
+
 
     args = parser.parse_args()
-    process_exam(args.folder)
+    process_exam(args.folder, restart=args.restart)

utils.py

@@ -69,7 +69,7 @@ def pdf_images_of_contexts(root_dir, label, all_labels):
 
     # Sort by first_idx to ensure contexts are returned in logical reading order
     pertinent_contexts.sort(key=lambda x: x[0])
-    return [path for _, path in pertinent_contexts]    
+    return [path for _, path in pertinent_contexts]
 
 
 def get_exam_file_content(folder_path, mode, label):
@@ -117,23 +117,13 @@ import tempfile
 import shutil
 import subprocess
 
-def compile_to_pdf(text, output_pdf_path): # 21 cm + 3.8 (dimension de la marge de gauche)
+from config import LATEX_AFTER, LATEX_BEFORE
-    """Wraps text in a standalone template and compiles it to PDF."""
+
-    latex_template = f"""\\documentclass[varwidth=24.8cm,margin=0.4cm]{{standalone}}
+
-\\usepackage[utf8]{{inputenc}}
+def compile_to_pdf(text, output_pdf_path):
-\\usepackage[T1]{{fontenc}}
+    """Wraps text in standalone header/footer templates and compiles it to PDF."""
-\\usepackage{{lmodern}}
+    latex_template = f"{LATEX_BEFORE}{text}{LATEX_AFTER}"
-\\usepackage{{amsmath, amssymb}}
+
-\\usepackage{{commands}}
-\\usepackage{{minted}}
-\\usepackage{{graphicx}}
-\\usepackage{{enumitem}}
-\\begin{{document}}
-    \\begin{{minipage}}{{24.8cm}}
-{text}
-    \\end{{minipage}}
-\\end{{document}}
-"""
     with tempfile.TemporaryDirectory() as temp_dir:
         tex_filename = 'text.tex'
         pdf_filename = 'text.pdf'