First functional version (still testing)

app.py

@@ -1,81 +1,21 @@
-import gradio as gr
-import threading
-import sys
-import io
-import time
-from dotenv import load_dotenv
+from datetime import datetime
 import os
-from contextlib import redirect_stdout
+import warnings
+import traceback
+import gradio as gr
+import subprocess
+from huggingface_hub import Repository
 from git import Repo
-from huggingface_hub import Repository, HfApi
+import requests
 
-load_dotenv()
+warnings.filterwarnings('ignore')
 
-# == PATHS and SETTINGS ==
 DOC_INDEXER = "indexer_multi.py"
 SPEC_INDEXER = "spec_indexer_multi.py"
 DOC_INDEX_FILE = "indexed_docs.json"
 SPEC_INDEX_FILE = "indexed_specifications.json"
-GIT_REPO_PATH = os.path.abspath(".")  # or absolute path to repo
-HF_REPO_ID = "OrganizedProgrammers/3GPPDocFinder"
-HF_TOKEN = os.environ.get("HF_TOKEN")  # set this as env var
-
-# == Helpers ==
-def run_python_module(module_path):
-    """
-    Dynamically run a python module, capture and yield stdout in real time.
-    """
-    def runner():
-        local_vars = {}
-        buffer = io.StringIO()
-        try:
-            with redirect_stdout(buffer):
-                # Import as module, call main()
-                import runpy
-                runpy.run_path(module_path, run_name="__main__")
-        except Exception as e:
-            print(f"\n❌ Error: {e}")
-        finally:
-            yield buffer.getvalue()
-            buffer.close()
-    yield from runner()
-
-def commit_and_push_github(files, message):
-    repo = Repo(GIT_REPO_PATH)
-    repo.git.add(files)
-    repo.index.commit(message)
-    try:
-        repo.git.push()
-    except Exception as e:
-        print(f"Git push failed: {e}")
-
-def commit_and_push_hf(files, message):
-    if not HF_TOKEN:
-        return "No HF_TOKEN provided. Skipping HuggingFace push."
-    hf_repo_dir = os.path.join(GIT_REPO_PATH, "hf_spaces")
-    repo = None
-    if not os.path.exists(hf_repo_dir):
-        repo = Repository(
-            local_dir=hf_repo_dir,
-            clone_from=HF_REPO_ID,
-            token=HF_TOKEN,
-            skip_lfs_files=True
-        )
-    else:
-        repo = Repository(
-            local_dir=hf_repo_dir,
-            token=HF_TOKEN,
-            skip_lfs_files=True
-        )
-        repo.git_pull()
-    # Copy artifact files to huggingface space
-    for f in files:
-        import shutil
-        shutil.copy2(f, os.path.join(hf_repo_dir, f))
-    repo.git_add(auto_lfs_track=True)
-    repo.git_commit(message)
-    repo.git_push()
-    return "Pushed to HuggingFace."
+HF_SEARCH_REPO = "OrganizedProgrammers/3GPPDocFinder"
+REPO_DIR = os.path.dirname(os.path.abspath(__file__))
 
 def get_docs_stats():
     if os.path.exists(DOC_INDEX_FILE):
@@ -101,39 +41,146 @@ def get_scopes_stats():
         return len(data['scopes'])
     return 0
 
-# == Gradio Functions ==
-
-def index_documents(progress=gr.Progress()):
-    progress(0, desc="Starting document indexing…")
-    log = ""
-    for output in run_python_module(DOC_INDEXER):
-        log = output
-        progress(0.7, desc="Indexing in progress...")
-        yield log
-    commit_and_push_github([DOC_INDEX_FILE], "Update doc index via Gradio")
-    commit_and_push_hf([DOC_INDEX_FILE], "Update doc index via Gradio")
-    progress(1, desc="Done!")
-    yield log + "\n\n✅ Finished! Committed and pushed."
-
-def index_specifications(progress=gr.Progress()):
-    progress(0, desc="Starting specifications indexing…")
-    log = ""
-    for output in run_python_module(SPEC_INDEXER):
-        log = output
-        progress(0.7, desc="Indexing in progress...")
-        yield log
-    commit_and_push_github([SPEC_INDEX_FILE], "Update spec index via Gradio")
-    commit_and_push_hf([SPEC_INDEX_FILE], "Update spec index via Gradio")
-    progress(1, desc="Done!")
-    yield log + "\n\n✅ Finished! Committed and pushed."
+def check_permissions(user: str, token: str):
+    try:
+        req = requests.get("https://huggingface.co/api/whoami-v2", verify=False, headers={"Accept": "application/json", "Authorization": f"Bearer {token}"})
+        if req.status_code != 200:
+            return False
+        reqJson: dict = req.json()
+        if not reqJson.get("name") or reqJson['name'] != user:
+            return False
+        if not reqJson.get("orgs") or len(reqJson['orgs']) == 0:
+            return False
+        for org in reqJson['orgs']:
+            if "645cfa1b5ebf379fd6d8a339" == org['id']:
+                return True
+        if not reqJson.get('auth') or reqJson['auth'] == {}:
+            return False
+        if reqJson['auth']['accessToken']['role'] != "fineGrained":
+            return False
+        for scope in reqJson['auth']['accessToken']['fineGrained']['scoped']:
+            if scope['entity']['type'] == "org" and scope['entity']['_id'] == "645cfa1b5ebf379fd6d8a339" and all(perm in scope['permissions'] for perm in ['repo.write', 'repo.content.read']):
+                return True
+        return False
+    except Exception as e:
+        traceback.print_exception(e)
+        return False
+
+def update_logged(user: str, token: str):
+    if check_permissions(user, token):
+        return gr.update(visible=False), gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
+    else:
+        return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
+
+def commit_and_push_3gppindexers(user, token, files, message):
+    repo = Repo(REPO_DIR)
+    origin = repo.remotes.origin
+    origin.pull()
+    yield "Git pull succeed !"
+    repo.git.add(files)
+    repo.index.commit(message)
+    try:
+        repo.git.push(f"https://{user}:{token}@huggingface.co/spaces/OrganizedProgrammers/3GPPIndexers")
+        yield "Git push succeed !"
+        yield "Wait for Huggingface to restart the Space"
+    except Exception as e:
+        yield f"Git push failed: {e}"
+
+def commit_and_push_3gppdocfinder(token, files, message):
+    if not token:
+        yield "No token provided. Skipping HuggingFace push."
+    hf_repo_dir = os.path.join(REPO_DIR, "hf_spaces")
+    repo = None
+    if not os.path.exists(hf_repo_dir):
+        repo = Repository(
+            local_dir=hf_repo_dir,
+            repo_type="space",
+            clone_from=HF_SEARCH_REPO,
+            token=token, 
+            skip_lfs_files=False
+        )
+    else:
+        repo = Repository(
+            local_dir=hf_repo_dir,
+            repo_type="space",
+            token=token,
+            skip_lfs_files=False
+        )
+        repo.git_pull()
+    # Copy artifact files to huggingface space
+    for f in files:
+        import shutil
+        shutil.copy2(f, os.path.join(hf_repo_dir, f))
+    repo.git_add(auto_lfs_track=True)
+    repo.git_commit(message)
+    repo.git_push()
+    yield "Pushed to HuggingFace."
 
 def refresh_stats():
     return str(get_docs_stats()), str(get_specs_stats()), str(get_scopes_stats())
 
-# == UI ==
+def stream_script_output(script_path):
+    process = subprocess.Popen(
+        ["python", script_path],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        bufsize=1,
+        universal_newlines=True,
+    )
+    output = ""
+    for line in process.stdout:
+        output += line
+        yield output
+    process.stdout.close()
+    process.wait()
+    yield output
+
+def index_documents(user, token):
+    # Désactiver tous les boutons
+    yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), "⏳ Indexation en cours..."
+
+    # Lancer l'indexation
+    if not check_permissions(user, token):
+        yield gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True), "❌ Identifiants invalides"
+        return
+
+    for log in stream_script_output(DOC_INDEXER):
+        yield gr.update(None), gr.update(None), gr.update(None), log
+
+    d = datetime.today().strftime("%d/%m/%Y-%H:%M:%S")
+    yield gr.update(None), gr.update(None), gr.update(None), commit_and_push_3gppdocfinder(token, ["indexed_docs.json"], f"Update documents indexer via Indexer: {d}")
+    yield gr.update(None), gr.update(None), gr.update(None), commit_and_push_3gppindexers(user, token, ["indexed_docs.json"], f"Update documents indexer via Indexer: {d}")
+
+    # Réactiver les boutons à la fin
+    yield gr.update(None), gr.update(None), gr.update(None), "✅ Terminé."
+
+def index_specifications(user, token):
+    # Désactiver tous les boutons
+    yield gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), "⏳ Indexation en cours..."
+
+    # Lancer l'indexation
+    if not check_permissions(user, token):
+        yield gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True), "❌ Identifiants invalides"
+        return
+
+    for log in stream_script_output(DOC_INDEXER):
+        yield gr.update(None), gr.update(None), gr.update(None), log
+
+    d = datetime.today().strftime("%d/%m/%Y-%H:%M:%S")
+    yield gr.update(None), gr.update(None), gr.update(None), commit_and_push_3gppdocfinder(token, ["indexed_specifications.json"], f"Update specifications indexer via Indexer: {d}")
+    yield gr.update(None), gr.update(None), gr.update(None), commit_and_push_3gppindexers(user, token, ["indexed_specifications.json"], f"Update specifications indexer via Indexer: {d}")
+
+    # Réactiver les boutons à la fin
+    yield gr.update(None), gr.update(None), gr.update(None), "✅ Terminé."
+
 with gr.Blocks(theme=gr.themes.Soft()) as demo:
     gr.Markdown("## 📄 3GPP Indexers")
-    with gr.Row():
+    with gr.Row() as r1:
+        with gr.Column():
+            git_user = gr.Textbox(label="Git user (for push/pull indexes)")
+            git_pass = gr.Textbox(label="Git Token", type="password")
+            btn_login = gr.Button("Login", variant="primary")
+    with gr.Row(visible=False) as r2:
         with gr.Column():
             doc_count = gr.Textbox(label="Docs Indexed", value=str(get_docs_stats()), interactive=False)
             btn_docs = gr.Button("Re-index Documents", variant="primary")
@@ -142,11 +189,11 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
             btn_specs = gr.Button("Re-index Specifications", variant="primary")
         with gr.Column():
             scope_count = gr.Textbox(label="Scopes Indexed", value=str(get_scopes_stats()), interactive=False)
-    out = gr.Textbox(label="Output/Log", lines=13)
-    refresh = gr.Button("🔄 Refresh Stats")
-    btn_docs.click(index_documents, outputs=out)
-    btn_specs.click(index_specifications, outputs=out)
+    out = gr.Textbox(label="Output/Log", lines=13, autoscroll=True, visible=False)
+    refresh = gr.Button(value="🔄 Refresh Stats", visible=False)
+    btn_login.click(update_logged, inputs=[git_user, git_pass], outputs=[r1, r2, out, refresh])
+    btn_docs.click(index_documents, inputs=[git_user, git_pass], outputs=[btn_docs, btn_specs, refresh, out])
+    btn_specs.click(index_specifications, inputs=[git_user, git_pass], outputs=[btn_docs, btn_specs, refresh, out])
     refresh.click(refresh_stats, outputs=[doc_count, spec_count, scope_count])
 
-if __name__ == "__main__":
-    demo.launch()
+demo.launch()

indexer_multi.py

@@ -0,0 +1,286 @@
+from datetime import datetime
+import requests
+from bs4 import BeautifulSoup
+import json
+import os
+import time
+import warnings
+import re
+import concurrent.futures
+import threading
+from typing import List, Dict, Any
+
+warnings.filterwarnings("ignore")
+
+class TsgDocIndexer:
+    def __init__(self, max_workers=10):
+        self.main_ftp_url = "https://www.3gpp.org/ftp"
+        self.indexer_file = "indexed_docs.json"
+        self.indexer, self.latest_date = self.load_indexer()
+        self.valid_doc_pattern = re.compile(r'^(S[1-6P]|C[1-6P])-\d+', flags=re.IGNORECASE)
+        self.max_workers = max_workers
+        
+        # Verrous pour les opérations thread-safe
+        self.print_lock = threading.Lock()
+        self.indexer_lock = threading.Lock()
+        
+        # Compteurs pour le suivi
+        self.total_indexed = 0
+        self.processed_count = 0
+        self.total_count = 0
+
+    def load_indexer(self):
+        """Load existing index if available"""
+        if os.path.exists(self.indexer_file):
+            with open(self.indexer_file, "r", encoding="utf-8") as f:
+                x = json.load(f)
+                return x["docs"], x["last_indexed_date"]
+        return {}, None
+    
+    def save_indexer(self):
+        """Save the updated index"""
+        with open(self.indexer_file, "w", encoding="utf-8") as f:
+            today = datetime.today()
+            self.latest_date = today.strftime("%d/%m/%Y-%H:%M:%S")
+            output = {"docs": self.indexer, "last_indexed_date": self.latest_date}
+            json.dump(output, f, indent=4, ensure_ascii=False)
+
+    def get_docs_from_url(self, url):
+        """Récupérer la liste des documents/répertoires depuis une URL"""
+        try:
+            response = requests.get(url, verify=False, timeout=10)
+            soup = BeautifulSoup(response.text, "html.parser")
+            return [item.get_text() for item in soup.select("tr td a")]
+        except Exception as e:
+            with self.print_lock:
+                print(f"Erreur lors de l'accès à {url}: {e}")
+            return []
+
+    def is_valid_document_pattern(self, filename):
+        """Vérifier si le nom de fichier correspond au motif requis"""
+        return bool(self.valid_doc_pattern.match(filename))
+
+    def is_zip_file(self, filename):
+        """Vérifier si le fichier est un ZIP"""
+        return filename.lower().endswith('.zip')
+
+    def extract_doc_id(self, filename):
+        """Extraire l'identifiant du document à partir du nom de fichier s'il correspond au motif"""
+        if self.is_valid_document_pattern(filename):
+            match = self.valid_doc_pattern.match(filename)
+            if match:
+                # Retourner le motif complet (comme S1-12345)
+                full_id = filename.split('.')[0]  # Enlever l'extension si présente
+                return full_id.split('_')[0]  # Enlever les suffixes après underscore si présents
+        return None
+
+    def process_zip_files(self, files_list, base_url, workshop=False):
+        """Traiter une liste de fichiers pour trouver et indexer les ZIP valides"""
+        indexed_count = 0
+        
+        for file in files_list:
+            if file in ['./', '../', 'ZIP/', 'zip/']:
+                continue
+                
+            # Vérifier si c'est un fichier ZIP et s'il correspond au motif
+            if self.is_zip_file(file) and (self.is_valid_document_pattern(file) or workshop):
+                file_url = f"{base_url}/{file}"
+                
+                # Extraire l'ID du document
+                doc_id = self.extract_doc_id(file)
+                if doc_id is None:
+                    doc_id = file.split('.')[0]
+                if doc_id:
+                    # Vérifier si ce fichier est déjà indexé
+                    with self.indexer_lock:
+                        if doc_id in self.indexer and self.indexer[doc_id] == file_url:
+                            continue
+                            
+                        # Ajouter ou mettre à jour l'index
+                        self.indexer[doc_id] = file_url
+                        indexed_count += 1
+                        self.total_indexed += 1
+                        
+        return indexed_count
+
+    def process_meeting(self, meeting, wg_url, workshop=False):
+        """Traiter une réunion individuelle avec multithreading"""
+        try:
+            if meeting in ['./', '../']:
+                return 0
+                
+            meeting_url = f"{wg_url}/{meeting}"
+            
+            with self.print_lock:
+                print(f" Vérification de la réunion: {meeting}")
+                
+            # Vérifier le contenu de la réunion
+            meeting_contents = self.get_docs_from_url(meeting_url)
+            
+            key = None
+            if "docs" in [x.lower() for x in meeting_contents]:
+                key = "docs"
+            elif "tdocs" in [x.lower() for x in meeting_contents]:
+                key = "tdocs"
+                
+            if key is not None:
+                docs_url = f"{meeting_url}/{key}"
+                
+                with self.print_lock:
+                    print(f" Vérification des documents dans {docs_url}")
+                    
+                # Récupérer la liste des fichiers dans le dossier Docs
+                docs_files = self.get_docs_from_url(docs_url)
+                
+                # 1. Indexer les fichiers ZIP directement dans le dossier Docs
+                docs_indexed_count = self.process_zip_files(docs_files, docs_url, workshop)
+                
+                if docs_indexed_count > 0:
+                    with self.print_lock:
+                        print(f" {docs_indexed_count} nouveaux fichiers ZIP indexés dans Docs")
+                
+                # 2. Vérifier le sous-dossier ZIP s'il existe
+                if "zip" in [x.lower() for x in docs_files]:
+                    zip_url = f"{docs_url}/zip"
+                    
+                    with self.print_lock:
+                        print(f" Vérification du dossier ZIP: {zip_url}")
+                        
+                    # Récupérer les fichiers dans le sous-dossier ZIP
+                    zip_files = self.get_docs_from_url(zip_url)
+                    
+                    # Indexer les fichiers ZIP dans le sous-dossier ZIP
+                    zip_indexed_count = self.process_zip_files(zip_files, zip_url, workshop)
+                    
+                    if zip_indexed_count > 0:
+                        with self.print_lock:
+                            print(f" {zip_indexed_count} nouveaux fichiers ZIP indexés dans le dossier ZIP")
+            
+            # Mise à jour du compteur de progression
+            with self.indexer_lock:
+                self.processed_count += 1
+                
+            # Affichage de la progression
+            with self.print_lock:
+                progress = (self.processed_count / self.total_count) * 100 if self.total_count > 0 else 0
+                print(f"\rProgression: {self.processed_count}/{self.total_count} réunions traitées ({progress:.1f}%)", end="")
+                
+            return 1  # Réunion traitée avec succès
+            
+        except Exception as e:
+            with self.print_lock:
+                print(f"\nErreur lors du traitement de la réunion {meeting}: {str(e)}")
+            return 0
+
+    def process_workgroup(self, wg, main_url):
+        """Traiter un groupe de travail avec multithreading pour ses réunions"""
+        if wg in ['./', '../']:
+            return
+            
+        wg_url = f"{main_url}/{wg}"
+        
+        with self.print_lock:
+            print(f" Vérification du groupe de travail: {wg}")
+            
+        # Récupérer les dossiers de réunion
+        meeting_folders = self.get_docs_from_url(wg_url)
+        
+        # Ajouter au compteur total
+        self.total_count += len([m for m in meeting_folders if m not in ['./', '../']])
+        
+        # Utiliser ThreadPoolExecutor pour traiter les réunions en parallèle
+        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(self.process_meeting, meeting, wg_url) 
+                      for meeting in meeting_folders if meeting not in ['./', '../']]
+            
+            # Attendre que toutes les tâches soient terminées
+            concurrent.futures.wait(futures)
+        
+        # Sauvegarder après chaque groupe de travail
+        with self.indexer_lock:
+            self.save_indexer()
+
+    def index_all_tdocs(self):
+        """Indexer tous les documents ZIP dans la structure FTP 3GPP avec multithreading"""
+        print("Démarrage de l'indexation des documents ZIP 3GPP (S1-S6, SP, C1-C6, CP)...")
+        
+        start_time = time.time()
+        docs_count_before = len(self.indexer)
+        
+        # Principaux groupes TSG
+        main_groups = ["tsg_sa", "tsg_ct"]  # Ajouter d'autres si nécessaire
+        
+        for main_tsg in main_groups:
+            print(f"\nIndexation de {main_tsg.upper()}...")
+            
+            main_url = f"{self.main_ftp_url}/{main_tsg}"
+            
+            # Récupérer les groupes de travail
+            workgroups = self.get_docs_from_url(main_url)
+            
+            # Traiter chaque groupe de travail séquentiellement
+            # (mais les réunions à l'intérieur seront traitées en parallèle)
+            for wg in workgroups:
+                self.process_workgroup(wg, main_url)
+        
+        docs_count_after = len(self.indexer)
+        new_docs_count = docs_count_after - docs_count_before
+        
+        print(f"\nIndexation terminée en {time.time() - start_time:.2f} secondes")
+        print(f"Nouveaux documents ZIP indexés: {new_docs_count}")
+        print(f"Total des documents dans l'index: {docs_count_after}")
+        
+        return self.indexer
+
+    def index_all_workshops(self):
+        print("Démarrage de l'indexation des workshops ZIP 3GPP...")
+        start_time = time.time()
+        docs_count_before = len(self.indexer)
+
+        print("\nIndexation du dossier 'workshop'")
+        main_url = f"{self.main_ftp_url}/workshop"
+            
+        # Récupérer les dossiers de réunion
+        meeting_folders = self.get_docs_from_url(main_url)
+        
+        # Ajouter au compteur total
+        self.total_count += len([m for m in meeting_folders if m not in ['./', '../']])
+        
+        # Utiliser ThreadPoolExecutor pour traiter les réunions en parallèle
+        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = [executor.submit(self.process_meeting, meeting, main_url, workshop=True) 
+                      for meeting in meeting_folders if meeting not in ['./', '../']]
+            concurrent.futures.wait(futures)
+        
+        # Sauvegarder après chaque groupe de travail
+        with self.indexer_lock:
+            self.save_indexer()
+        
+        docs_count_after = len(self.indexer)
+        new_docs_count = docs_count_after - docs_count_before
+        
+        print(f"\nIndexation terminée en {time.time() - start_time:.2f} secondes")
+        print(f"Nouveaux documents ZIP indexés: {new_docs_count}")
+        print(f"Total des documents dans l'index: {docs_count_after}")
+        
+        return self.indexer
+            
+
+
+def main():
+    # Nombre de workers pour le multithreading (ajustable selon les ressources disponibles)
+    max_workers = 64
+    
+    indexer = TsgDocIndexer(max_workers=max_workers)
+    
+    try:
+        indexer.index_all_tdocs()
+        indexer.index_all_workshops()
+    except Exception as e:
+        print(f"Erreur lors de l'indexation: {e}")
+    finally:
+        indexer.save_indexer()
+        print("Index sauvegardé.")
+
+if __name__ == "__main__":
+    main()

spec_indexer_multi.py

@@ -0,0 +1,294 @@
+import datetime
+import time
+import sys
+import json
+import traceback
+import requests
+import zipfile
+import uuid
+import os
+import re
+import subprocess
+import concurrent.futures
+import threading
+from io import StringIO, BytesIO
+from typing import List, Dict, Any
+
+import pandas as pd
+import numpy as np
+import warnings
+
+warnings.filterwarnings("ignore")
+
+# Caractères pour le formatage des versions
+chars = "0123456789abcdefghijklmnopqrstuvwxyz"
+
+# Verrous pour les opérations thread-safe
+print_lock = threading.Lock()
+dict_lock = threading.Lock()
+scope_lock = threading.Lock()
+
+# Dictionnaires globaux
+indexed_specifications = {}
+scopes_by_spec_num = {}
+processed_count = 0
+total_count = 0
+
+def get_text(specification: str, version: str):
+    """Récupère les bytes du PDF à partir d'une spécification et d'une version."""
+    doc_id = specification
+    series = doc_id.split(".")[0]
+    
+    response = requests.get(
+        f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version}.zip", 
+        verify=False,
+        headers={"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
+    )
+    
+    if response.status_code != 200:
+        raise Exception(f"Téléchargement du ZIP échoué pour {specification}-{version}")
+    
+    zip_bytes = BytesIO(response.content)
+    
+    with zipfile.ZipFile(zip_bytes) as zf:
+        for file_name in zf.namelist():
+            if file_name.endswith("zip"):
+                print("Another ZIP !")
+                zip_bytes = BytesIO(zf.read(file_name))
+                zf = zipfile.ZipFile(zip_bytes)
+                for file_name2 in zf.namelist():
+                    if file_name2.endswith("doc") or file_name2.endswith("docx"):
+                        if "cover" in file_name2.lower():
+                            print("COVER !")
+                            continue
+                        ext = file_name2.split(".")[-1]
+                        doc_bytes = zf.read(file_name2)
+                        temp_id = str(uuid.uuid4())
+                        input_path = f"/tmp/{temp_id}.{ext}"
+                        output_path = f"/tmp/{temp_id}.txt"
+                        
+                        with open(input_path, "wb") as f:
+                            f.write(doc_bytes)
+                        
+                        subprocess.run([
+                            "libreoffice",
+                            "--headless",
+                            "--convert-to", "txt",
+                            "--outdir", "/tmp",
+                            input_path
+                        ], check=True)
+                        
+                        with open(output_path, "r") as f:
+                            txt_data = [line.strip() for line in f if line.strip()]
+                        
+                        os.remove(input_path)
+                        os.remove(output_path)
+                        return txt_data
+            elif file_name.endswith("doc") or file_name.endswith("docx"):
+                if "cover" in file_name.lower():
+                    print("COVER !")
+                    continue
+                ext = file_name.split(".")[-1]
+                doc_bytes = zf.read(file_name)
+                temp_id = str(uuid.uuid4())
+                input_path = f"/tmp/{temp_id}.{ext}"
+                output_path = f"/tmp/{temp_id}.txt"
+                
+                print("Ecriture")
+                with open(input_path, "wb") as f:
+                    f.write(doc_bytes)
+
+                print("Convertissement")                
+                subprocess.run([
+                    "libreoffice",
+                    "--headless",
+                    "--convert-to", "txt",
+                    "--outdir", "/tmp",
+                    input_path
+                ], check=True)
+                
+                print("Ecriture TXT")
+                with open(output_path, "r", encoding="utf-8") as f:
+                    txt_data = [line.strip() for line in f if line.strip()]
+                
+                os.remove(input_path)
+                os.remove(output_path)
+                return txt_data
+    
+    raise Exception(f"Aucun fichier .doc/.docx trouvé dans le ZIP pour {specification}-{version}")
+
+def get_scope(specification: str, version: str):
+    try:
+        spec_text = get_text(specification, version)
+        scp_i = 0
+        nxt_i = 0
+        for x in range(len(spec_text)):
+            text = spec_text[x]
+            if re.search(r"scope$", text, flags=re.IGNORECASE):
+                scp_i = x
+                nxt_i = scp_i + 10
+            if re.search(r"references$", text, flags=re.IGNORECASE):
+                nxt_i = x
+
+        return re.sub(r"\s+", " ", " ".join(spec_text[scp_i+1:nxt_i])) if len(spec_text[scp_i+1:nxt_i]) < 2 else "Not found"
+    except Exception as e:
+        traceback.print_exception(e)
+        return "Not found (error)"
+
+def process_specification(spec: Dict[str, Any], columns: List[str]) -> None:
+    """Traite une spécification individuelle avec multithreading."""
+    global processed_count, indexed_specifications, scopes_by_spec_num
+    
+    try:
+        if spec.get('vers', None) is None:
+            return
+            
+        doc_id = str(spec["spec_num"])
+        series = doc_id.split(".")[0]
+        
+        a, b, c = str(spec["vers"]).split(".")
+        
+        # Formatage de l'URL selon la version
+        if not (int(a) > 35 or int(b) > 35 or int(c) > 35):
+            version_code = f"{chars[int(a)]}{chars[int(b)]}{chars[int(c)]}"
+            spec_url = f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version_code}.zip"
+        else:
+            x, y, z = str(a), str(b), str(c)
+            while len(x) < 2:
+                x = "0" + x
+            while len(y) < 2:
+                y = "0" + y
+            while len(z) < 2:
+                z = "0" + z
+            version_code = f"{x}{y}{z}"
+            spec_url = f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version_code}.zip"
+        
+        string = f"{spec['spec_num']}+-+{spec['title']}+-+{spec['type']}+-+{spec['vers']}+-+{spec['WG']}+-+Rel-{spec['vers'].split('.')[0]}"
+        
+        metadata = {
+            "id": str(spec["spec_num"]),
+            "title": spec["title"],
+            "type": spec["type"],
+            "release": str(spec["vers"].split(".")[0]),
+            "version": str(spec["vers"]),
+            "working_group": spec["WG"],
+            "url": spec_url
+        }
+        
+        # Vérification si le scope existe déjà pour ce numéro de spécification
+        spec_num = str(spec["spec_num"])
+        
+        with scope_lock:
+            if spec_num in scopes_by_spec_num:
+                # Réutilisation du scope existant
+                metadata["scope"] = scopes_by_spec_num[spec_num]
+                with print_lock:
+                    print(f"\nRéutilisation du scope pour {spec_num}")
+            else:
+                # Extraction du scope seulement si nécessaire
+                if not (int(a) > 35 or int(b) > 35 or int(c) > 35):
+                    version_for_scope = f"{chars[int(a)]}{chars[int(b)]}{chars[int(c)]}"
+                else:
+                    version_for_scope = version_code
+                
+                with print_lock:
+                    print(f"\nExtraction du scope pour {spec_num} (version {version_for_scope})")
+                
+                try:
+                    scope = get_scope(metadata["id"], version_for_scope)
+                    # Stockage du scope pour une utilisation future
+                    scopes_by_spec_num[spec_num] = scope
+                    metadata["scope"] = scope
+                except Exception as e:
+                    error_msg = f"Erreur lors de l'extraction du scope: {str(e)}"
+                    metadata["scope"] = error_msg
+                    scopes_by_spec_num[spec_num] = error_msg
+        
+        # Mise à jour du dictionnaire global avec verrou
+        with dict_lock:
+            string += f"+-+{metadata['scope']}" if metadata['scope'] != " " or metadata['scope'] != "" or "not found" not in metadata['scope'].lower() else ""
+            indexed_specifications[string] = metadata
+            processed_count += 1
+        
+        # Affichage de la progression avec verrou
+        with print_lock:
+            sys.stdout.write(f"\rTraitement: {processed_count}/{total_count} spécifications")
+            sys.stdout.flush()
+            
+    except Exception as e:
+        with print_lock:
+            print(f"\nErreur lors du traitement de {spec.get('spec_num', 'inconnu')}: {str(e)}")
+
+def main():
+    global total_count
+    old_length = 0
+    
+    start_time = time.time()
+    
+    # Récupération des spécifications depuis le site 3GPP
+    print("Récupération des spécifications depuis 3GPP...")
+    response = requests.get(
+        f'https://www.3gpp.org/dynareport?code=status-report.htm', 
+        headers={"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}, 
+        verify=False
+    )
+    
+    # Analyse des tableaux HTML
+    dfs = pd.read_html(
+        StringIO(response.text), 
+        storage_options={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}, 
+        encoding="utf-8"
+    )
+    
+    for x in range(len(dfs)):
+        dfs[x] = dfs[x].replace({np.nan: None})
+    
+    # Extraction des colonnes nécessaires
+    columns_needed = [0, 1, 2, 3, 4]
+    extracted_dfs = [df.iloc[:, columns_needed] for df in dfs]
+    columns = [x.replace("\xa0", "_") for x in extracted_dfs[0].columns]
+    
+    # Préparation des spécifications
+    specifications = []
+    for df in extracted_dfs:
+        for index, row in df.iterrows():
+            doc = row.to_list()
+            doc_dict = dict(zip(columns, doc))
+            specifications.append(doc_dict)
+    
+    total_count = len(specifications)
+    print(f"Traitement de {total_count} spécifications avec multithreading...")
+    
+    try:
+        # Vérification si un fichier de scopes existe déjà
+        if os.path.exists("indexed_specifications.json"):
+            with open("indexed_specifications.json", "r", encoding="utf-8") as f:
+                global scopes_by_spec_num
+                f_up = json.load(f)
+                scopes_by_spec_num = f_up['scopes']
+                before = len(f_up['specs'])
+                print(f"Chargement de {len(scopes_by_spec_num)} scopes depuis le cache.")
+        
+        # Utilisation de ThreadPoolExecutor pour le multithreading
+        with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+            futures = [executor.submit(process_specification, spec, columns) for spec in specifications]
+            concurrent.futures.wait(futures)
+    
+    finally:
+        # Sauvegarde des résultats
+        result = {
+            "specs": indexed_specifications,
+            "scopes": scopes_by_spec_num,
+            "last_indexed_date": datetime.datetime.today().strftime("%d-%m-%Y")
+        }
+        
+        with open("indexed_specifications.json", "w", encoding="utf-8") as f:
+            json.dump(result, f, indent=4, ensure_ascii=False)
+        
+        elapsed_time = time.time() - start_time
+        print(f"\nTraitement terminé en {elapsed_time:.2f} secondes")
+        print(f"Nouveaux specifications : {len(indexed_specifications) - before}")
+        print(f"Résultats sauvegardés dans indexed_specifications.json")
+
+if __name__ == "__main__":
+    main()