app.py

import requests
import json
import os
import uuid
import zipfile
import io
import subprocess
import os
import re
import warnings
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from typing import Any, Dict, List, Literal, Optional

warnings.filterwarnings("ignore")

app = FastAPI(title="3GPP Specification Splitter API", 
              description="API to split and display specifications by their chapters & sub-chapters")

app.mount("/static", StaticFiles(directory="static"), name="static")

origins = [
    "*",
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

def get_text(specification: str, version: str):
    """Récupère les bytes du PDF à partir d'une spécification et d'une version."""
    doc_id = specification
    series = doc_id.split(".")[0]
    
    response = requests.get(
        f"https://www.3gpp.org/ftp/Specs/archive/{series}_series/{doc_id}/{doc_id.replace('.', '')}-{version}.zip", 
        verify=False,
        headers={"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
    )
    
    if response.status_code != 200:
        raise Exception(f"Téléchargement du ZIP échoué pour {specification}-{version}")
    
    zip_bytes = io.BytesIO(response.content)
    
    with zipfile.ZipFile(zip_bytes) as zf:
        for file_name in zf.namelist():
            if file_name.endswith("zip"):
                print("Another ZIP !")
                zip_bytes = io.BytesIO(zf.read(file_name))
                zf = zipfile.ZipFile(zip_bytes)
                for file_name2 in zf.namelist():
                    if file_name2.endswith("doc") or file_name2.endswith("docx"):
                        if "cover" in file_name2.lower():
                            print("COVER !")
                            continue
                        ext = file_name2.split(".")[-1]
                        doc_bytes = zf.read(file_name2)
                        temp_id = str(uuid.uuid4())
                        input_path = f"/tmp/{temp_id}.{ext}"
                        output_path = f"/tmp/{temp_id}.txt"
                        
                        with open(input_path, "wb") as f:
                            f.write(doc_bytes)
                        
                        subprocess.run([
                            "libreoffice",
                            "--headless",
                            "--convert-to", "txt",
                            "--outdir", "/tmp",
                            input_path
                        ], check=True)
                        
                        with open(output_path, "r") as f:
                            txt_data = [line.strip() for line in f if line.strip()]
                        
                        os.remove(input_path)
                        os.remove(output_path)
                        return txt_data
            elif file_name.endswith("doc") or file_name.endswith("docx"):
                if "cover" in file_name.lower():
                    print("COVER !")
                    continue
                ext = file_name.split(".")[-1]
                doc_bytes = zf.read(file_name)
                temp_id = str(uuid.uuid4())
                input_path = f"/tmp/{temp_id}.{ext}"
                output_path = f"/tmp/{temp_id}.txt"
                
                print("Ecriture")
                with open(input_path, "wb") as f:
                    f.write(doc_bytes)

                print("Convertissement")                
                subprocess.run([
                    "libreoffice",
                    "--headless",
                    "--convert-to", "txt",
                    "--outdir", "/tmp",
                    input_path
                ], check=True)
                
                print("Ecriture TXT")
                with open(output_path, "r", encoding="utf-8") as f:
                    txt_data = [line.strip() for line in f if line.strip()]
                
                os.remove(input_path)
                os.remove(output_path)
                return txt_data
    
    raise Exception(f"Aucun fichier .doc/.docx trouvé dans le ZIP pour {specification}-{version}")

def get_latest_version(spec: str) -> str:
    try:
        req = requests.post("https://organizedprogrammers-3gppdocfinder.hf.space/find", headers={"Content-Type": "application/json"}, data=json.dumps({"doc_id": spec}), verify=False)
    except Exception as e:
        raise HTTPException(500, f"An error has occured while getting latest version: {e}")
    if req.status_code == 200:
        reqJS = req.json()
        return reqJS['version']
    else:
        raise HTTPException(req.status_code, "An error has occured while getting latest version")

def create_nested_structure(chapters, text, real_toc_indexes):
    """Crée une structure hiérarchique où les sous-sections sont imbriquées dans leurs sections parentes."""
    result = {}
    
    # Trier les chapitres par numéro de section
    sorted_chapters = sorted(chapters, key=lambda x: [int(p) if p.isdigit() else p for p in x.split()[0].split('.')])
    
    # Préparer les contenus des chapitres
    chapter_contents = {}
    for i, chapter in enumerate(sorted_chapters):
        current_index = real_toc_indexes[chapter]
        
        # Déterminer l'index de fin
        end_index = len(text)
        if i < len(sorted_chapters) - 1:
            next_chapter = sorted_chapters[i + 1]
            end_index = real_toc_indexes[next_chapter]
        
        # Extraire et nettoyer le contenu
        content = text[current_index + 1:end_index]
        cleaned_content = "\n".join(content).strip()
        chapter_contents[chapter] = cleaned_content
    
    # Fonction récursive pour construire la structure hiérarchique
    def insert_section(root, section_path, title, content):
        """Insère une section dans l'arborescence hiérarchique."""
        parts = section_path.split('.')
        
        # Ignorer les sections sans titre réel (seulement si le titre est vide ou juste des espaces)
        if not title.strip():
            # Si c'est une section sans titre mais avec du contenu, on peut le fusionner avec sa première sous-section
            # ou simplement l'ignorer selon votre besoin
            return None
        
        # Cas de base: section de premier niveau
        if len(parts) == 1:
            key = section_path + " " + title
            if key not in root:
                root[key] = {"content": content, "subsections": {}}
            else:
                root[key]["content"] = content
            return root[key]
        
        # Trouver ou créer le parent
        parent_path = '.'.join(parts[:-1])
        for key in root.keys():
            if key.startswith(parent_path + " "):
                # Parent trouvé, insérer dans ses sous-sections
                section_key = section_path + " " + title
                if section_key not in root[key]["subsections"]:
                    root[key]["subsections"][section_key] = {"content": content, "subsections": {}}
                else:
                    root[key]["subsections"][section_key]["content"] = content
                return root[key]["subsections"][section_key]
        
        # Parent non trouvé, il faut le créer d'abord
        # Rechercher le titre du parent
        parent_title = ""
        for chapter in sorted_chapters:
            if chapter.split()[0] == parent_path:
                parts = chapter.split(maxsplit=1)
                parent_title = parts[1] if len(parts) > 1 else ""
                break
        
        # Si le parent n'a pas de titre, on cherche un parent plus haut
        if not parent_title.strip():
            # On peut soit ignorer cette branche, soit essayer de trouver un parent valide plus haut
            grand_parent_parts = parent_path.split('.')
            if len(grand_parent_parts) > 1:
                grand_parent_path = '.'.join(grand_parent_parts[:-1])
                for key in root.keys():
                    if key.startswith(grand_parent_path + " "):
                        # On a trouvé un grand-parent valide, on insère directement dedans
                        section_key = section_path + " " + title
                        if section_key not in root[key]["subsections"]:
                            root[key]["subsections"][section_key] = {"content": content, "subsections": {}}
                        return root[key]["subsections"][section_key]
            # Si on n'a pas trouvé de grand-parent valide, on insère à la racine
            section_key = section_path + " " + title
            root[section_key] = {"content": content, "subsections": {}}
            return root[section_key]
        
        # Créer le parent récursivement
        parent_section = insert_section(root, parent_path, parent_title, "")
        
        # Si le parent n'a pas pu être créé (car sans titre), on insère à la racine
        if parent_section is None:
            section_key = section_path + " " + title
            root[section_key] = {"content": content, "subsections": {}}
            return root[section_key]
        
        # Maintenant insérer cette section dans le parent nouvellement créé
        section_key = section_path + " " + title
        parent_section["subsections"][section_key] = {"content": content, "subsections": {}}
        return parent_section["subsections"][section_key]
    
    # Traiter chaque chapitre
    for chapter in sorted_chapters:
        parts = chapter.split(maxsplit=1)
        section_num = parts[0]
        section_title = parts[1] if len(parts) > 1 else ""
        
        # Ne traiter que les sections avec un titre
        if section_title.strip():
            insert_section(result, section_num, section_title, chapter_contents[chapter])
    
    return result

class SpecRequest(BaseModel):
    specification: str
    version: Optional[str] = None

@app.get("/")
def main_page():
    return FileResponse(os.path.join("templates", "index.html"))

@app.post("/online/plain")
def get_file_from_spec_id_version(req: SpecRequest) -> Dict[str, str]:
    spec = req.specification
    version = req.version
    if not version:
        version = get_latest_version(spec)
    
    text = get_text(spec, version)
    forewords = []
    for x in range(len(text)):
        line = text[x]
        if "Foreword" in line:
            forewords.append(x)
        if len(forewords) >= 2:
            break

    toc_brut = text[forewords[0]:forewords[1]]
    chapters = []
    for line in toc_brut:
        x = line.split("\t")
        if re.fullmatch(r"^\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
        if re.fullmatch(r"^\d\.\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
        if re.fullmatch(r"^\d\.\d\.\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
        if re.fullmatch(r"^\d\.\d\.\d.\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
        if re.fullmatch(r"^\d\.\d\.\d.\d.\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))

    real_toc_indexes = {}

    for chapter in chapters:
        try:
            x = text.index(chapter)
            real_toc_indexes[chapter] = x
        except ValueError as e:
            real_toc_indexes[chapter] = -float("inf")

    document = {}
    toc = list(real_toc_indexes.keys())
    index_toc = list(real_toc_indexes.values())
    curr_index = 0
    for x in range(1, len(toc)):
        document[toc[curr_index].replace("\t", " ")] = re.sub(r"[\ \t]+", " ", "\n".join(text[index_toc[curr_index]+1:index_toc[x]]))
        curr_index = x

    document[toc[curr_index].replace("\t"," ")] = re.sub(r"\s+", " ", " ".join(text[index_toc[curr_index]+1:]))
    return document

@app.post("/online")
def get_file_from_spec_id_version(req: SpecRequest) -> Dict:
    spec = req.specification
    version = req.version
    if not version:
        version = get_latest_version(spec)
    
    text = get_text(spec, version)
    forewords = []
    for x in range(len(text)):
        line = text[x]
        if "Foreword" in line:
            forewords.append(x)
        if len(forewords) >= 2:
            break

    toc_brut = text[forewords[0]:forewords[1]]
    chapters = []
    for line in toc_brut:
        x = line.split("\t")
        if re.fullmatch(r"^\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
        if re.fullmatch(r"^\d\.\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
        if re.fullmatch(r"^\d\.\d\.\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
        if re.fullmatch(r"^\d\.\d\.\d.\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))
        if re.fullmatch(r"^\d\.\d\.\d.\d.\d\t[A-Z][a-zA-Z0-9\s,;!?'.-]*$", line):
            chapters.append(x[0] if len(x) == 1 else "\t".join(x[:2]))

    real_toc_indexes = {}

    for chapter in chapters:
        try:
            x = text.index(chapter)
            real_toc_indexes[chapter] = x
        except ValueError as e:
            real_toc_indexes[chapter] = -float("inf")

    return create_nested_structure(chapters, text, real_toc_indexes)