Update utils.py

utils.py

@@ -24,7 +24,7 @@ import uuid
 
 from transformers import pipeline, AutoModelForSeq2SeqLM, AutoTokenizer, AutoModelForCausalLM, GPTNeoForCausalLM, GPT2Tokenizer,  DistilBertTokenizer, DistilBertForQuestionAnswering
 from sentence_transformers import SentenceTransformer, util
-from huggingface_hub import HfApi
+from huggingface_hub import HfApi, hf_hub_download
 from typing import List, Dict
 
 import gradio as gr
@@ -265,6 +265,22 @@ def clean_text(text):
 # Directory Loader Konfigurieren um Text zu extrahieren
 ##################################################
 
+def access_pdf(self, filename):
+        # Temporäre Datei erstellen
+        with tempfile.NamedTemporaryFile(delete=False, suffix=self.file_type) as temp_file:
+            temp_path = temp_file.name
+
+        # Datei aus dem Hugging Face Space herunterladen
+        hf_hub_download(
+            repo_id=DATA_REPO_ID,
+            filename=os.path.join(self.directory_path, filename),
+            repo_type=DATA_REPO_TYPE,
+            local_dir=os.path.dirname(temp_path),
+            local_dir_use_symlinks=False
+        )
+
+        return temp_path
+
 #besseren directory Loader als CustomLoader definieren, der den inhalt des dokuemnts, die seitenzahlen, die überschriften und die pfadezu den dokumenten extrahieren
 def create_directory_loader(file_type, directory_path):
     loaders = {
@@ -272,7 +288,9 @@ def create_directory_loader(file_type, directory_path):
         '.word': load_word_with_metadata,
     }
 
+    
     class CustomLoader:
+    """
         def __init__(self, directory_path, file_type, loader_func):
             self.directory_path = directory_path
             self.file_type = file_type
@@ -288,7 +306,23 @@ def create_directory_loader(file_type, directory_path):
             return documents
 
     return CustomLoader(directory_path, file_type, loaders[file_type])
-    
+    """
+    def __init__(self, directory_path, file_type, loader_func):
+        self.directory_path = directory_path
+        self.file_type = file_type
+        self.loader_func = loader_func
+
+    def load(self):
+        documents = []
+        # Annahme: directory_path ist jetzt ein Pfad innerhalb des Hugging Face Spaces
+        files = self.list_files_in_hf_space(self.directory_path)
+        for file in files:
+            if file.endswith(self.file_type):
+                file_path = self.access_pdf(file)
+                documents.extend(self.loader_func(file_path))
+        return documents
+
+    return CustomLoader(directory_path, file_type, loaders[file_type])
 
 ################################################
 # Custom Loader-Funktionen zu dem DirektoryLoader