Update app.py

app.py

@@ -29,6 +29,23 @@ from tokenizers import Tokenizer
 from tokenizers.models import BPE
 from tokenizers.trainers import BpeTrainer
 
+
+
+def download_nltk_resources():
+    resources = [
+        'punkt',
+        'stopwords',
+        'snowball_data',
+    ]
+    for resource in resources:
+        try:
+            nltk.download(resource, quiet=True)
+        except Exception as e:
+            print(f"Failed to download {resource}: {str(e)}")
+
+download_nltk_resources()
+
+
 nltk.download('stopwords', quiet=True)
 nltk.download('punkt', quiet=True)
 
@@ -50,6 +67,10 @@ MODELS = {
     }
 }
 
+def simple_tokenize(text):
+    """Simple tokenization fallback method."""
+    return text.split()
+
 def preprocess_text(text, lang='german'):
     # Convert to lowercase
     text = text.lower()
@@ -58,15 +79,26 @@ def preprocess_text(text, lang='german'):
     text = re.sub(r'[^a-zA-Z\s]', '', text)
     
     # Tokenize
-    tokens = word_tokenize(text, language=lang)
+    try:
+        tokens = word_tokenize(text, language=lang)
+    except LookupError:
+        print(f"Warning: NLTK punkt tokenizer for {lang} not found. Using simple tokenization.")
+        tokens = simple_tokenize(text)
     
     # Remove stopwords
-    stop_words = set(stopwords.words(lang))
+    try:
+        stop_words = set(stopwords.words(lang))
+    except LookupError:
+        print(f"Warning: Stopwords for {lang} not found. Skipping stopword removal.")
+        stop_words = set()
     tokens = [token for token in tokens if token not in stop_words]
     
     # Stemming
-    stemmer = SnowballStemmer(lang)
-    tokens = [stemmer.stem(token) for token in tokens]
+    try:
+        stemmer = SnowballStemmer(lang)
+        tokens = [stemmer.stem(token) for token in tokens]
+    except ValueError:
+        print(f"Warning: SnowballStemmer for {lang} not available. Skipping stemming.")
     
     return ' '.join(tokens)