Fix flash attention error by patching model config to use eager attention

app.py

@@ -40,6 +40,13 @@ class ChatBot:
             )
             
             self.model.eval()
+            
+            # Patch model config to avoid flash attention issues
+            if hasattr(self.model.config, '_attn_implementation'):
+                self.model.config._attn_implementation = "eager"
+            if hasattr(self.model.config, 'attn_implementation'):
+                self.model.config.attn_implementation = "eager"
+            
             self.loaded = True
             
             # Get GPU distribution info
@@ -85,7 +92,7 @@ class ChatBot:
             inputs = self.tokenizer(prompt, return_tensors="pt")
             inputs = {k: v.to(self.model.device) for k, v in inputs.items()}
             
-            # Generate
+            # Generate with explicit attention settings
             with torch.no_grad():
                 outputs = self.model.generate(
                     **inputs,
@@ -94,6 +101,7 @@ class ChatBot:
                     top_p=top_p,
                     do_sample=temperature > 0,
                     pad_token_id=self.tokenizer.eos_token_id,
+                    use_cache=True,  # Enable KV caching
                 )
             
             # Decode