Upload folder using huggingface_hub

api.py

@@ -5,6 +5,8 @@ from io import BytesIO
 import base64
 from transformers import AutoProcessor, AutoModelForCausalLM
 import threading
+from unittest.mock import patch
+from transformers.dynamic_module_utils import get_imports
 
 app = Flask(__name__)
 
@@ -17,7 +19,19 @@ args = parser.parse_args()
 # Determine the device
 device = "cpu"
 # Initialize the model and processor
-model = AutoModelForCausalLM.from_pretrained(args.model_path, trust_remote_code=True, device_map=device)
+
+with patch("transformers.dynamic_module_utils.get_imports", fixed_get_imports): #workaround for unnecessary flash_attn requirement
+            model = AutoModelForCausalLM.from_pretrained(model_path, attn_implementation="sdpa", torch_dtype=dtype,trust_remote_code=True)
+
+def fixed_get_imports(filename: str | os.PathLike) -> list[str]:
+    if not str(filename).endswith("modeling_florence2.py"):
+        return get_imports(filename)
+    imports = get_imports(filename)
+    imports.remove("flash_attn")
+    return imports
+
+with patch("transformers.dynamic_module_utils.get_imports", fixed_get_imports): #workaround for unnecessary flash_attn requirement
+            model = AutoModelForCausalLM.from_pretrained(args.model_path, attn_implementation="sdpa", torch_dtype=dtype,trust_remote_code=True)
 processor = AutoProcessor.from_pretrained(args.model_path, trust_remote_code=True, device_map=device)
 
 lock = threading.Lock()  # Use a lock to ensure thread safety when accessing the model