Update app.py

app.py

@@ -12,7 +12,12 @@ description = "You can use [🐋🐳microsoft/Orca-2-13b](https://huggingface.co
 # os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:24'
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model_name = "microsoft/Orca-2-13b"
-model = AutoModelForCausalLM.from_pretrained(model_name, device_map='auto')
+offload_folder = './model_weights'
+
+if not os.path.exists(offload_folder):
+    os.makedirs(offload_folder)
+
+model = AutoModelForCausalLM.from_pretrained(model_name, device_map='auto', offload_folder=offload_folder)
 tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False,)
 
 class OrcaChatBot: