merge

app.py

@@ -1,11 +1,15 @@
 import os
 
 os.system('cd TimeSformer;'
-          'python setup.py build develop; cd ..')
+          'pip install .; cd ..')
 
 os.system('ls -l')
+os.system('pwd')
 
+import os, sys
+sys.path.append("/home/user/app/TimeSformer/")
 
+import timesformer
 
 
 import torch
@@ -39,7 +43,8 @@ yaml=YAML(typ='safe')
 
 
 use_cuda = torch.cuda.is_available()
-device = torch.deivce('cuda') if use_cuda else torch.deivce('cpu')
+device = torch.device('cuda') if use_cuda else torch.device('cpu')
+device_type = 'cuda' if use_cuda else 'cpu'
 
 ## Load model
 
@@ -107,7 +112,7 @@ num_beams=3
 max_length=30
 
 
-
+model.bfloat16()
 
 
 def inference(image, audio, video, task_type, instruction):
@@ -129,7 +134,7 @@ def inference(image, audio, video, task_type, instruction):
 
 
 
-    with torch.autocast(device_type='cuda', dtype=torch.float16, enabled=True):
+    with torch.autocast(device_type=device_type, dtype=torch.bfloat16, enabled=True):
 
         out = model(image=image, text=text_input, mode='generate', return_dict=True, max_length=max_length, 
                     do_sample=do_sample, num_beams=num_beams)

models/epalm.py

@@ -211,7 +211,7 @@ class ePALM(nn.Module):
         self.no_attention_mask = False
 
         if low_cpu:
-            self.model_text = OPTForCausalLM.from_pretrained(opt_model_name, config=config_opt, revision="float16", torch_dtype=torch.float16, low_cpu_mem_usage=False)
+            self.model_text = OPTForCausalLM.from_pretrained(opt_model_name, config=config_opt, torch_dtype=torch.float16, low_cpu_mem_usage=False)
         else:
             self.model_text = OPTForCausalLM.from_pretrained(opt_model_name, config=config_opt)
             

models/timesformer.py

@@ -10,6 +10,8 @@ import warnings
 import torch.nn.functional as F
 import numpy as np
 
+
+
 from timesformer.models.vit_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD
 from timesformer.models.helpers import load_pretrained
 from timesformer.models.vit_utils import DropPath, to_2tuple, trunc_normal_

requirements.txt

@@ -18,11 +18,10 @@ scikit_learn
 scipy
 sentencepiece
 setuptools
-skimage
-slowfast
+scikit-image
 submitit
 tensorflow
-timm
+timm==0.6.12
 torch
 torchaudio
 torchvision
@@ -32,6 +31,12 @@ torchtyping
 tqdm
 ruamel.yaml
 
+
+git+https://github.com/facebookresearch/fvcore
+simplejson
+psutil
+
+
 # # accelerate==0.11.0
 # apex==0.9.10.dev0
 # av==10.0.0