Commit
•
b2a5d3d
1
Parent(s):
15c7469
Update README.md
Browse files
README.md
CHANGED
@@ -16,6 +16,18 @@ from PIL import Image
|
|
16 |
import librosa
|
17 |
import requests
|
18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
19 |
def prepare_dataset(messages, images: List[str] = None, audio: List[str] = None, sr = 16000):
|
20 |
|
21 |
if images is not None:
|
@@ -73,18 +85,6 @@ def prepare_dataset(messages, images: List[str] = None, audio: List[str] = None,
|
|
73 |
|
74 |
return outputs
|
75 |
|
76 |
-
model = MM_LLMs.from_pretrained(
|
77 |
-
'mesolitica/malaysian-tinyllama-1.1b-mmmmodal',
|
78 |
-
flash_attention = True,
|
79 |
-
dtype = torch.bfloat16,
|
80 |
-
torch_dtype = torch.bfloat16
|
81 |
-
)
|
82 |
-
_ = model.cuda()
|
83 |
-
|
84 |
-
image_processor = AutoProcessor.from_pretrained('google/siglip-base-patch16-384')
|
85 |
-
audio_processor = AutoProcessor.from_pretrained('mesolitica/malaysian-whisper-small')
|
86 |
-
tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-tinyllama-1.1b-mmmmodal')
|
87 |
-
|
88 |
with open('Persian-cat-breed.jpg', 'wb') as fopen:
|
89 |
fopen.write(requests.get('https://cdn.beautifulnara.net/wp-content/uploads/2017/12/10201620/Persian-cat-breed.jpg').content)
|
90 |
|
|
|
16 |
import librosa
|
17 |
import requests
|
18 |
|
19 |
+
model = MM_LLMs.from_pretrained(
|
20 |
+
'mesolitica/malaysian-tinyllama-1.1b-mmmmodal',
|
21 |
+
flash_attention = True,
|
22 |
+
dtype = torch.bfloat16,
|
23 |
+
torch_dtype = torch.bfloat16
|
24 |
+
)
|
25 |
+
_ = model.cuda()
|
26 |
+
|
27 |
+
image_processor = AutoProcessor.from_pretrained('google/siglip-base-patch16-384')
|
28 |
+
audio_processor = AutoProcessor.from_pretrained('mesolitica/malaysian-whisper-small')
|
29 |
+
tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-tinyllama-1.1b-mmmmodal')
|
30 |
+
|
31 |
def prepare_dataset(messages, images: List[str] = None, audio: List[str] = None, sr = 16000):
|
32 |
|
33 |
if images is not None:
|
|
|
85 |
|
86 |
return outputs
|
87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
88 |
with open('Persian-cat-breed.jpg', 'wb') as fopen:
|
89 |
fopen.write(requests.get('https://cdn.beautifulnara.net/wp-content/uploads/2017/12/10201620/Persian-cat-breed.jpg').content)
|
90 |
|