Commit
•
2f2b2b1
1
Parent(s):
17b3a43
Add chat template examples
Browse files
README.md
CHANGED
@@ -6,6 +6,10 @@ datasets:
|
|
6 |
pipeline_tag: image-to-text
|
7 |
inference: false
|
8 |
arxiv: 2304.08485
|
|
|
|
|
|
|
|
|
9 |
---
|
10 |
# LLaVA Model Card
|
11 |
|
@@ -47,9 +51,21 @@ import requests
|
|
47 |
model_id = "llava-hf/llava-1.5-7b-hf"
|
48 |
pipe = pipeline("image-to-text", model=model_id)
|
49 |
url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg"
|
50 |
-
|
51 |
image = Image.open(requests.get(url, stream=True).raw)
|
52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
53 |
|
54 |
outputs = pipe(image, prompt=prompt, generate_kwargs={"max_new_tokens": 200})
|
55 |
print(outputs)
|
@@ -68,10 +84,6 @@ import torch
|
|
68 |
from transformers import AutoProcessor, LlavaForConditionalGeneration
|
69 |
|
70 |
model_id = "llava-hf/llava-1.5-7b-hf"
|
71 |
-
|
72 |
-
prompt = "USER: <image>\nWhat are these?\nASSISTANT:"
|
73 |
-
image_file = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
74 |
-
|
75 |
model = LlavaForConditionalGeneration.from_pretrained(
|
76 |
model_id,
|
77 |
torch_dtype=torch.float16,
|
@@ -80,7 +92,21 @@ model = LlavaForConditionalGeneration.from_pretrained(
|
|
80 |
|
81 |
processor = AutoProcessor.from_pretrained(model_id)
|
82 |
|
|
|
|
|
|
|
|
|
83 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
84 |
raw_image = Image.open(requests.get(image_file, stream=True).raw)
|
85 |
inputs = processor(prompt, raw_image, return_tensors='pt').to(0, torch.float16)
|
86 |
|
|
|
6 |
pipeline_tag: image-to-text
|
7 |
inference: false
|
8 |
arxiv: 2304.08485
|
9 |
+
license: llama2
|
10 |
+
tags:
|
11 |
+
- vision
|
12 |
+
- image-text-to-text
|
13 |
---
|
14 |
# LLaVA Model Card
|
15 |
|
|
|
51 |
model_id = "llava-hf/llava-1.5-7b-hf"
|
52 |
pipe = pipeline("image-to-text", model=model_id)
|
53 |
url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg"
|
|
|
54 |
image = Image.open(requests.get(url, stream=True).raw)
|
55 |
+
|
56 |
+
# Define a chat histiry and use `apply_chat_template` to get correctly formatted prompt
|
57 |
+
# Each value in "content" has to be a list of dicts with types ("text", "image")
|
58 |
+
conversation = [
|
59 |
+
{
|
60 |
+
|
61 |
+
"role": "user",
|
62 |
+
"content": [
|
63 |
+
{"type": "text", "text": "What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud"},
|
64 |
+
{"type": "image"},
|
65 |
+
],
|
66 |
+
},
|
67 |
+
]
|
68 |
+
prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
|
69 |
|
70 |
outputs = pipe(image, prompt=prompt, generate_kwargs={"max_new_tokens": 200})
|
71 |
print(outputs)
|
|
|
84 |
from transformers import AutoProcessor, LlavaForConditionalGeneration
|
85 |
|
86 |
model_id = "llava-hf/llava-1.5-7b-hf"
|
|
|
|
|
|
|
|
|
87 |
model = LlavaForConditionalGeneration.from_pretrained(
|
88 |
model_id,
|
89 |
torch_dtype=torch.float16,
|
|
|
92 |
|
93 |
processor = AutoProcessor.from_pretrained(model_id)
|
94 |
|
95 |
+
# Define a chat histiry and use `apply_chat_template` to get correctly formatted prompt
|
96 |
+
# Each value in "content" has to be a list of dicts with types ("text", "image")
|
97 |
+
conversation = [
|
98 |
+
{
|
99 |
|
100 |
+
"role": "user",
|
101 |
+
"content": [
|
102 |
+
{"type": "text", "text": "What are these?"},
|
103 |
+
{"type": "image"},
|
104 |
+
],
|
105 |
+
},
|
106 |
+
]
|
107 |
+
prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
|
108 |
+
|
109 |
+
image_file = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
110 |
raw_image = Image.open(requests.get(image_file, stream=True).raw)
|
111 |
inputs = processor(prompt, raw_image, return_tensors='pt').to(0, torch.float16)
|
112 |
|