RaushanTurganbay HF staff commited on
Commit
2f2b2b1
1 Parent(s): 17b3a43

Add chat template examples

Browse files
Files changed (1) hide show
  1. README.md +32 -6
README.md CHANGED
@@ -6,6 +6,10 @@ datasets:
6
  pipeline_tag: image-to-text
7
  inference: false
8
  arxiv: 2304.08485
 
 
 
 
9
  ---
10
  # LLaVA Model Card
11
 
@@ -47,9 +51,21 @@ import requests
47
  model_id = "llava-hf/llava-1.5-7b-hf"
48
  pipe = pipeline("image-to-text", model=model_id)
49
  url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg"
50
-
51
  image = Image.open(requests.get(url, stream=True).raw)
52
- prompt = "USER: <image>\nWhat does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud\nASSISTANT:"
 
 
 
 
 
 
 
 
 
 
 
 
 
53
 
54
  outputs = pipe(image, prompt=prompt, generate_kwargs={"max_new_tokens": 200})
55
  print(outputs)
@@ -68,10 +84,6 @@ import torch
68
  from transformers import AutoProcessor, LlavaForConditionalGeneration
69
 
70
  model_id = "llava-hf/llava-1.5-7b-hf"
71
-
72
- prompt = "USER: <image>\nWhat are these?\nASSISTANT:"
73
- image_file = "http://images.cocodataset.org/val2017/000000039769.jpg"
74
-
75
  model = LlavaForConditionalGeneration.from_pretrained(
76
  model_id,
77
  torch_dtype=torch.float16,
@@ -80,7 +92,21 @@ model = LlavaForConditionalGeneration.from_pretrained(
80
 
81
  processor = AutoProcessor.from_pretrained(model_id)
82
 
 
 
 
 
83
 
 
 
 
 
 
 
 
 
 
 
84
  raw_image = Image.open(requests.get(image_file, stream=True).raw)
85
  inputs = processor(prompt, raw_image, return_tensors='pt').to(0, torch.float16)
86
 
 
6
  pipeline_tag: image-to-text
7
  inference: false
8
  arxiv: 2304.08485
9
+ license: llama2
10
+ tags:
11
+ - vision
12
+ - image-text-to-text
13
  ---
14
  # LLaVA Model Card
15
 
 
51
  model_id = "llava-hf/llava-1.5-7b-hf"
52
  pipe = pipeline("image-to-text", model=model_id)
53
  url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg"
 
54
  image = Image.open(requests.get(url, stream=True).raw)
55
+
56
+ # Define a chat histiry and use `apply_chat_template` to get correctly formatted prompt
57
+ # Each value in "content" has to be a list of dicts with types ("text", "image")
58
+ conversation = [
59
+ {
60
+
61
+ "role": "user",
62
+ "content": [
63
+ {"type": "text", "text": "What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud"},
64
+ {"type": "image"},
65
+ ],
66
+ },
67
+ ]
68
+ prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
69
 
70
  outputs = pipe(image, prompt=prompt, generate_kwargs={"max_new_tokens": 200})
71
  print(outputs)
 
84
  from transformers import AutoProcessor, LlavaForConditionalGeneration
85
 
86
  model_id = "llava-hf/llava-1.5-7b-hf"
 
 
 
 
87
  model = LlavaForConditionalGeneration.from_pretrained(
88
  model_id,
89
  torch_dtype=torch.float16,
 
92
 
93
  processor = AutoProcessor.from_pretrained(model_id)
94
 
95
+ # Define a chat histiry and use `apply_chat_template` to get correctly formatted prompt
96
+ # Each value in "content" has to be a list of dicts with types ("text", "image")
97
+ conversation = [
98
+ {
99
 
100
+ "role": "user",
101
+ "content": [
102
+ {"type": "text", "text": "What are these?"},
103
+ {"type": "image"},
104
+ ],
105
+ },
106
+ ]
107
+ prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
108
+
109
+ image_file = "http://images.cocodataset.org/val2017/000000039769.jpg"
110
  raw_image = Image.open(requests.get(image_file, stream=True).raw)
111
  inputs = processor(prompt, raw_image, return_tensors='pt').to(0, torch.float16)
112