wenhu commited on
Commit
e9afa98
1 Parent(s): bb12c3f

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +7 -3
README.md CHANGED
@@ -41,6 +41,7 @@ Our model can outperform the existing baselines by a huge margin.
41
  First you can clone our github
42
  ```bash
43
  git clone https://github.com/TIGER-AI-Lab/VLM2Vec.git
 
44
  ```
45
 
46
  Then you can enter the directory to run the following command.
@@ -53,7 +54,7 @@ from PIL import Image
53
  import numpy as np
54
 
55
  model_args = ModelArguments(
56
- model_name='microsoft/Phi-3.5-vision-instruct',
57
  pooling='last',
58
  normalize=True,
59
  lora=True,
@@ -74,17 +75,19 @@ inputs = processor('<|image_1|> Represent the given image with the following que
74
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
75
  qry_output = model(qry=inputs)["qry_reps"]
76
 
77
- ## Compute the similarity;
78
  string = 'A cat and a dog'
79
  inputs = processor(string)
80
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
81
  tgt_output = model(tgt=inputs)["tgt_reps"]
82
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
 
83
 
 
84
  inputs = processor(string)
85
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
86
  tgt_output = model(tgt=inputs)["tgt_reps"]
87
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
 
88
 
89
  # Text -> Image
90
  inputs = processor('Find me an everyday image that matches the given caption: A cat and a dog.',)
@@ -92,10 +95,11 @@ inputs = {key: value.to('cuda') for key, value in inputs.items()}
92
  qry_output = model(qry=inputs)["qry_reps"]
93
 
94
  string = '<|image_1|> Represent the given image.'
95
- inputs = processor(string, [Image.open('figures/example.jpg')]])
96
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
97
  tgt_output = model(tgt=inputs)["tgt_reps"]
98
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
 
99
  ```
100
 
101
  ## Citation
 
41
  First you can clone our github
42
  ```bash
43
  git clone https://github.com/TIGER-AI-Lab/VLM2Vec.git
44
+ pip -r requirements.txt
45
  ```
46
 
47
  Then you can enter the directory to run the following command.
 
54
  import numpy as np
55
 
56
  model_args = ModelArguments(
57
+ model_name='microsoft/Phi-3.5-vision-instruct',
58
  pooling='last',
59
  normalize=True,
60
  lora=True,
 
75
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
76
  qry_output = model(qry=inputs)["qry_reps"]
77
 
 
78
  string = 'A cat and a dog'
79
  inputs = processor(string)
80
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
81
  tgt_output = model(tgt=inputs)["tgt_reps"]
82
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
83
+ ## A cat and a dog = tensor([[0.2969]], device='cuda:0', dtype=torch.bfloat16)
84
 
85
+ string = 'A cat and a tiger'
86
  inputs = processor(string)
87
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
88
  tgt_output = model(tgt=inputs)["tgt_reps"]
89
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
90
+ ## A cat and a tiger = tensor([[0.2080]], device='cuda:0', dtype=torch.bfloat16)
91
 
92
  # Text -> Image
93
  inputs = processor('Find me an everyday image that matches the given caption: A cat and a dog.',)
 
95
  qry_output = model(qry=inputs)["qry_reps"]
96
 
97
  string = '<|image_1|> Represent the given image.'
98
+ inputs = processor(string, [Image.open('figures/example.jpg')])
99
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
100
  tgt_output = model(tgt=inputs)["tgt_reps"]
101
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
102
+ ## <|image_1|> Represent the given image. = tensor([[0.3105]], device='cuda:0', dtype=torch.bfloat16)
103
  ```
104
 
105
  ## Citation