wenhu commited on
Commit
bb12c3f
1 Parent(s): 7f0d860

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +10 -13
README.md CHANGED
@@ -69,33 +69,30 @@ processor = AutoProcessor.from_pretrained(
69
  num_crops=4,
70
  )
71
 
72
- inputs = processor(
73
- '<|image_1|> Represent the given image with the following question: What is in the image',
74
- [Image.open('figures/example.jpg')])
75
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
76
  qry_output = model(qry=inputs)["qry_reps"]
77
 
78
- # Compute the similarity;
79
  string = 'A cat and a dog'
80
- inputs = processor(string, None, return_tensors="pt")
81
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
82
  tgt_output = model(tgt=inputs)["tgt_reps"]
83
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
84
 
85
- string = 'A cat and a tiger'
86
- inputs = processor(string, None, return_tensors="pt")
87
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
88
  tgt_output = model(tgt=inputs)["tgt_reps"]
89
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
90
 
91
- string = 'A pig'
92
- inputs = processor(string, None, return_tensors="pt")
93
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
94
- tgt_output = model(tgt=inputs)["tgt_reps"]
95
- print(string, '=', model.compute_similarity(qry_output, tgt_output))
96
 
97
- string = 'a flight'
98
- inputs = processor(string, None, return_tensors="pt")
99
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
100
  tgt_output = model(tgt=inputs)["tgt_reps"]
101
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
 
69
  num_crops=4,
70
  )
71
 
72
+ # Image + Text -> Text
73
+ inputs = processor('<|image_1|> Represent the given image with the following question: What is in the image', [Image.open('figures/example.jpg')])
 
74
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
75
  qry_output = model(qry=inputs)["qry_reps"]
76
 
77
+ ## Compute the similarity;
78
  string = 'A cat and a dog'
79
+ inputs = processor(string)
80
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
81
  tgt_output = model(tgt=inputs)["tgt_reps"]
82
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
83
 
84
+ inputs = processor(string)
 
85
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
86
  tgt_output = model(tgt=inputs)["tgt_reps"]
87
  print(string, '=', model.compute_similarity(qry_output, tgt_output))
88
 
89
+ # Text -> Image
90
+ inputs = processor('Find me an everyday image that matches the given caption: A cat and a dog.',)
91
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
92
+ qry_output = model(qry=inputs)["qry_reps"]
 
93
 
94
+ string = '<|image_1|> Represent the given image.'
95
+ inputs = processor(string, [Image.open('figures/example.jpg')]])
96
  inputs = {key: value.to('cuda') for key, value in inputs.items()}
97
  tgt_output = model(tgt=inputs)["tgt_reps"]
98
  print(string, '=', model.compute_similarity(qry_output, tgt_output))