Repository: embeddings-benchmark/mteb

Issue #1456: [mieb] TIGER-Lab/VLM2Vec-LoRA fails with dtype issue
Created: 2024-11-14T18:34:20Z
ERROR:mteb.evaluation.MTEB:Error while evaluating BLINKIT2IMultiChoice: self and mat2 must have the same dtype,
 but got BFloat16 and Float                                                                                    
Traceback (most recent call last):                                                                             
  File "/data/niklas/mieb/mteb/scripts/run_mieb.py", line 91, in <module>                                      
    results = evaluation.run(model, output_folder="/data/niklas/mieb/results-mieb-final", batch_size=32)       
  File "/data/niklas/mieb/mteb/mteb/evaluation/MTEB.py", line 464, in run                                      
    raise e                                                                                                    
  File "/data/niklas/mieb/mteb/mteb/evaluation/MTEB.py", line 425, in run                                      
    results, tick, tock = self._run_eval(                                                                      
  File "/data/niklas/mieb/mteb/mteb/evaluation/MTEB.py", line 300, in _run_eval                                
    results = task.evaluate(                                                                                   
  File "/data/niklas/mieb/mteb/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py", line 273, in evaluate        
    scores[hf_subset] = self._evaluate_subset(                                                                 
  File "/data/niklas/mieb/mteb/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py", line 282, in _evaluate_subset
    results = retriever(corpus, queries, relevant_docs)                                                        
  File "/data/niklas/mieb/mteb/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py", line 298, in _
_call__                                                                                                        
    return self.retriever.search(                                                                              
  File "/data/niklas/mieb/mteb/mteb/evaluation/evaluators/Image/Any2AnyMultiChoiceEvaluator.py", line 145, in s
earch                                                                                                          
    query_embeddings = self.model.get_fused_embeddings(                                                        
  File "/data/niklas/mieb/mteb/mteb/models/vlm2vec_models.py", line 274, in get_fused_embeddings               
    outputs = self.encode_input(inputs)                                                                        
  File "/data/niklas/mieb/mteb/mteb/models/vlm2vec_models.py", line 91, in encode_input                        
    hidden_states = self.mdl(**input, return_dict=True, output_hidden_states=True)                             
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrappe
d_call_impl                                                                                                    
    return self._call_impl(*args, **kwargs)                                                                    
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_i
mpl                                                                                                            
    return forward_call(*args, **kwargs)                                                                       
  File "/data/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb615
1705d1ee73c25a80ca/modeling_phi3_v.py", line 1603, in forward                                                  
    outputs = self.model(                                                                                      
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrappe
d_call_impl                                                                                                    
    return self._call_impl(*args, **kwargs)                                                                    
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_i
mpl                                                                                                            
    return forward_call(*args, **kwargs)                                                                       
  File "/data/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb615
1705d1ee73c25a80ca/modeling_phi3_v.py", line 1431, in forward                                                  
    inputs_embeds = self.vision_embed_tokens(input_ids, pixel_values=pixel_values, image_sizes=image_sizes)    
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, i[23/1864]
d_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_i
mpl
    return forward_call(*args, **kwargs)
  File "/data/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb615
1705d1ee73c25a80ca/modeling_phi3_v.py", line 237, in forward
    img_features = self.get_img_features(pixel_values.flatten(0, 1)).reshape(
  File "/data/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb615
1705d1ee73c25a80ca/modeling_phi3_v.py", line 212, in get_img_features
    img_processor_output = self.img_processor(img_embeds, output_hidden_states=True)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrappe
d_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_i
mpl
    return forward_call(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 1
171, in forward
    return self.vision_model(
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrappe
d_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_i
mpl
    return forward_call(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 1
097, in forward
    encoder_outputs = self.encoder(
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrappe
d_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_i
mpl
    return forward_call(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 8
77, in forward
    layer_outputs = encoder_layer(
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrappe
d_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
    return forward_call(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/transformers/models/clip/modeling_clip.py", line 6
08, in forward
    hidden_states, attn_weights = self.self_attn(
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrappe
d_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_i
mpl
    return forward_call(*args, **kwargs)
  File "/data/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb615
1705d1ee73c25a80ca/modeling_phi3_v.py", line 101, in forward
    query_states = self.q_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrappe
d_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_i
mpl
    return forward_call(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/linear.py", line 117, in forward
    return F.linear(input, self.weight, self.bias)
RuntimeError: self and mat2 must have the same dtype, but got BFloat16 and Float

Duplicate issue #1377: [mieb] TIGER-Lab/VLM2Vec-LoRA fails
Created: 2024-11-04T03:14:00Z
INFO:mteb.cli:Running with parameters: Namespace(model='TIGER-Lab/VLM2Vec-LoRA', task_types=None, categories=None, tasks=['BLINKIT2IRetrieval'], languages=None, benchmarks=None, device=None, output_folder='/data/niklas/mieb/results-mieb-final', verbosity=2, co2_tracker=True, eval_splits=None, model_revision=None, batch_size=4, overwrite=False, save_predictions=False, func=<function run at 0x7feee99e23b0>)
You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:04<00:04,  4.68s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:07<00:00,  3.42s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:07<00:00,  3.61s/it]
/env/lib/conda/gritkto4/lib/python3.10/site-packages/transformers/models/auto/image_processing_auto.py:520: FutureWarning: The image_processor_class argument is deprecated and will be removed in v4.42. Please use `slow_image_processor_class`, or `fast_image_processor_class` instead
  warnings.warn(
INFO:mteb.evaluation.MTEB:

## Evaluating 1 tasks:
─────────────────────────────── Selected tasks  ────────────────────────────────
Any2AnyRetrieval
    - BLINKIT2IRetrieval, it2i


INFO:mteb.evaluation.MTEB:

********************** Evaluating BLINKIT2IRetrieval **********************
INFO:mteb.evaluation.MTEB:Loading dataset for BLINKIT2IRetrieval
INFO:mteb.abstasks.Image.AbsTaskAny2AnyRetrieval:Loading Corpus...
INFO:mteb.abstasks.Image.AbsTaskAny2AnyRetrieval:Loaded 804 TEST Documents.
INFO:mteb.abstasks.Image.AbsTaskAny2AnyRetrieval:Doc Example: {'id': 'val_Art_Style_1_2', 'modality': 'image', 'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=1382x1718 at 0x7FF21EA29540>}
INFO:mteb.abstasks.Image.AbsTaskAny2AnyRetrieval:Loading Queries...

Map:   0%|          | 0/402 [00:00<?, ? examples/s]
Map: 100%|██████████| 402/402 [00:00<00:00, 43397.17 examples/s]
INFO:mteb.abstasks.Image.AbsTaskAny2AnyRetrieval:Loaded 402 TEST Queries.
INFO:mteb.abstasks.Image.AbsTaskAny2AnyRetrieval:Query Example: {'id': 'val_Art_Style_1_1', 'modality': 'image,text', 'text': 'Which image shares the same style as the reference image?', 'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=1382x2057 at 0x7FF21E967A60>}
INFO:mteb.abstasks.Image.AbsTaskAny2AnyRetrieval:Subset: default
INFO:mteb.evaluation.evaluators.Image.Any2AnyRetrievalEvaluator:Encoding Queries.

  0%|          | 0/101 [00:00<?, ?it/s]
  0%|          | 0/101 [00:00<?, ?it/s]
ERROR:mteb.evaluation.MTEB:Error while evaluating BLINKIT2IRetrieval: expected mat1 and mat2 to have the same dtype, but got: c10::BFloat16 != float
Traceback (most recent call last):
  File "/env/lib/conda/gritkto4/bin/mteb", line 8, in <module>
    sys.exit(main())
  File "/data/niklas/mieb/mteb/mteb/cli.py", line 387, in main
    args.func(args)
  File "/data/niklas/mieb/mteb/mteb/cli.py", line 145, in run
    eval.run(
  File "/data/niklas/mieb/mteb/mteb/evaluation/MTEB.py", line 464, in run
    raise e
  File "/data/niklas/mieb/mteb/mteb/evaluation/MTEB.py", line 412, in run
    results, tick, tock = self._run_eval(
  File "/data/niklas/mieb/mteb/mteb/evaluation/MTEB.py", line 300, in _run_eval
    results = task.evaluate(
  File "/data/niklas/mieb/mteb/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py", line 269, in evaluate
    scores[hf_subset] = self._evaluate_subset(
  File "/data/niklas/mieb/mteb/mteb/abstasks/Image/AbsTaskAny2AnyRetrieval.py", line 278, in _evaluate_subset
    results = retriever(corpus, queries)
  File "/data/niklas/mieb/mteb/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py", line 290, in __call__
    return self.retriever.search(
  File "/data/niklas/mieb/mteb/mteb/evaluation/evaluators/Image/Any2AnyRetrievalEvaluator.py", line 144, in search
    query_embeddings = self.model.get_fused_embeddings(
  File "/data/niklas/mieb/mteb/mteb/models/vlm2vec_models.py", line 243, in get_fused_embeddings
    text_embeddings = self.get_text_embeddings(texts, batch_size)
  File "/data/niklas/mieb/mteb/mteb/models/vlm2vec_models.py", line 223, in get_text_embeddings
    text_outputs = self.encode_input(inputs)
  File "/data/niklas/mieb/mteb/mteb/models/vlm2vec_models.py", line 91, in encode_input
    hidden_states = self.mdl(**input, return_dict=True, output_hidden_states=True)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py", line 1603, in forward
    outputs = self.model(
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py", line 1479, in forward
    layer_outputs = decoder_layer(
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py", line 1177, in forward
    attn_outputs, self_attn_weights, present_key_value = self.self_attn(
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/huggingface/modules/transformers_modules/microsoft/Phi-3.5-vision-instruct/4a0d683eba9f1d0cbfb6151705d1ee73c25a80ca/modeling_phi3_v.py", line 765, in forward
    qkv = self.qkv_proj(hidden_states)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
    return forward_call(*args, **kwargs)
  File "/env/lib/conda/gritkto4/lib/python3.10/site-packages/torch/nn/modules/linear.py", line 117, in forward
    return F.linear(input, self.weight, self.bias)
RuntimeError: expected mat1 and mat2 to have the same dtype, but got: c10::BFloat16 != float

Closing comment (2024-11-15T11:02:43Z): Issue persists. Investigating. Duplicate: #1456