Onnx export Data2vexAudio ValueError: Model and config inputs doesn't match
Author: xiadingZ
Pull request: no



- data2vec: @patrickvonplaten, @anton-l @edugp




The problem arises when using:
* [ ] my own modified scripts: (give details below)
```
from typing import Mapping, OrderedDict
from transformers.onnx import OnnxConfig

from transformers import AutoConfig
from pathlib import Path
from transformers.onnx import export
from transformers import AutoTokenizer, AutoModel

class Data2VecAudioOnnxConfig(OnnxConfig):
    @property
    def inputs(self):
        return OrderedDict(
            [
                ("input_values", {0: "batch", 1: "sequence"}),
                ("attention_mask", {0: "batch", 1: "sequence"}),
            ]
        )
    
config = AutoConfig.from_pretrained("facebook/data2vec-audio-base-960h")
onnx_config = Data2VecAudioOnnxConfig(config)



onnx_path = Path("facebook/data2vec-audio-base-960h")
model_ckpt = "facebook/data2vec-audio-base-960h"
base_model = AutoModel.from_pretrained(model_ckpt)
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)

onnx_inputs, onnx_outputs = export(tokenizer, base_model, onnx_config, onnx_config.default_onnx_opset, onnx_path)
```

errors throws
```
ValueError                                Traceback (most recent call last)
/var/folders/2t/0w65vdjs2m32w5mmzzgtqrhw0000gn/T/ipykernel_59977/667985886.py in <module>
     27 tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
     28 
---> 29 onnx_inputs, onnx_outputs = export(tokenizer, base_model, onnx_config, onnx_config.default_onnx_opset, onnx_path)

~/miniconda3/lib/python3.9/site-packages/transformers/onnx/convert.py in export(tokenizer, model, config, opset, output)
    255 
    256     if is_torch_available() and issubclass(type(model), PreTrainedModel):
--> 257         return export_pytorch(tokenizer, model, config, opset, output)
    258     elif is_tf_available() and issubclass(type(model), TFPreTrainedModel):
    259         return export_tensorflow(tokenizer, model, config, opset, output)

~/miniconda3/lib/python3.9/site-packages/transformers/onnx/convert.py in export_pytorch(tokenizer, model, config, opset, output)
    112 
    113             if not inputs_match:
--> 114                 raise ValueError("Model and config inputs doesn't match")
    115 
    116             config.patch_ops()

ValueError: Model and config inputs doesn't match
​


```


Comments:
Wow very cool to see that you guys are already working on Onnx and Data2VecAudio :partying_face: 

Pinging @anton-l and @lewtun here :-) 

Hi @xiadingZ this is cool indeed! 

I think the error message is coming from the fact that the base `generate_dummy_inputs()` method of `OnnxConfig` doesn't return inputs that match those the model expects. What I suggest is overriding this method to produce the desired inputs - you can check out the configuration files of models like LayoutLM to see what we've done for these type of modalities

@lewtun thanks very much! I succeed exporting `Data2vecAudio` to `onnx`. But `onnxruntime` can't load this model because of `shape inference`,  can you give some suggestion?

code
```
from collections import Mapping, OrderedDict
from transformers.onnx import OnnxConfig

from transformers import AutoConfig,PretrainedConfig, PreTrainedTokenizer, TensorType
from pathlib import Path
from transformers.onnx import export
from transformers import AutoTokenizer, AutoModel
import torch

class Data2VecAudioOnnxConfig(OnnxConfig):
    @property
    def inputs(self):
        return OrderedDict(
            [
                ("input_values", {0: "batch", 1: "sequence"}),
                ("attention_mask", {0: "batch", 1: "sequence"}),
            ]
        )
    
    def generate_dummy_inputs(
        self,
        tokenizer: PreTrainedTokenizer,
        batch_size: int = -1,
        seq_length: int = 151680,
        is_pair: bool = False,
        framework = None,
    ):
        """
        Generate inputs to provide to the ONNX exporter for the specific framework
        Args:
            tokenizer: The tokenizer associated with this model configuration
            batch_size: The batch size (int) to export the model for (-1 means dynamic axis)
            seq_length: The sequence length (int) to export the model for (-1 means dynamic axis)
            is_pair: Indicate if the input is a pair (sentence 1, sentence 2)
            framework: The framework (optional) the tokenizer will generate tensor for
        Returns:
            Mapping[str, Tensor] holding the kwargs to provide to the model's forward function
        """

        input_dict = super().generate_dummy_inputs(tokenizer, batch_size, seq_length, is_pair, framework)


        if not framework == TensorType.PYTORCH:
            raise NotImplementedError("Exporting Data2VecAudio to ONNX is currently only supported for PyTorch.")


        batch_size, seq_length = input_dict["input_ids"].shape
        input_dict["input_values"] = input_dict["input_ids"].to(torch.float)
        input_dict['attention_mask'] = input_dict['attention_mask'].to(torch.int)
        del input_dict['input_ids']
        return input_dict

model_ckpt = "data2vec_model"
base_model = AutoModel.from_pretrained(model_ckpt)
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
config = AutoConfig.from_pretrained("data2vec_model")
onnx_config = Data2VecAudioOnnxConfig(config)
onnx_path = Path("model.onnx")
onnx_inputs, onnx_outputs = export(tokenizer, base_model, onnx_config, 14, onnx_path)


import onnxruntime

ort_session = onnxruntime.InferenceSession("model.onnx",providers=['TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider'])
print("Exported model has been tested with ONNXRuntime, and the result looks good!")
```

errors
```
---------------------------------------------------------------------------
RuntimeException                          Traceback (most recent call last)
<ipython-input-6-5b62b3b0329e> in <module>
      1 import onnxruntime
      2 
----> 3 ort_session = onnxruntime.InferenceSession("model.onnx",providers=['TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider'])
      4 print("Exported model has been tested with ONNXRuntime, and the result looks good!")

/opt/conda/lib/python3.6/site-packages/onnxruntime/capi/onnxruntime_inference_collection.py in __init__(self, path_or_bytes, sess_options, providers, provider_options, **kwargs)
    333 
    334         try:
--> 335             self._create_inference_session(providers, provider_options, disabled_optimizers)
    336         except ValueError:
    337             if self._enable_fallback:

/opt/conda/lib/python3.6/site-packages/onnxruntime/capi/onnxruntime_inference_collection.py in _create_inference_session(self, providers, provider_options, disabled_optimizers)
    377 
    378         # initialize the C++ InferenceSession
--> 379         sess.initialize_session(providers, provider_options, disabled_optimizers)
    380 
    381         self._sess = sess

RuntimeException: [ONNXRuntimeError] : 6 : RUNTIME_EXCEPTION : Exception during initialization: /onnxruntime_src/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:925 SubGraphCollection_t 
onnxruntime::TensorrtExecutionProvider::GetSupportedList(SubGraphCollection_t, int, int, const 
onnxruntime::GraphViewer&, bool*) const [ONNXRuntimeError] : 1 : FAIL : TensorRT input: 573 has no shape specified. 
Please run shape inference on the onnx model first. Details can be found in 
https://www.onnxruntime.ai/docs/reference/execution-providers/TensorRT-ExecutionProvider.html#shape-inference-for-
tensorrt-subgraphs
```

Hi @xiadingZ this looks like an issue with the TensorRT backend not supporting all operators in the ONNX graph: https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#shape-inference-for-tensorrt-subgraphs

To eliminate other causes, could you run the inference session with just `CPUExecutionProvider`?

yes, you are right @lewtun .  inference session can succeed with `'CUDAExecutionProvider', 'CPUExecutionProvider'`.
can you give some suggestion about how to eliminate this error with `TensorrtExecutionProvider `,  such as find the unsupported operator and write the converter to tensor-rt

Hi @xiadingZ my suggestion would be to try feeding your exported ONNX model through this script: https://github.com/microsoft/onnxruntime/blob/master/onnxruntime/python/tools/symbolic_shape_infer.py

That should help infer the tensor shapes, but please report back to let us know if it works or not. We might have to take a closer look at the problematic ops in the graph to enable support for TensorRT out of the box

Hi   @lewtun.   It throws error:
```
Traceback (most recent call last):
  File "onnxruntime/onnxruntime/python/tools/symbolic_shape_infer.py", line 2117, in <module>
    args.guess_output_rank, args.verbose)
  File "onnxruntime/onnxruntime/python/tools/symbolic_shape_infer.py", line 2083, in infer_shapes
    raise Exception("Incomplete symbolic shape inference")
Exception: Incomplete symbolic shape inference
```

You can try with this script:
```
from collections import Mapping, OrderedDict
from transformers.onnx import OnnxConfig

from transformers import AutoConfig,PretrainedConfig, PreTrainedTokenizer, TensorType
from pathlib import Path
from transformers.onnx import export
from transformers import AutoTokenizer, AutoModel
import torch
import onnxruntime
from transformers import Wav2Vec2Processor, Data2VecAudioModel, Data2VecAudioConfig

class Data2VecAudioOnnxConfig(OnnxConfig):
    @property
    def inputs(self):
        return OrderedDict(
            [
                ("input_values", {0: "batch", 1: "sequence"}),
                ("attention_mask", {0: "batch", 1: "sequence"}),
            ]
        )
    
    def generate_dummy_inputs(
        self,
        tokenizer: PreTrainedTokenizer,
        batch_size: int = -1,
        seq_length: int = 151680,
        is_pair: bool = False,
        framework = None,
    ):
        """
        Generate inputs to provide to the ONNX exporter for the specific framework
        Args:
            tokenizer: The tokenizer associated with this model configuration
            batch_size: The batch size (int) to export the model for (-1 means dynamic axis)
            seq_length: The sequence length (int) to export the model for (-1 means dynamic axis)
            is_pair: Indicate if the input is a pair (sentence 1, sentence 2)
            framework: The framework (optional) the tokenizer will generate tensor for
        Returns:
            Mapping[str, Tensor] holding the kwargs to provide to the model's forward function
        """

        input_dict = super().generate_dummy_inputs(tokenizer, batch_size, seq_length, is_pair, framework)


        if not framework == TensorType.PYTORCH:
            raise NotImplementedError("Exporting Data2VecAudio to ONNX is currently only supported for PyTorch.")


        batch_size, seq_length = input_dict["input_ids"].shape
        input_dict["input_values"] = input_dict["input_ids"].to(torch.float)
        input_dict['attention_mask'] = input_dict['attention_mask'].to(torch.int)
        del input_dict['input_ids']
        return input_dict
    


#model_ckpt = "data2vec_model"
model_ckpt = "facebook/data2vec-audio-base-960h"
base_model = AutoModel.from_pretrained(model_ckpt)
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
config = AutoConfig.from_pretrained("facebook/data2vec-audio-base-960h")
onnx_config = Data2VecAudioOnnxConfig(config)
onnx_path = Path("model.onnx")
onnx_inputs, onnx_outputs = export(tokenizer, base_model, onnx_config, 14, onnx_path)
```

Hello, @lewtun . I try with this command and it succeed, doesn't throw error.
```
python onnxruntime/onnxruntime/python/tools/symbolic_shape_infer.py --input model.onnx --output new_model.onnx --auto_merge
```

but I run onnxruntime infer with `CUDAExecutionProvider`, it throws `Ops error`
```
import soundfile as sf
import onnxruntime

# sample_rate is 16000
singnal, sample_rate = sf.read('test.flac')

ort_session = onnxruntime.InferenceSession("model.onnx",providers=['CUDAExecutionProvider'])
inputs_onnx = {k: v.cpu().detach().numpy() for k, v in inputs.items()}
ort_outs = ort_session.run(None, inputs_onnx)

print("Exported model has been tested with ONNXRuntime, and the result looks good!")
```

error is:
```
2022-04-19 14:31:13.232096464 [E:onnxruntime:, sequential_executor.cc:364 Execute] Non-zero status code returned while running Where node. Name:'Where_389' Status Message: Where_389: condition operand cannot broadcast on dim 1 Condition Shape: {1,292}, X Shape: {1,292,768}, Y Shape: {}
---------------------------------------------------------------------------
Fail                                      Traceback (most recent call last)
Input In [11], in <cell line: 7>()
      3 ort_session = onnxruntime.InferenceSession("model.onnx",providers=['CUDAExecutionProvider'])
      4 inputs_onnx = {k: v.cpu().detach().numpy() for k, v in inputs.items()}
----> 7 ort_outs = ort_session.run(None, inputs_onnx)
      9 # compare ONNX Runtime and PyTorch results
     10 np.testing.assert_allclose(to_numpy(outputs[0]), ort_outs[0], rtol=1e-03, atol=1e-05)

File /opt/conda/envs/py39/lib/python3.9/site-packages/onnxruntime/capi/onnxruntime_inference_collection.py:192, in Session.run(self, output_names, input_feed, run_options)
    190     output_names = [output.name for output in self._outputs_meta]
    191 try:
--> 192     return self._sess.run(output_names, input_feed, run_options)
    193 except C.EPFail as err:
    194     if self._enable_fallback:

Fail: [ONNXRuntimeError] : 1 : FAIL : Non-zero status code returned while running Where node. Name:'Where_389' Status Message: Where_389: condition operand cannot broadcast on dim 1 Condition Shape: {1,292}, X Shape: {1,292,768}, Y Shape: {}
```

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) are likely to be ignored.