ConformerCTC with input problems after ONNX conversion
State: closed | Pull request: no
Labels: name: bug

I'm trying to convert the ConformerCTC (small) to ONNX, which works without problems, but when I'm testing the model with a single audio file I get a shape error:

```
 [E:onnxruntime:, sequential_executor.cc:339 Execute] Non-zero status code returned while running Reshape node. Name:'Reshape_378' 
Status Message: /onnxruntime/onnxruntime/core/providers/cpu/tensor/reshape_helper.h:37 onnxruntime::ReshapeHelper::ReshapeHelper(const onnxruntime::TensorShape&, std::vector<long int>&) size != 0 && (input_shape.Size() % size) == 0 was false. 
The input tensor cannot be reshaped to the requested shape. Input shape:{1,115,176}, requested shape:{16,-1,4,44}

  0%|                                                                                       | 0/1 [00:00<?, ?it/s]
Traceback (most recent call last):
  File "/dsp_nemo/testing_nemo.py", line 186, in <module>
    run_full_test("stt_en_conformer_ctc_small", batch_size=1, print_predictions=True)
  File "/dsp_nemo/testing_nemo.py", line 161, in run_full_test
    ologits = ort_session.run(None, ort_inputs)
  File "/opt/conda/lib/python3.8/site-packages/onnxruntime/capi/onnxruntime_inference_collection.py", line 188, in run
    return self._sess.run(output_names, input_feed, run_options)
```

The code is the following:
```python3
test_manifest = "/dsp_nemo/data/test.json"

def run_full_test(nemo_model_name, batch_size, print_predictions):

    config = {"batch_size": batch_size, "manifestpath": test_manifest}
    model = nemo_asr.models.EncDecCTCModel.restore_from(
        restore_path="/dsp_nemo/models/" + nemo_model_name + ".nemo"
    )
    ort_session = onnxruntime.InferenceSession(
        "/dsp_nemo/models/" + nemo_model_name + ".onnx"
    )
    temporary_datalayer = setup_transcribe_dataloader(config, model.decoder.vocabulary)
    wer_calc = WER(
        vocabulary=model.decoder.vocabulary,
        batch_dim_index=0,
        use_cer=False,
        ctc_decode=True,
    )

    wer_nums = []
    wer_denoms = []

    for test_batch in tqdm.tqdm(temporary_datalayer):
        processed_signal, processed_signal_len = model.preprocessor(
            input_signal=test_batch[0].to(model.device),
            length=test_batch[1].to(model.device),
        )
        ort_inputs = {ort_session.get_inputs()[0].name: to_numpy(processed_signal)}

        print(processed_signal.shape, processed_signal_len)  # torch.Size([1, 80, 458]) tensor([458], device='cuda:0')
        if "conformer" in nemo_model_name:
            ort_inputs[ort_session.get_inputs()[1].name] = to_numpy(
                processed_signal_len
            )

        ologits = ort_session.run(None, ort_inputs)
        alogits = np.asarray(ologits)
        logits = torch.from_numpy(alogits[0])
        greedy_predictions = logits.argmax(dim=-1, keepdim=False)

        if print_predictions:
            hypotheses = wer_calc.ctc_decoder_predictions_tensor(greedy_predictions)
            print(hypotheses)

        targets = test_batch[2]
        targets_lengths = test_batch[3]
        model._wer.update(greedy_predictions, targets, targets_lengths)
        _, wer_num, wer_denom = model._wer.compute()
        wer_nums.append(wer_num.detach().cpu().numpy())
        wer_denoms.append(wer_denom.detach().cpu().numpy())

    # We need to sum all numerators and denominators first. Then divide.
    print(f"WER = {sum(wer_nums) / sum(wer_denoms)}")

run_full_test("stt_en_conformer_ctc_small", batch_size=1, print_predictions=True)
# run_full_test("QuartzNet5x5LS-En", batch_size=1, print_predictions=True)
```

Running the same code with the QuartzNet model works as expected.

Tested with NeMo docker container (nvcr.io/nvidia/nemo:1.4.0) with onnxruntime (v1.7.2) additionally installed.
Onnx export with:
```
python3 /NeMo/scripts/export.py \
    /dsp_nemo/models/stt_en_conformer_ctc_small.nemo /dsp_nemo/models/stt_en_conformer_ctc_small.onnx
```
<br>

Any ideas on this?