Qurtznet, inference high deviation after onnx export
State: closed | Pull request: no
Labels: name: bug

name: stale

Tried to export quartznet encoder with onnx. 
Compered inference result with the same input tensor. 
Found unexplainably high  deviations between the original and exported model inference results.


```
import onnx
from onnx import checker
import onnxruntime
import torch
import nemo
import nemo.collections.asr as nemo_asr
import librosa
import numpy as np


quartznet = nemo.collections.asr.models.EncDecCTCModel.restore_from('QuartzNet5x5LS-En.nemo').to('cpu').eval()

audio, sample_rate = librosa.load('/an4/wav/an4_clstk/mgah/cen2-mgah-b.wav', sr=16e3)
audio_torch = torch.Tensor(audio[np.newaxis])
audio_torch_len = torch.Tensor([audio_torch.shape[1]])

processed_signal, processed_signal_length = quartznet.preprocessor(input_signal=audio_torch, length=audio_torch_len)
encoded_direct = quartznet.encoder(audio_signal=processed_signal, length=processed_signal_length)

quartznet.encoder.export('qn_enc.onnx')
ort_session_encoder = onnxruntime.InferenceSession('qn_enc.onnx')

ort_inputs_encoder = {ort_session_encoder.get_inputs()[0].name: processed_signal.numpy(),
                                        ort_session_encoder.get_inputs()[1].name: processed_signal_length.numpy()}

ologits_enc = ort_session_encoder.run(None, ort_inputs_encoder)


np.testing.assert_allclose(encoded_direct[0].detach().numpy(), ologits_enc[0], rtol=1e-02, atol=1e-02)
# raises:
# AssertionError: 
# Not equal to tolerance rtol=0.01, atol=0.01

# Mismatched elements: 32002 / 139264 (23%)
# Max absolute difference: 5.5700774
# Max relative difference: 6858.9634

```
Expected the differences to be much smaller (less then 1e-4)

If i try to export the whole model (includes encoder + decoder), as suggested in the comments, the differences become even bigger :

```
import onnx
from onnx import checker
import onnxruntime
import torch
import nemo
import nemo.collections.asr as nemo_asr
import librosa
import numpy as np


quartznet = nemo.collections.asr.models.EncDecCTCModel.restore_from('QuartzNet5x5LS-En.nemo').to('cpu').eval()

audio, sample_rate = librosa.load('/an4/wav/an4_clstk/mgah/cen2-mgah-b.wav', sr=16e3)
audio_torch = torch.Tensor(audio[np.newaxis])
audio_torch_len = torch.Tensor([audio_torch.shape[1]])

processed_signal, processed_signal_length = quartznet.preprocessor(input_signal=audio_torch, length=audio_torch_len)
encoded_direct = quartznet.encoder(audio_signal=processed_signal, length=processed_signal_length)
decoder_direct = quartznet.decoder(encoder_output=encoded_direct[0])

quartznet.export('qn.onnx')
ort_session = onnxruntime.InferenceSession('qn.onnx')

ort_inputs = {ort_session.get_inputs()[0].name: processed_signal.numpy()}
ologits = ort_session.run(None, ort_inputs)

np.testing.assert_allclose(decoder_direct.detach().numpy(), ologits[0], rtol=1e-02, atol=1e-02)

#raises:
#AssertionError: 
#Not equal to tolerance rtol=0.01, atol=0.01

#Mismatched elements: 1593 / 3944 (40.4%)
#Max absolute difference: 12.401756
#Max relative difference: 47469.176



```

package versions:
torch                             1.13.1
onnxruntime               1.12.1
nemo-toolkit               1.13.0