Repository: apache/tvm

Issue:
[Bug] [Relay] attribute track_running_stats of InstanceNorm lead to wrong inference results
For the layer `InstanceNorm1d` or `InstanceNorm3d`, if attribute `track_running_stats` was set as `True`, TVM will give different inference results with PyTorch.

### Expected behavior
For the same input data, TVM and PyTorch give the same inference results.

### Actual behavior

![image](https://github.com/apache/tvm/assets/29506758/54096b21-6ae3-4522-a422-4972b6609fec)


### Steps to reproduce
```
import torch
from tvm import relay
import tvm
import numpy as np

m = torch.nn.InstanceNorm1d(3, track_running_stats=True).eval()
input_data = [torch.randn([1, 3, 5], dtype=torch.float32)]

torch_outputs = m(*[input_.clone() for input_ in input_data ])
trace = torch.jit.trace(m, input_data)
input_shapes = [('input0', torch.Size([1,3,5]))]

mod, params = relay.frontend.from_pytorch(trace, input_shapes)

with tvm.transform.PassContext(opt_level=3):
    exe = relay.create_executor('aot', mod=mod, params=params, device=tvm.cpu(0), target='llvm').evaluate()
input_tvm = {'input0': np.array(input_data[0], dtype='float32')}
tvm_outputs = exe(**input_tvm).asnumpy()

np.testing.assert_allclose(torch_outputs, tvm_outputs, rtol=1e-3, atol=1e-3)
```

### Triage
* frontend:torch
* needs-triage


cc @shingjan


Discussion:

Resolving patch:
diff --git a/python/tvm/relay/frontend/pytorch.py b/python/tvm/relay/frontend/pytorch.py
--- a/python/tvm/relay/frontend/pytorch.py
+++ b/python/tvm/relay/frontend/pytorch.py
@@ -1385,6 +1385,9 @@ def instance_norm(self, inputs, input_types):
         data = inputs[0]
         data_type = input_types[0]
         channels = self.infer_shape(data)
+        running_mean = inputs[3]
+        running_var = inputs[4]
+        use_input_stats = inputs[5]
 
         if isinstance(inputs[1], _expr.Expr) and isinstance(inputs[2], _expr.Expr):
             scale = center = True
@@ -1401,6 +1404,20 @@ def instance_norm(self, inputs, input_types):
             beta = _create_typed_const(np.zeros([int(channels[1])]), data_type)
 
         epsilon = float(inputs[7])
+
+        if not use_input_stats:
+            return _op.nn.batch_norm(
+                data,
+                gamma,
+                beta,
+                running_mean,
+                running_var,
+                axis=1,
+                epsilon=epsilon,
+                center=center,
+                scale=scale,
+            )[0]
+
         return _op.nn.instance_norm(
             data, gamma, beta, axis=1, epsilon=epsilon, center=center, scale=scale
         )


Test patch:
diff --git a/tests/python/frontend/pytorch/test_forward.py b/tests/python/frontend/pytorch/test_forward.py
--- a/tests/python/frontend/pytorch/test_forward.py
+++ b/tests/python/frontend/pytorch/test_forward.py
@@ -1414,6 +1414,8 @@ def test_forward_instancenorm():
     for ins_norm, inp in [
         (torch.nn.InstanceNorm2d(16), inp_2d),
         (torch.nn.InstanceNorm3d(16), inp_3d),
+        (torch.nn.InstanceNorm2d(16, track_running_stats=True), inp_2d),
+        (torch.nn.InstanceNorm3d(16, track_running_stats=True), inp_3d),
     ]:
         verify_model(ins_norm.eval(), input_data=inp)