relu-gru mse is 0.022 much greater than 0.003 with half dtype.
Labels: needs reproduction, triaged, module: half

### 🐛 Describe the bug

gru op with nonlinearity type relu gets low percision when run fp16 datatype.

python code:
`
from __future__ import print_function

import time
import numpy as np
import torch
import torch.nn.utils.rnn as rnn_utils
from itertools import product

def two_tensor_diff_by_mse(self, other):
    if self.dtype in (torch.bool, torch.double, torch.half):
        self = self.float()
    if other.dtype in (torch.bool, torch.double, torch.half):
        other = other.float()
    epsilon = 1.0 / 16384
    assert self.size() == other.size(), "tensor size need be equal."
    assert self.numel() > 0, "Tensor numel need be greater than 0."
    # check that NaNs are in the same locations
    nan_mask = self != self
    assert torch.equal(nan_mask, other != other), "nan is in different position."
    diff = self - other
    diff[nan_mask] = 0
    diff = diff.abs().pow(2).sum()
    self_pow_sum = self.pow(2).sum()
    if diff <= (2 * epsilon) * (2 * epsilon):
        diff = 0.0
    if self_pow_sum <= epsilon:
        self_pow_sum = self_pow_sum + epsilon
    diff = torch.div(diff, (self_pow_sum * 1.0))
    assert diff.sqrt() < 0.003, "diff value {0} is greater than default_mse: {1}.".format(diff.sqrt(), 0.003)

def TensorGenerator(shape, dtype, func =  lambda x:x):
    cpu_tensor = torch.randn(shape).to(torch.half).to(torch.float)
    mlu_tensor = func(cpu_tensor.to("cuda").to(dtype))       
    cpu_tensor = func(cpu_tensor)                                           
    return cpu_tensor, mlu_tensor           

# almost copy from origin pytorch test case.
def test_RNN_cpu_vs_mlu():
    def forward_backward(mlu, dtype, rnn, input_val, grad_output, \
                         hx_val, grad_hy, grad_cy=None):

        if isinstance(input_val, rnn_utils.PackedSequence):
            input = rnn_utils.PackedSequence(
                input_val.data.data.requires_grad_(True), input_val.batch_sizes)
            input_var = input.data
        else:
            input = input_val.clone().requires_grad_(True)
            input_var = input
        hx = hx_val.clone().requires_grad_(True)

        if mlu:
            rnn.cuda().to(dtype)

        output, hy = rnn(input, hx)

        if isinstance(output, rnn_utils.PackedSequence):
            output = output.data
        torch.autograd.backward([output, hy], [grad_output, grad_hy])

        return {'output': output.data,
                'hy': hy.data,
                'weights': rnn.all_weights,
                'grad_input': input_var.grad.data,
                'grad_hx': hx.grad.data}

    def compare_cpu_mlu(outputs_cpu, outputs_mlu):
        #self.assertEqual(list(outputs_cpu.keys()), list(outputs_mlu.keys()))
        for key in outputs_cpu.keys():
            print(key, flush = True)
            if key != 'weights':
                two_tensor_diff_by_mse(outputs_cpu[key], outputs_mlu[key].cpu())

        # check grad weights separately, as nested dict
        for cpu_layer_weight, mlu_layer_weight in zip(outputs_cpu['weights'], \
                                                      outputs_mlu['weights']):
            for (cpu_weight, mlu_weight) in zip(cpu_layer_weight, mlu_layer_weight):
                two_tensor_diff_by_mse(cpu_weight.grad.data, mlu_weight.grad.data.cpu())

    seed = int(time.time())
    torch.backends.cudnn.enabled = True
    torch.backends.cudnn.allow_tf32 = False
    torch.backends.cuda.matmul.allow_tf32 = False
    input_size = 2; hidden_size= 128; batch = 6; seq_length = 50; bidirectional = True; bias = True
    batch_first = False; dtype = torch.half; num_layers = 2

    num_directions = 2 if bidirectional else 1

    hx_val_cpu, hx_val_mlu = TensorGenerator((num_layers * num_directions, batch, hidden_size), dtype)
    input_val_cpu, input_val_mlu = TensorGenerator((seq_length, batch, input_size), dtype)
    grad_output_cpu, grad_output_mlu = TensorGenerator((seq_length, batch, hidden_size * num_directions), dtype)
    grad_hy_cpu, grad_hy_mlu = TensorGenerator((num_layers * num_directions, batch, hidden_size), dtype)

    torch.manual_seed(seed)
    rnn = torch.nn.RNN(input_size, hidden_size, num_layers, bias=bias, \
                       bidirectional=bidirectional, batch_first=False, \
                       nonlinearity="relu").to(dtype).float()
    outputs_cpu = forward_backward(False, dtype, rnn, input_val_cpu, \
                                   grad_output_cpu, hx_val_cpu, grad_hy_cpu)

    torch.manual_seed(seed)
    rnn_mlu = torch.nn.RNN(input_size, hidden_size, num_layers, bias=bias, \
                           bidirectional=bidirectional, batch_first=False, \
                           nonlinearity="relu").to(dtype).float()
    outputs_mlu = forward_backward(True, dtype, rnn_mlu, \
                                   input_val_mlu, grad_output_mlu, \
                                   hx_val_mlu, grad_hy_mlu)
    compare_cpu_mlu(outputs_cpu, outputs_mlu)

if __name__ == '__main__':
    test_RNN_cpu_vs_mlu()


`

### Versions

pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html