import torch
from torch._inductor.select_algorithm import extern_kernels
import triton
import triton.language as tl
from torch._inductor.runtime.triton_heuristics import grid
from torch._C import _cuda_getCurrentRawStream as get_raw_stream
from torch._inductor.runtime.triton_helpers import libdevice
from torch.autograd import Function
import torch.nn as nn
import torch.utils
assert_size_stride = torch._C._dynamo.guards.assert_size_stride
empty_strided_cuda = torch._C._dynamo.guards._empty_strided_cuda
reinterpret_tensor = torch._C._dynamo.guards._reinterpret_tensor


@triton.jit
def triton_poi_fused_mul_0(in_ptr0, out_ptr0, xnumel, XBLOCK: tl.constexpr):
    xnumel = 256
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    xmask = xindex < xnumel
    x0 = xindex % 64
    x1 = xindex // 64
    x2 = xindex
    tmp0 = tl.load(in_ptr0 + (16 * x1 + 64 * (x0 // 16) + x0 % 16), xmask)
    tmp1 = tmp0 * tmp0
    tl.store(out_ptr0 + x2, tmp1, xmask)


@triton.jit
def triton_poi_fused_clone_1(in_ptr0, in_ptr1, in_ptr2, out_ptr0, ynumel,
    xnumel, YBLOCK: tl.constexpr, XBLOCK: tl.constexpr):
    ynumel = 16
    xnumel = 16
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    ymask = yindex < ynumel
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y3 = yindex
    y0 = yindex % 4
    y1 = yindex // 4
    tmp0 = tl.load(in_ptr0 + (x2 + 16 * y3), xmask & ymask)
    tmp1 = tl.load(in_ptr1 + (y0 + 4 * x2 + 64 * y1), xmask & ymask)
    tmp2 = tl.load(in_ptr2 + y0, ymask, eviction_policy='evict_last')
    tmp3 = tmp1 + tmp2
    tmp4 = libdevice.sqrt(tmp3)
    tmp5 = tmp0 / tmp4
    tl.store(out_ptr0 + (x2 + 16 * y3), tmp5, xmask & ymask)


def call(args):
    arg0_1, arg1_1, arg2_1 = args
    args.clear()
    assert_size_stride(arg0_1, (4, 4, 4, 4), (64, 16, 4, 1))
    assert_size_stride(arg1_1, (4, 4), (4, 1))
    assert_size_stride(arg2_1, (4,), (1,))
    with torch.cuda._DeviceGuard(0):
        torch.cuda.set_device(0)
        buf0 = empty_strided_cuda((64, 4), (1, 64), torch.float32)
        get_raw_stream(0)
        triton_poi_fused_mul_0[grid(256)](arg0_1, buf0, 256, XBLOCK=128,
            num_warps=4, num_stages=1)
        buf1 = empty_strided_cuda((64, 4), (4, 1), torch.float32)
        extern_kernels.mm(buf0, arg1_1, out=buf1)
        del arg1_1
        buf2 = reinterpret_tensor(buf0, (4, 4, 4, 4), (64, 16, 4, 1), 0)
        del buf0
        triton_poi_fused_clone_1[grid(16, 16)](arg0_1, buf1, arg2_1, buf2, 
            16, 16, XBLOCK=16, YBLOCK=16, num_warps=4, num_stages=1)
        del arg0_1
        del arg2_1
        del buf1
    return buf2,


class GdnFunction(Function):

    @staticmethod
    def forward(ctx, x, gamma, beta):
        ctx.save_for_backward(x, gamma, beta)
        n, c, h, w = list(x.size())
        tx = x.permute(0, 2, 3, 1).contiguous()
        tx = tx.view(-1, c)
        tx2 = tx * tx
        denominator = tx2.mm(gamma) + beta
        ty = tx / torch.sqrt(denominator)
        y = ty.view(n, h, w, c)
        y = y.permute(0, 3, 1, 2).contiguous()
        return y

    @staticmethod
    def backward(ctx, grad_output):
        x, gamma, beta = ctx.saved_variables
        n, c, h, w = list(grad_output.size())
        tx = x.permute(0, 2, 3, 1).contiguous()
        tx = tx.view(-1, c)
        tx2 = tx * tx
        denominator = tx2.mm(gamma) + beta
        tdzdy = grad_output.permute(0, 2, 3, 1).contiguous()
        tdzdy = tdzdy.view(-1, c)
        gy = tdzdy * torch.pow(denominator, -0.5) - (tdzdy * tx * torch.pow
            (denominator, -1.5)).mm(gamma.t()) * tx
        gy = gy.view(n, h, w, c)
        grad_input = gy.permute(0, 3, 1, 2).contiguous()
        tmp = -0.5 * torch.pow(denominator, -1.5) * tx * tdzdy
        grad_beta = torch.sum(tmp, 0)
        grad_gamma = tx2.t().mm(tmp)
        return grad_input, grad_gamma, grad_beta


class GdnNew(nn.Module):

    def __init__(self, input_channel):
        super(GdnNew, self).__init__()
        self.input_channel = input_channel
        self.gamma = nn.Parameter(torch.Tensor(input_channel, input_channel))
        self.beta = nn.Parameter(torch.Tensor(input_channel))

    def __str__(self):
        return (self.__class__.__name__ + 
            '(gamma_size=(%d, %d), beta_size=(%d))' % (self.gamma.size()[0],
            self.gamma.size()[1], self.beta.size()[0]))
    __repr__ = __str__

    def forward(self, input_0):
        arg1_1 = self.gamma
        arg2_1 = self.beta
        arg0_1 = input_0
        output = call([arg0_1, arg1_1, arg2_1])
        return output[0]


Measured latency_ms: 0.521154