import torch
import triton
import triton.language as tl
from torch._inductor.runtime.triton_heuristics import grid
from torch._C import _cuda_getCurrentRawStream as get_raw_stream
from torch._inductor.runtime.triton_helpers import libdevice
import torch.nn as nn
import torch.cuda
import torch.distributed
assert_size_stride = torch._C._dynamo.guards.assert_size_stride
empty_strided_cuda = torch._C._dynamo.guards._empty_strided_cuda


@triton.jit
def triton_red_fused_add_div_mean_mul_pow_reciprocal_sqrt_std_sub_sum_0(in_ptr0
    , out_ptr1, xnumel, rnumel, XBLOCK: tl.constexpr, RBLOCK: tl.constexpr):
    rnumel = 256
    xoffset = tl.program_id(0) * XBLOCK
    xoffset + tl.arange(0, XBLOCK)[:, None]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    rbase = tl.arange(0, RBLOCK)[None, :]
    _tmp3 = tl.full([XBLOCK, RBLOCK], 0, tl.float32)
    for roffset in range(0, rnumel, RBLOCK):
        rindex = roffset + rbase
        rmask = rindex < rnumel
        r0 = rindex
        tmp0 = tl.load(in_ptr0 + r0, rmask, eviction_policy='evict_last',
            other=0.0)
        tmp1 = tmp0 * tmp0
        tmp2 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
        tmp4 = _tmp3 + tmp2
        _tmp3 = tl.where(rmask, tmp4, _tmp3)
    tmp3 = tl.sum(_tmp3, 1)[:, None]
    for roffset in range(0, rnumel, RBLOCK):
        rindex = roffset + rbase
        rmask = rindex < rnumel
        r0 = rindex
        r2 = rindex // 4
        tmp10 = tl.load(in_ptr0 + r0, rmask, eviction_policy='evict_first',
            other=0.0)
        tmp11 = tl.load(in_ptr0 + 4 * r2, rmask, eviction_policy=
            'evict_last', other=0.0)
        tmp12 = tl.load(in_ptr0 + (1 + 4 * r2), rmask, eviction_policy=
            'evict_last', other=0.0)
        tmp14 = tl.load(in_ptr0 + (2 + 4 * r2), rmask, eviction_policy=
            'evict_last', other=0.0)
        tmp16 = tl.load(in_ptr0 + (3 + 4 * r2), rmask, eviction_policy=
            'evict_last', other=0.0)
        tmp5 = tl.full([1, 1], 1, tl.int32)
        tmp6 = tmp5 / tmp3
        tmp7 = 16.0
        tmp8 = tmp6 * tmp7
        tmp9 = libdevice.sqrt(tmp8)
        tmp13 = tmp11 + tmp12
        tmp15 = tmp13 + tmp14
        tmp17 = tmp15 + tmp16
        tmp18 = 4.0
        tmp19 = tmp17 / tmp18
        tmp20 = tmp10 - tmp19
        tmp21 = tmp11 - tmp19
        tmp22 = tmp21 * tmp21
        tmp23 = tmp12 - tmp19
        tmp24 = tmp23 * tmp23
        tmp25 = tmp22 + tmp24
        tmp26 = tmp14 - tmp19
        tmp27 = tmp26 * tmp26
        tmp28 = tmp25 + tmp27
        tmp29 = tmp16 - tmp19
        tmp30 = tmp29 * tmp29
        tmp31 = tmp28 + tmp30
        tmp32 = 3.0
        tmp33 = tmp31 / tmp32
        tmp34 = libdevice.sqrt(tmp33)
        tmp35 = 1e-06
        tmp36 = tmp34 + tmp35
        tmp37 = tmp20 / tmp36
        tmp38 = tmp9 * tmp37
        tl.store(out_ptr1 + tl.broadcast_to(r0, [XBLOCK, RBLOCK]), tmp38, rmask
            )


def call(args):
    arg0_1, = args
    args.clear()
    assert_size_stride(arg0_1, (4, 4, 4, 4), (64, 16, 4, 1))
    with torch.cuda._DeviceGuard(0):
        torch.cuda.set_device(0)
        buf1 = empty_strided_cuda((4, 4, 4, 4), (64, 16, 4, 1), torch.float32)
        get_raw_stream(0)
        triton_red_fused_add_div_mean_mul_pow_reciprocal_sqrt_std_sub_sum_0[
            grid(1)](arg0_1, buf1, 1, 256, XBLOCK=1, RBLOCK=256, num_warps=
            8, num_stages=1)
        del arg0_1
    return buf1,


class LayerNormAVGNew(nn.Module):
    """
        Layer Normalization class inspired by Transformer normalization, but here we normalize to given average
        to preserve magnitue of USE
    """

    def __init__(self, features, desired_avg, eps=1e-06):
        super(LayerNormAVGNew, self).__init__()
        self.desiredAVG = desired_avg
        self.eps = eps
        self.size = features

    def forward(self, input_0):
        arg0_1 = input_0
        output = call([arg0_1])
        return output[0]


Measured latency_ms: 0.014816