import torch
from torch._inductor.select_algorithm import extern_kernels
import triton
import triton.language as tl
from torch._inductor.runtime.triton_heuristics import grid
from torch._C import _cuda_getCurrentRawStream as get_raw_stream
from torch._inductor.runtime import triton_helpers
from torch._inductor.runtime.triton_helpers import libdevice
from torch import nn
assert_size_stride = torch._C._dynamo.guards.assert_size_stride
empty_strided_cuda = torch._C._dynamo.guards._empty_strided_cuda
reinterpret_tensor = torch._C._dynamo.guards._reinterpret_tensor


@triton.jit
def triton_poi_fused_0(in_ptr0, out_ptr0, ynumel, xnumel, YBLOCK: tl.
    constexpr, XBLOCK: tl.constexpr):
    ynumel = 192
    xnumel = 9
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    ymask = yindex < ynumel
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y3 = yindex
    y0 = yindex % 3
    y1 = yindex // 3
    tmp0 = tl.load(in_ptr0 + (x2 + 9 * y3), xmask & ymask, eviction_policy=
        'evict_last')
    tl.store(out_ptr0 + (y0 + 3 * x2 + 27 * y1), tmp0, xmask & ymask)


@triton.jit
def triton_poi_fused_1(in_ptr0, out_ptr0, ynumel, xnumel, YBLOCK: tl.
    constexpr, XBLOCK: tl.constexpr):
    xnumel = 9
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y3 = yindex
    y0 = yindex % 64
    y1 = yindex // 64
    tmp0 = tl.load(in_ptr0 + (x2 + 9 * y3), xmask, eviction_policy='evict_last'
        )
    tl.store(out_ptr0 + (y0 + 64 * x2 + 576 * y1), tmp0, xmask)


@triton.jit
def triton_poi_fused_2(in_ptr0, out_ptr0, ynumel, xnumel, YBLOCK: tl.
    constexpr, XBLOCK: tl.constexpr):
    xnumel = 9
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y3 = yindex
    y0 = yindex % 64
    y1 = yindex // 64
    tmp0 = tl.load(in_ptr0 + (x2 + 9 * y3), xmask, eviction_policy='evict_last'
        )
    tl.store(out_ptr0 + (y0 + 64 * x2 + 576 * y1), tmp0, xmask)


@triton.jit
def triton_poi_fused_3(in_ptr0, out_ptr0, ynumel, xnumel, YBLOCK: tl.
    constexpr, XBLOCK: tl.constexpr):
    xnumel = 9
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y3 = yindex
    y0 = yindex % 128
    y1 = yindex // 128
    tmp0 = tl.load(in_ptr0 + (x2 + 9 * y3), xmask, eviction_policy='evict_last'
        )
    tl.store(out_ptr0 + (y0 + 128 * x2 + 1152 * y1), tmp0, xmask)


@triton.jit
def triton_poi_fused_4(in_ptr0, out_ptr0, ynumel, xnumel, YBLOCK: tl.
    constexpr, XBLOCK: tl.constexpr):
    xnumel = 9
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y3 = yindex
    y0 = yindex % 128
    y1 = yindex // 128
    tmp0 = tl.load(in_ptr0 + (x2 + 9 * y3), xmask, eviction_policy='evict_last'
        )
    tl.store(out_ptr0 + (y0 + 128 * x2 + 1152 * y1), tmp0, xmask)


@triton.jit
def triton_poi_fused_5(in_ptr0, out_ptr0, ynumel, xnumel, YBLOCK: tl.
    constexpr, XBLOCK: tl.constexpr):
    xnumel = 9
    yoffset = (tl.program_id(1) + tl.program_id(2) * tl.num_programs(1)
        ) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y3 = yindex
    y0 = yindex % 256
    y1 = yindex // 256
    tmp0 = tl.load(in_ptr0 + (x2 + 9 * y3), xmask, eviction_policy='evict_last'
        )
    tl.store(out_ptr0 + (y0 + 256 * x2 + 2304 * y1), tmp0, xmask)


@triton.jit
def triton_poi_fused_6(in_ptr0, out_ptr0, ynumel, xnumel, YBLOCK: tl.
    constexpr, XBLOCK: tl.constexpr):
    xnumel = 9
    yoffset = (tl.program_id(1) + tl.program_id(2) * tl.num_programs(1)
        ) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y3 = yindex
    y0 = yindex % 256
    y1 = yindex // 256
    tmp0 = tl.load(in_ptr0 + (x2 + 9 * y3), xmask, eviction_policy='evict_last'
        )
    tl.store(out_ptr0 + (y0 + 256 * x2 + 2304 * y1), tmp0, xmask)


@triton.jit
def triton_poi_fused_7(in_ptr0, out_ptr0, ynumel, xnumel, YBLOCK: tl.
    constexpr, XBLOCK: tl.constexpr):
    xnumel = 9
    yoffset = (tl.program_id(1) + tl.program_id(2) * tl.num_programs(1)
        ) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y3 = yindex
    y0 = yindex % 512
    y1 = yindex // 512
    tmp0 = tl.load(in_ptr0 + (x2 + 9 * y3), xmask, eviction_policy='evict_last'
        )
    tl.store(out_ptr0 + (y0 + 512 * x2 + 4608 * y1), tmp0, xmask)


@triton.jit
def triton_poi_fused_view_8(in_ptr0, out_ptr0, ynumel, xnumel, YBLOCK: tl.
    constexpr, XBLOCK: tl.constexpr):
    ynumel = 12
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    ymask = yindex < ynumel
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    x2 = xindex
    y3 = yindex
    y0 = yindex % 3
    y1 = yindex // 3
    tmp0 = tl.load(in_ptr0 + (x2 + 4096 * y3), ymask, eviction_policy=
        'evict_last')
    tl.store(out_ptr0 + (y0 + 3 * x2 + 12288 * y1), tmp0, ymask)


@triton.jit
def triton_poi_fused_convolution_9(in_out_ptr0, in_ptr0, xnumel, XBLOCK: tl
    .constexpr):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x2 = xindex
    x0 = xindex % 64
    tmp0 = tl.load(in_out_ptr0 + x2, None)
    tmp1 = tl.load(in_ptr0 + x0, None, eviction_policy='evict_last')
    tmp2 = tmp0 + tmp1
    tl.store(in_out_ptr0 + x2, tmp2, None)


@triton.jit
def triton_poi_fused_add_div_10(in_ptr0, out_ptr0, xnumel, XBLOCK: tl.constexpr
    ):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x0 = xindex % 64
    x1 = xindex // 64 % 32
    x2 = xindex // 2048
    x3 = xindex
    tmp0 = tl.load(in_ptr0 + (x0 + 128 * x1 + 8192 * x2), None)
    tmp1 = tl.load(in_ptr0 + (4096 + x0 + 128 * x1 + 8192 * x2), None)
    tmp3 = tl.load(in_ptr0 + (64 + x0 + 128 * x1 + 8192 * x2), None)
    tmp5 = tl.load(in_ptr0 + (4160 + x0 + 128 * x1 + 8192 * x2), None)
    tmp2 = tmp0 + tmp1
    tmp4 = tmp2 + tmp3
    tmp6 = tmp4 + tmp5
    tmp7 = 0.25
    tmp8 = tmp6 * tmp7
    tl.store(out_ptr0 + x3, tmp8, None)


@triton.jit
def triton_per_fused_native_layer_norm_11(in_ptr0, out_ptr0, out_ptr1,
    out_ptr2, xnumel, rnumel, XBLOCK: tl.constexpr):
    RBLOCK: tl.constexpr = 128
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r3 = rindex
    x0 = xindex % 32
    x1 = xindex // 32 % 64
    x2 = xindex // 2048
    x4 = xindex
    tmp0 = tl.load(in_ptr0 + (2 * x0 + 64 * (r3 % 64) + 4096 * ((r3 + 128 *
        x1) // 64 % 64) + 262144 * x2 + (r3 + 128 * x1) // 4096), None,
        eviction_policy='evict_last')
    tmp1 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp3 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.sum(tmp3, 1)[:, None]
    tmp6 = tl.full([XBLOCK, 1], 128, tl.int32)
    tmp7 = tmp6.to(tl.float32)
    tmp8 = tmp5 / tmp7
    tmp9 = tmp1 - tmp8
    tmp10 = tmp9 * tmp9
    tmp11 = tl.broadcast_to(tmp10, [XBLOCK, RBLOCK])
    tmp13 = tl.sum(tmp11, 1)[:, None]
    tl.store(out_ptr0 + x4, tmp8, None)
    tl.store(out_ptr1 + x4, tmp13, None)
    tl.store(out_ptr2 + x4, tmp7, None)


@triton.jit
def triton_per_fused_native_layer_norm_12(in_ptr0, in_ptr1, in_ptr2,
    out_ptr0, out_ptr1, out_ptr2, xnumel, rnumel, XBLOCK: tl.constexpr):
    xnumel = 128
    RBLOCK: tl.constexpr = 64
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r2 = rindex
    x0 = xindex % 32
    x1 = xindex // 32
    x3 = xindex
    tmp0 = tl.load(in_ptr0 + (x0 + 32 * r2 + 2048 * x1), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (x0 + 32 * r2 + 2048 * x1), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (x0 + 32 * r2 + 2048 * x1), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp15 = tmp12[:, None]
    tl.store(out_ptr0 + x3, tmp13, xmask)
    tl.store(out_ptr1 + x3, tmp14, xmask)
    tl.store(out_ptr2 + x3, tmp15, xmask)


@triton.jit
def triton_per_fused_native_layer_norm_13(in_out_ptr0, in_ptr0, in_ptr1,
    in_ptr2, out_ptr0, xnumel, rnumel, XBLOCK: tl.constexpr):
    xnumel = 4
    RBLOCK: tl.constexpr = 32
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r1 = rindex
    x0 = xindex
    tmp0 = tl.load(in_ptr0 + (r1 + 32 * x0), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (r1 + 32 * x0), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (r1 + 32 * x0), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp12[:, None]
    tmp16 = 262144.0
    tmp17 = tmp14 / tmp16
    tmp18 = 1e-05
    tmp19 = tmp17 + tmp18
    tmp20 = libdevice.rsqrt(tmp19)
    tl.debug_barrier()
    tl.store(in_out_ptr0 + x0, tmp20, xmask)
    tl.store(out_ptr0 + x0, tmp13, xmask)


@triton.jit
def triton_poi_fused_native_layer_norm_relu_14(in_ptr0, in_ptr1, in_ptr2,
    in_ptr3, in_ptr4, out_ptr0, ynumel, xnumel, YBLOCK: tl.constexpr,
    XBLOCK: tl.constexpr):
    ynumel = 256
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    ymask = yindex < ynumel
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    x2 = xindex
    y0 = yindex % 64
    y1 = yindex // 64
    tmp0 = tl.load(in_ptr0 + (y0 + 64 * x2 + 262144 * y1), ymask,
        eviction_policy='evict_last')
    tmp1 = tl.load(in_ptr1 + y1, ymask, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr2 + y1, ymask, eviction_policy='evict_last')
    tmp5 = tl.load(in_ptr3 + (x2 + 4096 * y0), ymask, eviction_policy=
        'evict_last')
    tmp7 = tl.load(in_ptr4 + (x2 + 4096 * y0), ymask, eviction_policy=
        'evict_last')
    tmp2 = tmp0 - tmp1
    tmp4 = tmp2 * tmp3
    tmp6 = tmp4 * tmp5
    tmp8 = tmp6 + tmp7
    tmp9 = tl.full([1, 1], 0, tl.int32)
    tmp10 = triton_helpers.maximum(tmp9, tmp8)
    tl.store(out_ptr0 + (y0 + 64 * x2 + 262144 * y1), tmp10, ymask)


@triton.jit
def triton_poi_fused_add_convolution_div_15(in_out_ptr0, in_ptr0, in_ptr1,
    in_ptr2, xnumel, XBLOCK: tl.constexpr):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x3 = xindex
    x0 = xindex % 128
    x1 = xindex // 128 % 32
    x2 = xindex // 4096
    tmp0 = tl.load(in_out_ptr0 + x3, None)
    tmp1 = tl.load(in_ptr0 + x0, None, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr1 + (x0 + 256 * x1 + 16384 * x2), None)
    tmp4 = tl.load(in_ptr2 + x0, None, eviction_policy='evict_last')
    tmp6 = tl.load(in_ptr1 + (8192 + x0 + 256 * x1 + 16384 * x2), None)
    tmp9 = tl.load(in_ptr1 + (128 + x0 + 256 * x1 + 16384 * x2), None)
    tmp12 = tl.load(in_ptr1 + (8320 + x0 + 256 * x1 + 16384 * x2), None)
    tmp2 = tmp0 + tmp1
    tmp5 = tmp3 + tmp4
    tmp7 = tmp6 + tmp4
    tmp8 = tmp5 + tmp7
    tmp10 = tmp9 + tmp4
    tmp11 = tmp8 + tmp10
    tmp13 = tmp12 + tmp4
    tmp14 = tmp11 + tmp13
    tmp15 = 0.25
    tmp16 = tmp14 * tmp15
    tmp17 = tmp2 + tmp16
    tl.store(in_out_ptr0 + x3, tmp17, None)


@triton.jit
def triton_poi_fused_add_div_16(in_ptr0, out_ptr0, xnumel, XBLOCK: tl.constexpr
    ):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x0 = xindex % 128
    x1 = xindex // 128 % 16
    x2 = xindex // 2048
    x3 = xindex
    tmp0 = tl.load(in_ptr0 + (x0 + 256 * x1 + 8192 * x2), None)
    tmp1 = tl.load(in_ptr0 + (4096 + x0 + 256 * x1 + 8192 * x2), None)
    tmp3 = tl.load(in_ptr0 + (128 + x0 + 256 * x1 + 8192 * x2), None)
    tmp5 = tl.load(in_ptr0 + (4224 + x0 + 256 * x1 + 8192 * x2), None)
    tmp2 = tmp0 + tmp1
    tmp4 = tmp2 + tmp3
    tmp6 = tmp4 + tmp5
    tmp7 = 0.25
    tmp8 = tmp6 * tmp7
    tl.store(out_ptr0 + x3, tmp8, None)


@triton.jit
def triton_per_fused_native_layer_norm_17(in_ptr0, out_ptr0, out_ptr1,
    out_ptr2, xnumel, rnumel, XBLOCK: tl.constexpr):
    RBLOCK: tl.constexpr = 128
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r3 = rindex
    x0 = xindex % 16
    x1 = xindex // 16 % 64
    x2 = xindex // 1024
    x4 = xindex
    tmp0 = tl.load(in_ptr0 + (8 * x0 + 128 * (r3 % 32) + 4096 * ((r3 + 128 *
        x1) // 32 % 32) + 131072 * x2 + (r3 + 128 * x1) // 1024), None,
        eviction_policy='evict_last')
    tmp1 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp3 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.sum(tmp3, 1)[:, None]
    tmp6 = tl.full([XBLOCK, 1], 128, tl.int32)
    tmp7 = tmp6.to(tl.float32)
    tmp8 = tmp5 / tmp7
    tmp9 = tmp1 - tmp8
    tmp10 = tmp9 * tmp9
    tmp11 = tl.broadcast_to(tmp10, [XBLOCK, RBLOCK])
    tmp13 = tl.sum(tmp11, 1)[:, None]
    tl.store(out_ptr0 + x4, tmp8, None)
    tl.store(out_ptr1 + x4, tmp13, None)
    tl.store(out_ptr2 + x4, tmp7, None)


@triton.jit
def triton_per_fused_native_layer_norm_18(in_ptr0, in_ptr1, in_ptr2,
    out_ptr0, out_ptr1, out_ptr2, xnumel, rnumel, XBLOCK: tl.constexpr):
    xnumel = 64
    RBLOCK: tl.constexpr = 64
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r2 = rindex
    x0 = xindex % 16
    x1 = xindex // 16
    x3 = xindex
    tmp0 = tl.load(in_ptr0 + (x0 + 16 * r2 + 1024 * x1), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (x0 + 16 * r2 + 1024 * x1), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (x0 + 16 * r2 + 1024 * x1), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp15 = tmp12[:, None]
    tl.store(out_ptr0 + x3, tmp13, xmask)
    tl.store(out_ptr1 + x3, tmp14, xmask)
    tl.store(out_ptr2 + x3, tmp15, xmask)


@triton.jit
def triton_per_fused_native_layer_norm_native_layer_norm_backward_19(in_ptr0,
    in_ptr1, in_ptr2, out_ptr0, out_ptr1, out_ptr2, xnumel, rnumel, XBLOCK:
    tl.constexpr):
    xnumel = 4
    RBLOCK: tl.constexpr = 16
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r1 = rindex
    x0 = xindex
    tmp0 = tl.load(in_ptr0 + (r1 + 16 * x0), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (r1 + 16 * x0), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (r1 + 16 * x0), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp12[:, None]
    tmp16 = 131072.0
    tmp17 = tmp14 / tmp16
    tmp18 = 1e-05
    tmp19 = tmp17 + tmp18
    tmp20 = libdevice.rsqrt(tmp19)
    tmp21 = 7.62939453125e-06
    tmp22 = tmp20 * tmp21
    tl.store(out_ptr2 + x0, tmp22, xmask)
    tl.store(out_ptr0 + x0, tmp13, xmask)
    tl.store(out_ptr1 + x0, tmp14, xmask)


@triton.jit
def triton_poi_fused_native_layer_norm_20(in_out_ptr0, in_ptr0, in_ptr1,
    xnumel, XBLOCK: tl.constexpr):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x2 = xindex
    x1 = xindex // 131072
    tmp0 = tl.load(in_out_ptr0 + x2, None)
    tmp1 = tl.load(in_ptr0 + x1, None, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr1 + x1, None, eviction_policy='evict_last')
    tmp2 = tmp0 - tmp1
    tmp4 = 131072.0
    tmp5 = tmp3 / tmp4
    tmp6 = 1e-05
    tmp7 = tmp5 + tmp6
    tmp8 = libdevice.rsqrt(tmp7)
    tmp9 = tmp2 * tmp8
    tl.store(in_out_ptr0 + x2, tmp9, None)


@triton.jit
def triton_poi_fused_native_layer_norm_relu_21(in_ptr0, in_ptr1, in_ptr2,
    out_ptr0, ynumel, xnumel, YBLOCK: tl.constexpr, XBLOCK: tl.constexpr):
    ynumel = 512
    xnumel = 1024
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    ymask = yindex < ynumel
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y0 = yindex % 128
    y1 = yindex // 128
    tmp0 = tl.load(in_ptr0 + (y0 + 128 * x2 + 131072 * y1), xmask & ymask,
        eviction_policy='evict_last')
    tmp1 = tl.load(in_ptr1 + (x2 + 1024 * y0), xmask & ymask,
        eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr2 + (x2 + 1024 * y0), xmask & ymask,
        eviction_policy='evict_last')
    tmp2 = tmp0 * tmp1
    tmp4 = tmp2 + tmp3
    tmp5 = tl.full([1, 1], 0, tl.int32)
    tmp6 = triton_helpers.maximum(tmp5, tmp4)
    tl.store(out_ptr0 + (y0 + 128 * x2 + 131072 * y1), tmp6, xmask & ymask)


@triton.jit
def triton_per_fused_native_layer_norm_22(in_out_ptr0, in_ptr0, in_ptr1,
    in_ptr2, out_ptr0, xnumel, rnumel, XBLOCK: tl.constexpr):
    xnumel = 4
    RBLOCK: tl.constexpr = 16
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r1 = rindex
    x0 = xindex
    tmp0 = tl.load(in_ptr0 + (r1 + 16 * x0), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (r1 + 16 * x0), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (r1 + 16 * x0), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp12[:, None]
    tmp16 = 131072.0
    tmp17 = tmp14 / tmp16
    tmp18 = 1e-05
    tmp19 = tmp17 + tmp18
    tmp20 = libdevice.rsqrt(tmp19)
    tl.debug_barrier()
    tl.store(in_out_ptr0 + x0, tmp20, xmask)
    tl.store(out_ptr0 + x0, tmp13, xmask)


@triton.jit
def triton_poi_fused_native_layer_norm_relu_23(in_ptr0, in_ptr1, in_ptr2,
    in_ptr3, in_ptr4, out_ptr0, ynumel, xnumel, YBLOCK: tl.constexpr,
    XBLOCK: tl.constexpr):
    ynumel = 512
    xnumel = 1024
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    ymask = yindex < ynumel
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y0 = yindex % 128
    y1 = yindex // 128
    tmp0 = tl.load(in_ptr0 + (y0 + 128 * x2 + 131072 * y1), xmask & ymask,
        eviction_policy='evict_last')
    tmp1 = tl.load(in_ptr1 + y1, ymask, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr2 + y1, ymask, eviction_policy='evict_last')
    tmp5 = tl.load(in_ptr3 + (x2 + 1024 * y0), xmask & ymask,
        eviction_policy='evict_last')
    tmp7 = tl.load(in_ptr4 + (x2 + 1024 * y0), xmask & ymask,
        eviction_policy='evict_last')
    tmp2 = tmp0 - tmp1
    tmp4 = tmp2 * tmp3
    tmp6 = tmp4 * tmp5
    tmp8 = tmp6 + tmp7
    tmp9 = tl.full([1, 1], 0, tl.int32)
    tmp10 = triton_helpers.maximum(tmp9, tmp8)
    tl.store(out_ptr0 + (y0 + 128 * x2 + 131072 * y1), tmp10, xmask & ymask)


@triton.jit
def triton_poi_fused_add_convolution_div_24(in_out_ptr0, in_ptr0, in_ptr1,
    in_ptr2, xnumel, XBLOCK: tl.constexpr):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x3 = xindex
    x0 = xindex % 256
    x1 = xindex // 256 % 16
    x2 = xindex // 4096
    tmp0 = tl.load(in_out_ptr0 + x3, None)
    tmp1 = tl.load(in_ptr0 + x0, None, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr1 + (x0 + 512 * x1 + 16384 * x2), None)
    tmp4 = tl.load(in_ptr2 + x0, None, eviction_policy='evict_last')
    tmp6 = tl.load(in_ptr1 + (8192 + x0 + 512 * x1 + 16384 * x2), None)
    tmp9 = tl.load(in_ptr1 + (256 + x0 + 512 * x1 + 16384 * x2), None)
    tmp12 = tl.load(in_ptr1 + (8448 + x0 + 512 * x1 + 16384 * x2), None)
    tmp2 = tmp0 + tmp1
    tmp5 = tmp3 + tmp4
    tmp7 = tmp6 + tmp4
    tmp8 = tmp5 + tmp7
    tmp10 = tmp9 + tmp4
    tmp11 = tmp8 + tmp10
    tmp13 = tmp12 + tmp4
    tmp14 = tmp11 + tmp13
    tmp15 = 0.25
    tmp16 = tmp14 * tmp15
    tmp17 = tmp2 + tmp16
    tl.store(in_out_ptr0 + x3, tmp17, None)


@triton.jit
def triton_poi_fused_add_div_25(in_ptr0, out_ptr0, xnumel, XBLOCK: tl.constexpr
    ):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x0 = xindex % 256
    x1 = xindex // 256 % 8
    x2 = xindex // 2048
    x3 = xindex
    tmp0 = tl.load(in_ptr0 + (x0 + 512 * x1 + 8192 * x2), None)
    tmp1 = tl.load(in_ptr0 + (4096 + x0 + 512 * x1 + 8192 * x2), None)
    tmp3 = tl.load(in_ptr0 + (256 + x0 + 512 * x1 + 8192 * x2), None)
    tmp5 = tl.load(in_ptr0 + (4352 + x0 + 512 * x1 + 8192 * x2), None)
    tmp2 = tmp0 + tmp1
    tmp4 = tmp2 + tmp3
    tmp6 = tmp4 + tmp5
    tmp7 = 0.25
    tmp8 = tmp6 * tmp7
    tl.store(out_ptr0 + x3, tmp8, None)


@triton.jit
def triton_per_fused_native_layer_norm_26(in_ptr0, out_ptr0, out_ptr1,
    out_ptr2, xnumel, rnumel, XBLOCK: tl.constexpr):
    RBLOCK: tl.constexpr = 128
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r3 = rindex
    x0 = xindex % 8
    x1 = xindex // 8 % 64
    x2 = xindex // 512
    x4 = xindex
    tmp0 = tl.load(in_ptr0 + (32 * x0 + 256 * (r3 % 16) + 4096 * ((r3 + 128 *
        x1) // 16 % 16) + 65536 * x2 + (r3 + 128 * x1) // 256), None,
        eviction_policy='evict_last')
    tmp1 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp3 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.sum(tmp3, 1)[:, None]
    tmp6 = tl.full([XBLOCK, 1], 128, tl.int32)
    tmp7 = tmp6.to(tl.float32)
    tmp8 = tmp5 / tmp7
    tmp9 = tmp1 - tmp8
    tmp10 = tmp9 * tmp9
    tmp11 = tl.broadcast_to(tmp10, [XBLOCK, RBLOCK])
    tmp13 = tl.sum(tmp11, 1)[:, None]
    tl.store(out_ptr0 + x4, tmp8, None)
    tl.store(out_ptr1 + x4, tmp13, None)
    tl.store(out_ptr2 + x4, tmp7, None)


@triton.jit
def triton_per_fused_native_layer_norm_27(in_ptr0, in_ptr1, in_ptr2,
    out_ptr0, out_ptr1, out_ptr2, xnumel, rnumel, XBLOCK: tl.constexpr):
    xnumel = 32
    RBLOCK: tl.constexpr = 64
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r2 = rindex
    x0 = xindex % 8
    x1 = xindex // 8
    x3 = xindex
    tmp0 = tl.load(in_ptr0 + (x0 + 8 * r2 + 512 * x1), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (x0 + 8 * r2 + 512 * x1), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (x0 + 8 * r2 + 512 * x1), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp15 = tmp12[:, None]
    tl.store(out_ptr0 + x3, tmp13, xmask)
    tl.store(out_ptr1 + x3, tmp14, xmask)
    tl.store(out_ptr2 + x3, tmp15, xmask)


@triton.jit
def triton_per_fused_native_layer_norm_native_layer_norm_backward_28(in_ptr0,
    in_ptr1, in_ptr2, out_ptr0, out_ptr1, out_ptr2, xnumel, rnumel, XBLOCK:
    tl.constexpr):
    xnumel = 4
    RBLOCK: tl.constexpr = 8
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r1 = rindex
    x0 = xindex
    tmp0 = tl.load(in_ptr0 + (r1 + 8 * x0), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (r1 + 8 * x0), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (r1 + 8 * x0), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp12[:, None]
    tmp16 = 65536.0
    tmp17 = tmp14 / tmp16
    tmp18 = 1e-05
    tmp19 = tmp17 + tmp18
    tmp20 = libdevice.rsqrt(tmp19)
    tmp21 = 1.52587890625e-05
    tmp22 = tmp20 * tmp21
    tl.store(out_ptr2 + x0, tmp22, xmask)
    tl.store(out_ptr0 + x0, tmp13, xmask)
    tl.store(out_ptr1 + x0, tmp14, xmask)


@triton.jit
def triton_poi_fused_native_layer_norm_29(in_out_ptr0, in_ptr0, in_ptr1,
    xnumel, XBLOCK: tl.constexpr):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x2 = xindex
    x1 = xindex // 65536
    tmp0 = tl.load(in_out_ptr0 + x2, None)
    tmp1 = tl.load(in_ptr0 + x1, None, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr1 + x1, None, eviction_policy='evict_last')
    tmp2 = tmp0 - tmp1
    tmp4 = 65536.0
    tmp5 = tmp3 / tmp4
    tmp6 = 1e-05
    tmp7 = tmp5 + tmp6
    tmp8 = libdevice.rsqrt(tmp7)
    tmp9 = tmp2 * tmp8
    tl.store(in_out_ptr0 + x2, tmp9, None)


@triton.jit
def triton_poi_fused_native_layer_norm_relu_30(in_ptr0, in_ptr1, in_ptr2,
    out_ptr0, ynumel, xnumel, YBLOCK: tl.constexpr, XBLOCK: tl.constexpr):
    xnumel = 256
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y0 = yindex % 256
    y1 = yindex // 256
    tmp0 = tl.load(in_ptr0 + (y0 + 256 * x2 + 65536 * y1), xmask,
        eviction_policy='evict_last')
    tmp1 = tl.load(in_ptr1 + (x2 + 256 * y0), xmask, eviction_policy=
        'evict_last')
    tmp3 = tl.load(in_ptr2 + (x2 + 256 * y0), xmask, eviction_policy=
        'evict_last')
    tmp2 = tmp0 * tmp1
    tmp4 = tmp2 + tmp3
    tmp5 = tl.full([1, 1], 0, tl.int32)
    tmp6 = triton_helpers.maximum(tmp5, tmp4)
    tl.store(out_ptr0 + (y0 + 256 * x2 + 65536 * y1), tmp6, xmask)


@triton.jit
def triton_per_fused_native_layer_norm_31(in_out_ptr0, in_ptr0, in_ptr1,
    in_ptr2, out_ptr0, xnumel, rnumel, XBLOCK: tl.constexpr):
    xnumel = 4
    RBLOCK: tl.constexpr = 8
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r1 = rindex
    x0 = xindex
    tmp0 = tl.load(in_ptr0 + (r1 + 8 * x0), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (r1 + 8 * x0), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (r1 + 8 * x0), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp12[:, None]
    tmp16 = 65536.0
    tmp17 = tmp14 / tmp16
    tmp18 = 1e-05
    tmp19 = tmp17 + tmp18
    tmp20 = libdevice.rsqrt(tmp19)
    tl.debug_barrier()
    tl.store(in_out_ptr0 + x0, tmp20, xmask)
    tl.store(out_ptr0 + x0, tmp13, xmask)


@triton.jit
def triton_poi_fused_native_layer_norm_relu_32(in_ptr0, in_ptr1, in_ptr2,
    in_ptr3, in_ptr4, out_ptr0, ynumel, xnumel, YBLOCK: tl.constexpr,
    XBLOCK: tl.constexpr):
    xnumel = 256
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y0 = yindex % 256
    y1 = yindex // 256
    tmp0 = tl.load(in_ptr0 + (y0 + 256 * x2 + 65536 * y1), xmask,
        eviction_policy='evict_last')
    tmp1 = tl.load(in_ptr1 + y1, None, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr2 + y1, None, eviction_policy='evict_last')
    tmp5 = tl.load(in_ptr3 + (x2 + 256 * y0), xmask, eviction_policy=
        'evict_last')
    tmp7 = tl.load(in_ptr4 + (x2 + 256 * y0), xmask, eviction_policy=
        'evict_last')
    tmp2 = tmp0 - tmp1
    tmp4 = tmp2 * tmp3
    tmp6 = tmp4 * tmp5
    tmp8 = tmp6 + tmp7
    tmp9 = tl.full([1, 1], 0, tl.int32)
    tmp10 = triton_helpers.maximum(tmp9, tmp8)
    tl.store(out_ptr0 + (y0 + 256 * x2 + 65536 * y1), tmp10, xmask)


@triton.jit
def triton_poi_fused_add_convolution_div_33(in_out_ptr0, in_ptr0, in_ptr1,
    in_ptr2, xnumel, XBLOCK: tl.constexpr):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x3 = xindex
    x0 = xindex % 512
    x1 = xindex // 512 % 8
    x2 = xindex // 4096
    tmp0 = tl.load(in_out_ptr0 + x3, None)
    tmp1 = tl.load(in_ptr0 + x0, None, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr1 + (x0 + 1024 * x1 + 16384 * x2), None)
    tmp4 = tl.load(in_ptr2 + x0, None, eviction_policy='evict_last')
    tmp6 = tl.load(in_ptr1 + (8192 + x0 + 1024 * x1 + 16384 * x2), None)
    tmp9 = tl.load(in_ptr1 + (512 + x0 + 1024 * x1 + 16384 * x2), None)
    tmp12 = tl.load(in_ptr1 + (8704 + x0 + 1024 * x1 + 16384 * x2), None)
    tmp2 = tmp0 + tmp1
    tmp5 = tmp3 + tmp4
    tmp7 = tmp6 + tmp4
    tmp8 = tmp5 + tmp7
    tmp10 = tmp9 + tmp4
    tmp11 = tmp8 + tmp10
    tmp13 = tmp12 + tmp4
    tmp14 = tmp11 + tmp13
    tmp15 = 0.25
    tmp16 = tmp14 * tmp15
    tmp17 = tmp2 + tmp16
    tl.store(in_out_ptr0 + x3, tmp17, None)


@triton.jit
def triton_poi_fused_add_div_34(in_ptr0, out_ptr0, xnumel, XBLOCK: tl.constexpr
    ):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x0 = xindex % 512
    x1 = xindex // 512 % 4
    x2 = xindex // 2048
    x3 = xindex
    tmp0 = tl.load(in_ptr0 + (x0 + 1024 * x1 + 8192 * x2), None)
    tmp1 = tl.load(in_ptr0 + (4096 + x0 + 1024 * x1 + 8192 * x2), None)
    tmp3 = tl.load(in_ptr0 + (512 + x0 + 1024 * x1 + 8192 * x2), None)
    tmp5 = tl.load(in_ptr0 + (4608 + x0 + 1024 * x1 + 8192 * x2), None)
    tmp2 = tmp0 + tmp1
    tmp4 = tmp2 + tmp3
    tmp6 = tmp4 + tmp5
    tmp7 = 0.25
    tmp8 = tmp6 * tmp7
    tl.store(out_ptr0 + x3, tmp8, None)


@triton.jit
def triton_per_fused_native_layer_norm_35(in_ptr0, out_ptr0, out_ptr1,
    out_ptr2, xnumel, rnumel, XBLOCK: tl.constexpr):
    xnumel = 1024
    RBLOCK: tl.constexpr = 128
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r2 = rindex
    x0 = xindex % 256
    x1 = xindex // 256
    x3 = xindex
    tmp0 = tl.load(in_ptr0 + (2 * x0 + 512 * (r2 % 64) + 32768 * x1 + r2 //
        64), xmask, eviction_policy='evict_last', other=0.0)
    tmp1 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tl.where(xmask, tmp1, 0)
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp6 = tl.where(xmask, tmp4, 0)
    tmp7 = tl.sum(tmp6, 1)[:, None]
    tmp8 = tl.full([XBLOCK, 1], 128, tl.int32)
    tmp9 = tmp8.to(tl.float32)
    tmp10 = tmp7 / tmp9
    tmp11 = tmp1 - tmp10
    tmp12 = tmp11 * tmp11
    tmp13 = tl.broadcast_to(tmp12, [XBLOCK, RBLOCK])
    tmp15 = tl.where(xmask, tmp13, 0)
    tmp16 = tl.sum(tmp15, 1)[:, None]
    tl.store(out_ptr0 + x3, tmp10, xmask)
    tl.store(out_ptr1 + x3, tmp16, xmask)
    tl.store(out_ptr2 + x3, tmp9, xmask)


@triton.jit
def triton_per_fused_native_layer_norm_36(in_ptr0, in_ptr1, in_ptr2,
    out_ptr0, out_ptr1, out_ptr2, xnumel, rnumel, XBLOCK: tl.constexpr):
    xnumel = 16
    RBLOCK: tl.constexpr = 64
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r1 = rindex
    x0 = xindex
    tmp0 = tl.load(in_ptr0 + (r1 + 64 * x0), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (r1 + 64 * x0), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (r1 + 64 * x0), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp15 = tmp12[:, None]
    tl.store(out_ptr0 + x0, tmp13, xmask)
    tl.store(out_ptr1 + x0, tmp14, xmask)
    tl.store(out_ptr2 + x0, tmp15, xmask)


@triton.jit
def triton_per_fused_native_layer_norm_native_layer_norm_backward_37(in_ptr0,
    in_ptr1, in_ptr2, out_ptr0, out_ptr1, out_ptr2, xnumel, rnumel, XBLOCK:
    tl.constexpr):
    xnumel = 4
    RBLOCK: tl.constexpr = 4
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r1 = rindex
    x0 = xindex
    tmp0 = tl.load(in_ptr0 + (r1 + 4 * x0), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (r1 + 4 * x0), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (r1 + 4 * x0), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp12[:, None]
    tmp16 = 32768.0
    tmp17 = tmp14 / tmp16
    tmp18 = 1e-05
    tmp19 = tmp17 + tmp18
    tmp20 = libdevice.rsqrt(tmp19)
    tmp21 = 3.0517578125e-05
    tmp22 = tmp20 * tmp21
    tl.store(out_ptr2 + x0, tmp22, xmask)
    tl.store(out_ptr0 + x0, tmp13, xmask)
    tl.store(out_ptr1 + x0, tmp14, xmask)


@triton.jit
def triton_poi_fused_native_layer_norm_38(in_out_ptr0, in_ptr0, in_ptr1,
    xnumel, XBLOCK: tl.constexpr):
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:]
    tl.full([XBLOCK], True, tl.int1)
    x2 = xindex
    x1 = xindex // 32768
    tmp0 = tl.load(in_out_ptr0 + x2, None)
    tmp1 = tl.load(in_ptr0 + x1, None, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr1 + x1, None, eviction_policy='evict_last')
    tmp2 = tmp0 - tmp1
    tmp4 = 32768.0
    tmp5 = tmp3 / tmp4
    tmp6 = 1e-05
    tmp7 = tmp5 + tmp6
    tmp8 = libdevice.rsqrt(tmp7)
    tmp9 = tmp2 * tmp8
    tl.store(in_out_ptr0 + x2, tmp9, None)


@triton.jit
def triton_poi_fused_native_layer_norm_relu_39(in_ptr0, in_ptr1, in_ptr2,
    out_ptr0, ynumel, xnumel, YBLOCK: tl.constexpr, XBLOCK: tl.constexpr):
    xnumel = 64
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y0 = yindex % 512
    y1 = yindex // 512
    tmp0 = tl.load(in_ptr0 + (y0 + 512 * x2 + 32768 * y1), xmask,
        eviction_policy='evict_last')
    tmp1 = tl.load(in_ptr1 + (x2 + 64 * y0), xmask, eviction_policy=
        'evict_last')
    tmp3 = tl.load(in_ptr2 + (x2 + 64 * y0), xmask, eviction_policy=
        'evict_last')
    tmp2 = tmp0 * tmp1
    tmp4 = tmp2 + tmp3
    tmp5 = tl.full([1, 1], 0, tl.int32)
    tmp6 = triton_helpers.maximum(tmp5, tmp4)
    tl.store(out_ptr0 + (y0 + 512 * x2 + 32768 * y1), tmp6, xmask)


@triton.jit
def triton_per_fused_native_layer_norm_40(in_out_ptr0, in_ptr0, in_ptr1,
    in_ptr2, out_ptr0, xnumel, rnumel, XBLOCK: tl.constexpr):
    xnumel = 4
    RBLOCK: tl.constexpr = 4
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    rindex = tl.arange(0, RBLOCK)[None, :]
    tl.full([XBLOCK, RBLOCK], True, tl.int1)
    r1 = rindex
    x0 = xindex
    tmp0 = tl.load(in_ptr0 + (r1 + 4 * x0), xmask, other=0.0)
    tmp1 = tl.load(in_ptr1 + (r1 + 4 * x0), xmask, other=0.0)
    tmp2 = tl.load(in_ptr2 + (r1 + 4 * x0), xmask, other=0.0)
    tmp3 = tl.broadcast_to(tmp0, [XBLOCK, RBLOCK])
    tmp4 = tl.broadcast_to(tmp1, [XBLOCK, RBLOCK])
    tmp5 = tl.broadcast_to(tmp2, [XBLOCK, RBLOCK])
    tmp7 = tl.where(xmask, tmp3, 0)
    tmp8 = tl.where(xmask, tmp4, 0)
    tmp9 = tl.where(xmask, tmp5, 0)
    tmp10, tmp11, tmp12 = triton_helpers.welford(tmp7, tmp8, tmp9, 1)
    tmp13 = tmp10[:, None]
    tmp14 = tmp11[:, None]
    tmp12[:, None]
    tmp16 = 32768.0
    tmp17 = tmp14 / tmp16
    tmp18 = 1e-05
    tmp19 = tmp17 + tmp18
    tmp20 = libdevice.rsqrt(tmp19)
    tl.debug_barrier()
    tl.store(in_out_ptr0 + x0, tmp20, xmask)
    tl.store(out_ptr0 + x0, tmp13, xmask)


@triton.jit
def triton_poi_fused_native_layer_norm_relu_41(in_ptr0, in_ptr1, in_ptr2,
    in_ptr3, in_ptr4, out_ptr0, ynumel, xnumel, YBLOCK: tl.constexpr,
    XBLOCK: tl.constexpr):
    xnumel = 64
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    tl.full([XBLOCK, YBLOCK], True, tl.int1)
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x2 = xindex
    y0 = yindex % 512
    y1 = yindex // 512
    tmp0 = tl.load(in_ptr0 + (y0 + 512 * x2 + 32768 * y1), xmask,
        eviction_policy='evict_last')
    tmp1 = tl.load(in_ptr1 + y1, None, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr2 + y1, None, eviction_policy='evict_last')
    tmp5 = tl.load(in_ptr3 + (x2 + 64 * y0), xmask, eviction_policy=
        'evict_last')
    tmp7 = tl.load(in_ptr4 + (x2 + 64 * y0), xmask, eviction_policy=
        'evict_last')
    tmp2 = tmp0 - tmp1
    tmp4 = tmp2 * tmp3
    tmp6 = tmp4 * tmp5
    tmp8 = tmp6 + tmp7
    tmp9 = tl.full([1, 1], 0, tl.int32)
    tmp10 = triton_helpers.maximum(tmp9, tmp8)
    tl.store(out_ptr0 + (y0 + 512 * x2 + 32768 * y1), tmp10, xmask)


@triton.jit
def triton_poi_fused_add_convolution_div_42(in_ptr0, in_ptr1, in_ptr2,
    in_ptr3, out_ptr0, ynumel, xnumel, YBLOCK: tl.constexpr, XBLOCK: tl.
    constexpr):
    ynumel = 64
    xnumel = 512
    yoffset = tl.program_id(1) * YBLOCK
    yindex = yoffset + tl.arange(0, YBLOCK)[None, :]
    ymask = yindex < ynumel
    xoffset = tl.program_id(0) * XBLOCK
    xindex = xoffset + tl.arange(0, XBLOCK)[:, None]
    xmask = xindex < xnumel
    x3 = xindex
    y4 = yindex
    y0 = yindex % 4
    y5 = yindex // 4
    y2 = yindex // 16
    y6 = yindex % 16
    tmp0 = tl.load(in_ptr0 + (x3 + 512 * y4), xmask & ymask,
        eviction_policy='evict_last')
    tmp1 = tl.load(in_ptr1 + x3, xmask, eviction_policy='evict_last')
    tmp3 = tl.load(in_ptr2 + (x3 + 1024 * y0 + 8192 * y5), xmask & ymask,
        eviction_policy='evict_last')
    tmp4 = tl.load(in_ptr3 + x3, xmask, eviction_policy='evict_last')
    tmp6 = tl.load(in_ptr2 + (4096 + x3 + 1024 * y0 + 8192 * y5), xmask &
        ymask, eviction_policy='evict_last')
    tmp9 = tl.load(in_ptr2 + (512 + x3 + 1024 * y0 + 8192 * y5), xmask &
        ymask, eviction_policy='evict_last')
    tmp12 = tl.load(in_ptr2 + (4608 + x3 + 1024 * y0 + 8192 * y5), xmask &
        ymask, eviction_policy='evict_last')
    tmp2 = tmp0 + tmp1
    tmp5 = tmp3 + tmp4
    tmp7 = tmp6 + tmp4
    tmp8 = tmp5 + tmp7
    tmp10 = tmp9 + tmp4
    tmp11 = tmp8 + tmp10
    tmp13 = tmp12 + tmp4
    tmp14 = tmp11 + tmp13
    tmp15 = 0.25
    tmp16 = tmp14 * tmp15
    tmp17 = tmp2 + tmp16
    tl.store(out_ptr0 + (y6 + 16 * x3 + 8192 * y2), tmp17, xmask & ymask)


def call(args):
    (primals_1, primals_2, primals_3, primals_4, primals_5, primals_6,
        primals_7, primals_8, primals_9, primals_10, primals_11, primals_12,
        primals_13, primals_14, primals_15, primals_16, primals_17,
        primals_18, primals_19, primals_20, primals_21, primals_22,
        primals_23, primals_24, primals_25, primals_26, primals_27,
        primals_28, primals_29, primals_30, primals_31, primals_32,
        primals_33, primals_34, primals_35, primals_36, primals_37,
        primals_38, primals_39, primals_40, primals_41) = args
    args.clear()
    assert_size_stride(primals_1, (4, 3, 64, 64), (12288, 4096, 64, 1))
    assert_size_stride(primals_2, (64, 3, 3, 3), (27, 9, 3, 1))
    assert_size_stride(primals_3, (64,), (1,))
    assert_size_stride(primals_4, (128, 64, 1, 1), (64, 1, 1, 1))
    assert_size_stride(primals_5, (128,), (1,))
    assert_size_stride(primals_6, (64, 64, 64), (4096, 64, 1))
    assert_size_stride(primals_7, (64, 64, 64), (4096, 64, 1))
    assert_size_stride(primals_8, (64, 64, 3, 3), (576, 9, 3, 1))
    assert_size_stride(primals_9, (64, 64, 64), (4096, 64, 1))
    assert_size_stride(primals_10, (64, 64, 64), (4096, 64, 1))
    assert_size_stride(primals_11, (128, 64, 3, 3), (576, 9, 3, 1))
    assert_size_stride(primals_12, (128,), (1,))
    assert_size_stride(primals_13, (256, 128, 1, 1), (128, 1, 1, 1))
    assert_size_stride(primals_14, (256,), (1,))
    assert_size_stride(primals_15, (128, 32, 32), (1024, 32, 1))
    assert_size_stride(primals_16, (128, 32, 32), (1024, 32, 1))
    assert_size_stride(primals_17, (128, 128, 3, 3), (1152, 9, 3, 1))
    assert_size_stride(primals_18, (128, 32, 32), (1024, 32, 1))
    assert_size_stride(primals_19, (128, 32, 32), (1024, 32, 1))
    assert_size_stride(primals_20, (256, 128, 3, 3), (1152, 9, 3, 1))
    assert_size_stride(primals_21, (256,), (1,))
    assert_size_stride(primals_22, (512, 256, 1, 1), (256, 1, 1, 1))
    assert_size_stride(primals_23, (512,), (1,))
    assert_size_stride(primals_24, (256, 16, 16), (256, 16, 1))
    assert_size_stride(primals_25, (256, 16, 16), (256, 16, 1))
    assert_size_stride(primals_26, (256, 256, 3, 3), (2304, 9, 3, 1))
    assert_size_stride(primals_27, (256, 16, 16), (256, 16, 1))
    assert_size_stride(primals_28, (256, 16, 16), (256, 16, 1))
    assert_size_stride(primals_29, (512, 256, 3, 3), (2304, 9, 3, 1))
    assert_size_stride(primals_30, (512,), (1,))
    assert_size_stride(primals_31, (512, 512, 1, 1), (512, 1, 1, 1))
    assert_size_stride(primals_32, (512,), (1,))
    assert_size_stride(primals_33, (512, 8, 8), (64, 8, 1))
    assert_size_stride(primals_34, (512, 8, 8), (64, 8, 1))
    assert_size_stride(primals_35, (512, 512, 3, 3), (4608, 9, 3, 1))
    assert_size_stride(primals_36, (512, 8, 8), (64, 8, 1))
    assert_size_stride(primals_37, (512, 8, 8), (64, 8, 1))
    assert_size_stride(primals_38, (512, 512, 3, 3), (4608, 9, 3, 1))
    assert_size_stride(primals_39, (512,), (1,))
    assert_size_stride(primals_40, (1, 8192), (8192, 1))
    assert_size_stride(primals_41, (1,), (1,))
    with torch.cuda._DeviceGuard(0):
        torch.cuda.set_device(0)
        buf0 = empty_strided_cuda((64, 3, 3, 3), (27, 1, 9, 3), torch.float32)
        get_raw_stream(0)
        triton_poi_fused_0[grid(192, 9)](primals_2, buf0, 192, 9, XBLOCK=16,
            YBLOCK=64, num_warps=4, num_stages=1)
        del primals_2
        buf1 = empty_strided_cuda((64, 64, 3, 3), (576, 1, 192, 64), torch.
            float32)
        triton_poi_fused_1[grid(4096, 9)](primals_8, buf1, 4096, 9, XBLOCK=
            16, YBLOCK=64, num_warps=4, num_stages=1)
        del primals_8
        buf2 = empty_strided_cuda((128, 64, 3, 3), (576, 1, 192, 64), torch
            .float32)
        triton_poi_fused_2[grid(8192, 9)](primals_11, buf2, 8192, 9, XBLOCK
            =16, YBLOCK=64, num_warps=4, num_stages=1)
        del primals_11
        buf3 = empty_strided_cuda((128, 128, 3, 3), (1152, 1, 384, 128),
            torch.float32)
        triton_poi_fused_3[grid(16384, 9)](primals_17, buf3, 16384, 9,
            XBLOCK=16, YBLOCK=64, num_warps=4, num_stages=1)
        del primals_17
        buf4 = empty_strided_cuda((256, 128, 3, 3), (1152, 1, 384, 128),
            torch.float32)
        triton_poi_fused_4[grid(32768, 9)](primals_20, buf4, 32768, 9,
            XBLOCK=16, YBLOCK=64, num_warps=4, num_stages=1)
        del primals_20
        buf5 = empty_strided_cuda((256, 256, 3, 3), (2304, 1, 768, 256),
            torch.float32)
        triton_poi_fused_5[grid(65536, 9)](primals_26, buf5, 65536, 9,
            XBLOCK=16, YBLOCK=64, num_warps=4, num_stages=1)
        del primals_26
        buf6 = empty_strided_cuda((512, 256, 3, 3), (2304, 1, 768, 256),
            torch.float32)
        triton_poi_fused_6[grid(131072, 9)](primals_29, buf6, 131072, 9,
            XBLOCK=16, YBLOCK=64, num_warps=4, num_stages=1)
        del primals_29
        buf7 = empty_strided_cuda((512, 512, 3, 3), (4608, 1, 1536, 512),
            torch.float32)
        triton_poi_fused_7[grid(262144, 9)](primals_35, buf7, 262144, 9,
            XBLOCK=16, YBLOCK=64, num_warps=4, num_stages=1)
        del primals_35
        buf8 = empty_strided_cuda((512, 512, 3, 3), (4608, 1, 1536, 512),
            torch.float32)
        triton_poi_fused_7[grid(262144, 9)](primals_38, buf8, 262144, 9,
            XBLOCK=16, YBLOCK=64, num_warps=4, num_stages=1)
        del primals_38
        buf9 = empty_strided_cuda((4, 3, 64, 64), (12288, 1, 192, 3), torch
            .float32)
        triton_poi_fused_view_8[grid(12, 4096)](primals_1, buf9, 12, 4096,
            XBLOCK=64, YBLOCK=16, num_warps=4, num_stages=1)
        del primals_1
        buf10 = extern_kernels.convolution(buf9, buf0, stride=(1, 1),
            padding=(1, 1), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf10, (4, 64, 64, 64), (262144, 1, 4096, 64))
        buf11 = buf10
        del buf10
        triton_poi_fused_convolution_9[grid(1048576)](buf11, primals_3, 
            1048576, XBLOCK=1024, num_warps=4, num_stages=1)
        del primals_3
        buf12 = empty_strided_cuda((4, 64, 32, 32), (65536, 1, 2048, 64),
            torch.float32)
        triton_poi_fused_add_div_10[grid(262144)](buf11, buf12, 262144,
            XBLOCK=512, num_warps=8, num_stages=1)
        buf13 = extern_kernels.convolution(buf12, primals_4, stride=(1, 1),
            padding=(0, 0), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf13, (4, 128, 32, 32), (131072, 1, 4096, 128))
        buf14 = empty_strided_cuda((4, 1, 1, 1, 32, 64), (2048, 8192, 8192,
            8192, 1, 32), torch.float32)
        buf15 = empty_strided_cuda((4, 1, 1, 1, 32, 64), (2048, 8192, 8192,
            8192, 1, 32), torch.float32)
        buf16 = empty_strided_cuda((4, 1, 1, 1, 32, 64), (2048, 8192, 8192,
            8192, 1, 32), torch.float32)
        triton_per_fused_native_layer_norm_11[grid(8192)](buf11, buf14,
            buf15, buf16, 8192, 128, XBLOCK=8, num_warps=8, num_stages=1)
        buf17 = empty_strided_cuda((4, 1, 1, 1, 32), (32, 128, 128, 128, 1),
            torch.float32)
        buf18 = empty_strided_cuda((4, 1, 1, 1, 32), (32, 128, 128, 128, 1),
            torch.float32)
        buf19 = empty_strided_cuda((4, 1, 1, 1, 32), (32, 128, 128, 128, 1),
            torch.float32)
        triton_per_fused_native_layer_norm_12[grid(128)](buf14, buf15,
            buf16, buf17, buf18, buf19, 128, 64, XBLOCK=1, num_warps=2,
            num_stages=1)
        buf20 = empty_strided_cuda((4, 1, 1, 1), (1, 1, 1, 1), torch.float32)
        buf21 = empty_strided_cuda((4, 1, 1, 1), (1, 4, 4, 4), torch.float32)
        buf23 = reinterpret_tensor(buf21, (4, 1, 1, 1), (1, 1, 1, 1), 0)
        del buf21
        triton_per_fused_native_layer_norm_13[grid(4)](buf23, buf17, buf18,
            buf19, buf20, 4, 32, XBLOCK=1, num_warps=2, num_stages=1)
        buf24 = empty_strided_cuda((4, 64, 64, 64), (262144, 1, 4096, 64),
            torch.float32)
        triton_poi_fused_native_layer_norm_relu_14[grid(256, 4096)](buf11,
            buf20, buf23, primals_6, primals_7, buf24, 256, 4096, XBLOCK=32,
            YBLOCK=32, num_warps=4, num_stages=1)
        del primals_7
        buf25 = extern_kernels.convolution(buf24, buf1, stride=(1, 1),
            padding=(1, 1), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf25, (4, 64, 64, 64), (262144, 1, 4096, 64))
        buf26 = buf16
        del buf16
        buf27 = buf15
        del buf15
        buf28 = buf14
        del buf14
        triton_per_fused_native_layer_norm_11[grid(8192)](buf25, buf26,
            buf27, buf28, 8192, 128, XBLOCK=8, num_warps=8, num_stages=1)
        buf29 = buf19
        del buf19
        buf30 = buf18
        del buf18
        buf31 = buf17
        del buf17
        triton_per_fused_native_layer_norm_12[grid(128)](buf26, buf27,
            buf28, buf29, buf30, buf31, 128, 64, XBLOCK=1, num_warps=2,
            num_stages=1)
        del buf26
        del buf27
        del buf28
        buf32 = empty_strided_cuda((4, 1, 1, 1), (1, 1, 1, 1), torch.float32)
        buf33 = empty_strided_cuda((4, 1, 1, 1), (1, 4, 4, 4), torch.float32)
        buf35 = reinterpret_tensor(buf33, (4, 1, 1, 1), (1, 1, 1, 1), 0)
        del buf33
        triton_per_fused_native_layer_norm_13[grid(4)](buf35, buf29, buf30,
            buf31, buf32, 4, 32, XBLOCK=1, num_warps=2, num_stages=1)
        del buf29
        del buf30
        del buf31
        buf36 = empty_strided_cuda((4, 64, 64, 64), (262144, 1, 4096, 64),
            torch.float32)
        triton_poi_fused_native_layer_norm_relu_14[grid(256, 4096)](buf25,
            buf32, buf35, primals_9, primals_10, buf36, 256, 4096, XBLOCK=
            32, YBLOCK=32, num_warps=4, num_stages=1)
        del primals_10
        buf37 = extern_kernels.convolution(buf36, buf2, stride=(1, 1),
            padding=(1, 1), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf37, (4, 128, 64, 64), (524288, 1, 8192, 128))
        buf38 = buf13
        del buf13
        triton_poi_fused_add_convolution_div_15[grid(524288)](buf38,
            primals_5, buf37, primals_12, 524288, XBLOCK=512, num_warps=8,
            num_stages=1)
        del buf37
        del primals_12
        del primals_5
        buf39 = empty_strided_cuda((4, 128, 16, 16), (32768, 1, 2048, 128),
            torch.float32)
        triton_poi_fused_add_div_16[grid(131072)](buf38, buf39, 131072,
            XBLOCK=512, num_warps=8, num_stages=1)
        buf40 = extern_kernels.convolution(buf39, primals_13, stride=(1, 1),
            padding=(0, 0), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf40, (4, 256, 16, 16), (65536, 1, 4096, 256))
        buf41 = empty_strided_cuda((4, 1, 1, 1, 16, 64), (1024, 4096, 4096,
            4096, 1, 16), torch.float32)
        buf42 = empty_strided_cuda((4, 1, 1, 1, 16, 64), (1024, 4096, 4096,
            4096, 1, 16), torch.float32)
        buf43 = empty_strided_cuda((4, 1, 1, 1, 16, 64), (1024, 4096, 4096,
            4096, 1, 16), torch.float32)
        triton_per_fused_native_layer_norm_17[grid(4096)](buf38, buf41,
            buf42, buf43, 4096, 128, XBLOCK=8, num_warps=8, num_stages=1)
        buf44 = empty_strided_cuda((4, 1, 1, 1, 16), (16, 64, 64, 64, 1),
            torch.float32)
        buf45 = empty_strided_cuda((4, 1, 1, 1, 16), (16, 64, 64, 64, 1),
            torch.float32)
        buf46 = empty_strided_cuda((4, 1, 1, 1, 16), (16, 64, 64, 64, 1),
            torch.float32)
        triton_per_fused_native_layer_norm_18[grid(64)](buf41, buf42, buf43,
            buf44, buf45, buf46, 64, 64, XBLOCK=1, num_warps=2, num_stages=1)
        buf47 = empty_strided_cuda((4, 1, 1, 1), (1, 4, 4, 4), torch.float32)
        buf48 = empty_strided_cuda((4, 1, 1, 1), (1, 4, 4, 4), torch.float32)
        buf124 = empty_strided_cuda((4, 1, 1, 1), (1, 1, 1, 1), torch.float32)
        triton_per_fused_native_layer_norm_native_layer_norm_backward_19[grid
            (4)](buf44, buf45, buf46, buf47, buf48, buf124, 4, 16, XBLOCK=1,
            num_warps=2, num_stages=1)
        buf50 = buf38
        del buf38
        triton_poi_fused_native_layer_norm_20[grid(524288)](buf50, buf47,
            buf48, 524288, XBLOCK=1024, num_warps=4, num_stages=1)
        buf51 = empty_strided_cuda((4, 128, 32, 32), (131072, 1, 4096, 128),
            torch.float32)
        triton_poi_fused_native_layer_norm_relu_21[grid(512, 1024)](buf50,
            primals_15, primals_16, buf51, 512, 1024, XBLOCK=32, YBLOCK=32,
            num_warps=4, num_stages=1)
        del primals_16
        buf52 = extern_kernels.convolution(buf51, buf3, stride=(1, 1),
            padding=(1, 1), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf52, (4, 128, 32, 32), (131072, 1, 4096, 128))
        buf53 = buf43
        del buf43
        buf54 = buf42
        del buf42
        buf55 = buf41
        del buf41
        triton_per_fused_native_layer_norm_17[grid(4096)](buf52, buf53,
            buf54, buf55, 4096, 128, XBLOCK=8, num_warps=8, num_stages=1)
        buf56 = buf46
        del buf46
        buf57 = buf45
        del buf45
        buf58 = buf44
        del buf44
        triton_per_fused_native_layer_norm_18[grid(64)](buf53, buf54, buf55,
            buf56, buf57, buf58, 64, 64, XBLOCK=1, num_warps=2, num_stages=1)
        del buf53
        del buf54
        del buf55
        buf59 = reinterpret_tensor(buf48, (4, 1, 1, 1), (1, 1, 1, 1), 0)
        del buf48
        buf60 = buf47
        del buf47
        buf62 = reinterpret_tensor(buf60, (4, 1, 1, 1), (1, 1, 1, 1), 0)
        del buf60
        triton_per_fused_native_layer_norm_22[grid(4)](buf62, buf56, buf57,
            buf58, buf59, 4, 16, XBLOCK=1, num_warps=2, num_stages=1)
        del buf56
        del buf57
        del buf58
        buf63 = empty_strided_cuda((4, 128, 32, 32), (131072, 1, 4096, 128),
            torch.float32)
        triton_poi_fused_native_layer_norm_relu_23[grid(512, 1024)](buf52,
            buf59, buf62, primals_18, primals_19, buf63, 512, 1024, XBLOCK=
            32, YBLOCK=32, num_warps=4, num_stages=1)
        del primals_19
        buf64 = extern_kernels.convolution(buf63, buf4, stride=(1, 1),
            padding=(1, 1), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf64, (4, 256, 32, 32), (262144, 1, 8192, 256))
        buf65 = buf40
        del buf40
        triton_poi_fused_add_convolution_div_24[grid(262144)](buf65,
            primals_14, buf64, primals_21, 262144, XBLOCK=512, num_warps=8,
            num_stages=1)
        del buf64
        del primals_14
        del primals_21
        buf66 = empty_strided_cuda((4, 256, 8, 8), (16384, 1, 2048, 256),
            torch.float32)
        triton_poi_fused_add_div_25[grid(65536)](buf65, buf66, 65536,
            XBLOCK=512, num_warps=4, num_stages=1)
        buf67 = extern_kernels.convolution(buf66, primals_22, stride=(1, 1),
            padding=(0, 0), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf67, (4, 512, 8, 8), (32768, 1, 4096, 512))
        buf68 = empty_strided_cuda((4, 1, 1, 1, 8, 64), (512, 2048, 2048, 
            2048, 1, 8), torch.float32)
        buf69 = empty_strided_cuda((4, 1, 1, 1, 8, 64), (512, 2048, 2048, 
            2048, 1, 8), torch.float32)
        buf70 = empty_strided_cuda((4, 1, 1, 1, 8, 64), (512, 2048, 2048, 
            2048, 1, 8), torch.float32)
        triton_per_fused_native_layer_norm_26[grid(2048)](buf65, buf68,
            buf69, buf70, 2048, 128, XBLOCK=32, num_warps=8, num_stages=1)
        buf71 = empty_strided_cuda((4, 1, 1, 1, 8), (8, 32, 32, 32, 1),
            torch.float32)
        buf72 = empty_strided_cuda((4, 1, 1, 1, 8), (8, 32, 32, 32, 1),
            torch.float32)
        buf73 = empty_strided_cuda((4, 1, 1, 1, 8), (8, 32, 32, 32, 1),
            torch.float32)
        triton_per_fused_native_layer_norm_27[grid(32)](buf68, buf69, buf70,
            buf71, buf72, buf73, 32, 64, XBLOCK=1, num_warps=2, num_stages=1)
        buf74 = empty_strided_cuda((4, 1, 1, 1), (1, 4, 4, 4), torch.float32)
        buf75 = empty_strided_cuda((4, 1, 1, 1), (1, 4, 4, 4), torch.float32)
        buf123 = empty_strided_cuda((4, 1, 1, 1), (1, 1, 1, 1), torch.float32)
        triton_per_fused_native_layer_norm_native_layer_norm_backward_28[grid
            (4)](buf71, buf72, buf73, buf74, buf75, buf123, 4, 8, XBLOCK=1,
            num_warps=2, num_stages=1)
        buf77 = buf65
        del buf65
        triton_poi_fused_native_layer_norm_29[grid(262144)](buf77, buf74,
            buf75, 262144, XBLOCK=1024, num_warps=4, num_stages=1)
        buf78 = empty_strided_cuda((4, 256, 16, 16), (65536, 1, 4096, 256),
            torch.float32)
        triton_poi_fused_native_layer_norm_relu_30[grid(1024, 256)](buf77,
            primals_24, primals_25, buf78, 1024, 256, XBLOCK=32, YBLOCK=32,
            num_warps=4, num_stages=1)
        del primals_25
        buf79 = extern_kernels.convolution(buf78, buf5, stride=(1, 1),
            padding=(1, 1), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf79, (4, 256, 16, 16), (65536, 1, 4096, 256))
        buf80 = buf70
        del buf70
        buf81 = buf69
        del buf69
        buf82 = buf68
        del buf68
        triton_per_fused_native_layer_norm_26[grid(2048)](buf79, buf80,
            buf81, buf82, 2048, 128, XBLOCK=32, num_warps=8, num_stages=1)
        buf83 = buf73
        del buf73
        buf84 = buf72
        del buf72
        buf85 = buf71
        del buf71
        triton_per_fused_native_layer_norm_27[grid(32)](buf80, buf81, buf82,
            buf83, buf84, buf85, 32, 64, XBLOCK=1, num_warps=2, num_stages=1)
        del buf80
        del buf81
        del buf82
        buf86 = reinterpret_tensor(buf75, (4, 1, 1, 1), (1, 1, 1, 1), 0)
        del buf75
        buf87 = buf74
        del buf74
        buf89 = reinterpret_tensor(buf87, (4, 1, 1, 1), (1, 1, 1, 1), 0)
        del buf87
        triton_per_fused_native_layer_norm_31[grid(4)](buf89, buf83, buf84,
            buf85, buf86, 4, 8, XBLOCK=1, num_warps=2, num_stages=1)
        del buf83
        del buf84
        del buf85
        buf90 = empty_strided_cuda((4, 256, 16, 16), (65536, 1, 4096, 256),
            torch.float32)
        triton_poi_fused_native_layer_norm_relu_32[grid(1024, 256)](buf79,
            buf86, buf89, primals_27, primals_28, buf90, 1024, 256, XBLOCK=
            32, YBLOCK=32, num_warps=4, num_stages=1)
        del primals_28
        buf91 = extern_kernels.convolution(buf90, buf6, stride=(1, 1),
            padding=(1, 1), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf91, (4, 512, 16, 16), (131072, 1, 8192, 512))
        buf92 = buf67
        del buf67
        triton_poi_fused_add_convolution_div_33[grid(131072)](buf92,
            primals_23, buf91, primals_30, 131072, XBLOCK=512, num_warps=8,
            num_stages=1)
        del buf91
        del primals_23
        del primals_30
        buf93 = empty_strided_cuda((4, 512, 4, 4), (8192, 1, 2048, 512),
            torch.float32)
        triton_poi_fused_add_div_34[grid(32768)](buf92, buf93, 32768,
            XBLOCK=256, num_warps=4, num_stages=1)
        buf94 = extern_kernels.convolution(buf93, primals_31, stride=(1, 1),
            padding=(0, 0), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf94, (4, 512, 4, 4), (8192, 1, 2048, 512))
        buf95 = empty_strided_cuda((4, 1, 1, 1, 4, 64), (256, 1024, 1024, 
            1024, 64, 1), torch.float32)
        buf96 = empty_strided_cuda((4, 1, 1, 1, 4, 64), (256, 1024, 1024, 
            1024, 64, 1), torch.float32)
        buf97 = empty_strided_cuda((4, 1, 1, 1, 4, 64), (256, 1024, 1024, 
            1024, 64, 1), torch.float32)
        triton_per_fused_native_layer_norm_35[grid(1024)](buf92, buf95,
            buf96, buf97, 1024, 128, XBLOCK=8, num_warps=8, num_stages=1)
        buf98 = empty_strided_cuda((4, 1, 1, 1, 4), (4, 16, 16, 16, 1),
            torch.float32)
        buf99 = empty_strided_cuda((4, 1, 1, 1, 4), (4, 16, 16, 16, 1),
            torch.float32)
        buf100 = empty_strided_cuda((4, 1, 1, 1, 4), (4, 16, 16, 16, 1),
            torch.float32)
        triton_per_fused_native_layer_norm_36[grid(16)](buf95, buf96, buf97,
            buf98, buf99, buf100, 16, 64, XBLOCK=8, num_warps=4, num_stages=1)
        buf101 = empty_strided_cuda((4, 1, 1, 1), (1, 4, 4, 4), torch.float32)
        buf102 = empty_strided_cuda((4, 1, 1, 1), (1, 4, 4, 4), torch.float32)
        buf122 = empty_strided_cuda((4, 1, 1, 1), (1, 1, 1, 1), torch.float32)
        triton_per_fused_native_layer_norm_native_layer_norm_backward_37[grid
            (4)](buf98, buf99, buf100, buf101, buf102, buf122, 4, 4, XBLOCK
            =1, num_warps=2, num_stages=1)
        buf104 = buf92
        del buf92
        triton_poi_fused_native_layer_norm_38[grid(131072)](buf104, buf101,
            buf102, 131072, XBLOCK=512, num_warps=8, num_stages=1)
        buf105 = empty_strided_cuda((4, 512, 8, 8), (32768, 1, 4096, 512),
            torch.float32)
        triton_poi_fused_native_layer_norm_relu_39[grid(2048, 64)](buf104,
            primals_33, primals_34, buf105, 2048, 64, XBLOCK=32, YBLOCK=32,
            num_warps=4, num_stages=1)
        del primals_34
        buf106 = extern_kernels.convolution(buf105, buf7, stride=(1, 1),
            padding=(1, 1), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf106, (4, 512, 8, 8), (32768, 1, 4096, 512))
        buf107 = buf97
        del buf97
        buf108 = buf96
        del buf96
        buf109 = buf95
        del buf95
        triton_per_fused_native_layer_norm_35[grid(1024)](buf106, buf107,
            buf108, buf109, 1024, 128, XBLOCK=8, num_warps=8, num_stages=1)
        buf110 = buf99
        del buf99
        buf111 = buf98
        del buf98
        buf112 = buf100
        del buf100
        triton_per_fused_native_layer_norm_36[grid(16)](buf107, buf108,
            buf109, buf110, buf111, buf112, 16, 64, XBLOCK=8, num_warps=4,
            num_stages=1)
        del buf107
        del buf108
        del buf109
        buf113 = reinterpret_tensor(buf102, (4, 1, 1, 1), (1, 1, 1, 1), 0)
        del buf102
        buf114 = buf101
        del buf101
        buf116 = reinterpret_tensor(buf114, (4, 1, 1, 1), (1, 1, 1, 1), 0)
        del buf114
        triton_per_fused_native_layer_norm_40[grid(4)](buf116, buf110,
            buf111, buf112, buf113, 4, 4, XBLOCK=1, num_warps=2, num_stages=1)
        del buf110
        del buf111
        del buf112
        buf117 = empty_strided_cuda((4, 512, 8, 8), (32768, 1, 4096, 512),
            torch.float32)
        triton_poi_fused_native_layer_norm_relu_41[grid(2048, 64)](buf106,
            buf113, buf116, primals_36, primals_37, buf117, 2048, 64,
            XBLOCK=32, YBLOCK=32, num_warps=4, num_stages=1)
        del primals_37
        buf118 = extern_kernels.convolution(buf117, buf8, stride=(1, 1),
            padding=(1, 1), dilation=(1, 1), transposed=False,
            output_padding=(0, 0), groups=1, bias=None)
        assert_size_stride(buf118, (4, 512, 8, 8), (32768, 1, 4096, 512))
        buf119 = empty_strided_cuda((4, 512, 4, 4), (8192, 16, 4, 1), torch
            .float32)
        triton_poi_fused_add_convolution_div_42[grid(64, 512)](buf94,
            primals_32, buf118, primals_39, buf119, 64, 512, XBLOCK=4,
            YBLOCK=64, num_warps=4, num_stages=1)
        del buf118
        del buf94
        del primals_32
        del primals_39
        buf121 = empty_strided_cuda((4, 1), (1, 1), torch.float32)
        extern_kernels.addmm(primals_41, reinterpret_tensor(buf119, (4, 
            8192), (8192, 1), 0), reinterpret_tensor(primals_40, (8192, 1),
            (1, 8192), 0), alpha=1, beta=1, out=buf121)
        del primals_41
    return (reinterpret_tensor(buf121, (4,), (1,), 0), buf0, primals_4,
        primals_6, buf1, primals_9, buf2, primals_13, primals_15, buf3,
        primals_18, buf4, primals_22, primals_24, buf5, primals_27, buf6,
        primals_31, primals_33, buf7, primals_36, buf8, buf9, buf11, buf12,
        buf20, buf23, buf24, buf25, buf32, buf35, buf36, buf39, buf50,
        buf51, buf52, buf59, buf62, buf63, buf66, buf77, buf78, buf79,
        buf86, buf89, buf90, buf93, buf104, buf105, buf106, buf113, buf116,
        buf117, reinterpret_tensor(buf119, (4, 8192), (8192, 1), 0),
        primals_40, buf122, buf123, buf124)


class IWConv2d(nn.Module):

    def __init__(self, input_dim, output_dim, kernel_size, he_init=True,
        stride=1, bias=True):
        super(IWConv2d, self).__init__()
        self.he_init = he_init
        self.padding = int((kernel_size - 1) / 2)
        self.conv = nn.Conv2d(input_dim, output_dim, kernel_size, stride=1,
            padding=self.padding, bias=bias)

    def forward(self, input):
        output = self.conv(input)
        return output


class ConvMeanPool(nn.Module):

    def __init__(self, input_dim, output_dim, kernel_size, he_init=True):
        super(ConvMeanPool, self).__init__()
        self.he_init = he_init
        self.conv = IWConv2d(input_dim, output_dim, kernel_size, he_init=
            self.he_init)

    def forward(self, input):
        output = self.conv(input)
        output = (output[:, :, ::2, ::2] + output[:, :, 1::2, ::2] + output
            [:, :, ::2, 1::2] + output[:, :, 1::2, 1::2]) / 4
        return output


class MeanPoolConv(nn.Module):

    def __init__(self, input_dim, output_dim, kernel_size, he_init=True):
        super(MeanPoolConv, self).__init__()
        self.he_init = he_init
        self.conv = IWConv2d(input_dim, output_dim, kernel_size, he_init=
            self.he_init)

    def forward(self, input):
        output = input
        output = (output[:, :, ::2, ::2] + output[:, :, 1::2, ::2] + output
            [:, :, ::2, 1::2] + output[:, :, 1::2, 1::2]) / 4
        output = self.conv(output)
        return output


class DepthToSpace(nn.Module):

    def __init__(self, block_size):
        super(DepthToSpace, self).__init__()
        self.block_size = block_size
        self.block_size_sq = block_size * block_size

    def forward(self, input):
        output = input.permute(0, 2, 3, 1)
        batch_size, input_height, input_width, input_depth = output.size()
        output_depth = int(input_depth / self.block_size_sq)
        output_width = int(input_width * self.block_size)
        output_height = int(input_height * self.block_size)
        t_1 = output.reshape(batch_size, input_height, input_width, self.
            block_size_sq, output_depth)
        spl = t_1.split(self.block_size, 3)
        stacks = [t_t.reshape(batch_size, input_height, output_width,
            output_depth) for t_t in spl]
        output = torch.stack(stacks, 0).transpose(0, 1).permute(0, 2, 1, 3, 4
            ).reshape(batch_size, output_height, output_width, output_depth)
        output = output.permute(0, 3, 1, 2)
        return output


class UpSampleConv(nn.Module):

    def __init__(self, input_dim, output_dim, kernel_size, he_init=True,
        bias=True):
        super(UpSampleConv, self).__init__()
        self.he_init = he_init
        self.conv = IWConv2d(input_dim, output_dim, kernel_size, he_init=
            self.he_init, bias=bias)
        self.depth_to_space = DepthToSpace(2)

    def forward(self, input):
        output = input
        output = torch.cat((output, output, output, output), 1)
        output = self.depth_to_space(output)
        output = self.conv(output)
        return output


class ResidualBlock(nn.Module):

    def __init__(self, input_dim, output_dim, kernel_size, resample=None, hw=64
        ):
        super(ResidualBlock, self).__init__()
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.kernel_size = kernel_size
        self.resample = resample
        self.bn1 = None
        self.bn2 = None
        self.relu1 = nn.ReLU()
        self.relu2 = nn.ReLU()
        if resample == 'down':
            self.bn1 = nn.LayerNorm([input_dim, hw, hw])
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        elif resample == 'up':
            self.bn1 = nn.BatchNorm2d(input_dim)
            self.bn2 = nn.BatchNorm2d(output_dim)
        elif resample is None:
            self.bn1 = nn.BatchNorm2d(output_dim)
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        else:
            raise Exception('invalid resample value')
        if resample == 'down':
            self.conv_shortcut = MeanPoolConv(input_dim, output_dim,
                kernel_size=1, he_init=False)
            self.conv_1 = IWConv2d(input_dim, input_dim, kernel_size=
                kernel_size, bias=False)
            self.conv_2 = ConvMeanPool(input_dim, output_dim, kernel_size=
                kernel_size)
        elif resample == 'up':
            self.conv_shortcut = UpSampleConv(input_dim, output_dim,
                kernel_size=1, he_init=False)
            self.conv_1 = UpSampleConv(input_dim, output_dim, kernel_size=
                kernel_size, bias=False)
            self.conv_2 = IWConv2d(output_dim, output_dim, kernel_size=
                kernel_size)
        elif resample is None:
            self.conv_shortcut = IWConv2d(input_dim, output_dim,
                kernel_size=1, he_init=False)
            self.conv_1 = IWConv2d(input_dim, input_dim, kernel_size=
                kernel_size, bias=False)
            self.conv_2 = IWConv2d(input_dim, output_dim, kernel_size=
                kernel_size)
        else:
            raise Exception('invalid resample value')

    def forward(self, input):
        if self.input_dim == self.output_dim and self.resample is None:
            shortcut = input
        else:
            shortcut = self.conv_shortcut(input)
        output = input
        output = self.bn1(output)
        output = self.relu1(output)
        output = self.conv_1(output)
        output = self.bn2(output)
        output = self.relu2(output)
        output = self.conv_2(output)
        return shortcut + output


class IWDiscriminatorNew(nn.Module):

    def __init__(self, input_size=64, n_image_channels=3):
        super(IWDiscriminatorNew, self).__init__()
        self.size = input_size
        self.n_image_channels = n_image_channels
        self.ssize = self.size // 16
        self.conv1 = IWConv2d(n_image_channels, self.size, 3, he_init=False)
        self.rb1 = ResidualBlock(self.size, 2 * self.size, 3, resample=
            'down', hw=self.size)
        self.rb2 = ResidualBlock(2 * self.size, 4 * self.size, 3, resample=
            'down', hw=int(self.size / 2))
        self.rb3 = ResidualBlock(4 * self.size, 8 * self.size, 3, resample=
            'down', hw=int(self.size / 4))
        self.rb4 = ResidualBlock(8 * self.size, 8 * self.size, 3, resample=
            'down', hw=int(self.size / 8))
        self.ln1 = nn.Linear(self.ssize * self.ssize * 8 * self.size, 1)

    def forward_last_feature(self, input):
        output = input.contiguous()
        output = output.view(-1, self.n_image_channels, self.size, self.size)
        output = self.conv1(output)
        output = self.rb1(output)
        output = self.rb2(output)
        output = self.rb3(output)
        output = self.rb4(output)
        output = output.view(-1, self.ssize * self.ssize * 8 * self.size)
        out_features = output
        output = self.ln1(output)
        output = output.view(-1)
        return output, out_features

    def forward_all_feature(self, input):
        out_features_list = []
        output = input.contiguous()
        output = output.view(-1, self.n_image_channels, self.size, self.size)
        output = self.conv1(output)
        out_features_list.append(output)
        output = self.rb1(output)
        out_features_list.append(output)
        output = self.rb2(output)
        out_features_list.append(output)
        output = self.rb3(output)
        out_features_list.append(output)
        output = self.rb4(output)
        output = output.view(-1, self.ssize * self.ssize * 8 * self.size)
        out_features_list.append(output)
        output = self.ln1(output)
        out_features_list.append(output)
        output = output.view(-1)
        return output, out_features_list

    def forward(self, input_0):
        primals_2 = self.conv1.conv.weight
        primals_3 = self.conv1.conv.bias
        primals_6 = self.rb1.bn1.weight
        primals_7 = self.rb1.bn1.bias
        primals_9 = self.rb1.bn2.weight
        primals_10 = self.rb1.bn2.bias
        primals_4 = self.rb1.conv_shortcut.conv.conv.weight
        primals_5 = self.rb1.conv_shortcut.conv.conv.bias
        primals_8 = self.rb1.conv_1.conv.weight
        primals_11 = self.rb1.conv_2.conv.conv.weight
        primals_12 = self.rb1.conv_2.conv.conv.bias
        primals_15 = self.rb2.bn1.weight
        primals_16 = self.rb2.bn1.bias
        primals_18 = self.rb2.bn2.weight
        primals_19 = self.rb2.bn2.bias
        primals_13 = self.rb2.conv_shortcut.conv.conv.weight
        primals_14 = self.rb2.conv_shortcut.conv.conv.bias
        primals_17 = self.rb2.conv_1.conv.weight
        primals_20 = self.rb2.conv_2.conv.conv.weight
        primals_21 = self.rb2.conv_2.conv.conv.bias
        primals_24 = self.rb3.bn1.weight
        primals_25 = self.rb3.bn1.bias
        primals_27 = self.rb3.bn2.weight
        primals_28 = self.rb3.bn2.bias
        primals_22 = self.rb3.conv_shortcut.conv.conv.weight
        primals_23 = self.rb3.conv_shortcut.conv.conv.bias
        primals_26 = self.rb3.conv_1.conv.weight
        primals_29 = self.rb3.conv_2.conv.conv.weight
        primals_30 = self.rb3.conv_2.conv.conv.bias
        primals_33 = self.rb4.bn1.weight
        primals_34 = self.rb4.bn1.bias
        primals_36 = self.rb4.bn2.weight
        primals_37 = self.rb4.bn2.bias
        primals_31 = self.rb4.conv_shortcut.conv.conv.weight
        primals_32 = self.rb4.conv_shortcut.conv.conv.bias
        primals_35 = self.rb4.conv_1.conv.weight
        primals_38 = self.rb4.conv_2.conv.conv.weight
        primals_39 = self.rb4.conv_2.conv.conv.bias
        primals_40 = self.ln1.weight
        primals_41 = self.ln1.bias
        primals_1 = input_0
        output = call([primals_1, primals_2, primals_3, primals_4,
            primals_5, primals_6, primals_7, primals_8, primals_9,
            primals_10, primals_11, primals_12, primals_13, primals_14,
            primals_15, primals_16, primals_17, primals_18, primals_19,
            primals_20, primals_21, primals_22, primals_23, primals_24,
            primals_25, primals_26, primals_27, primals_28, primals_29,
            primals_30, primals_31, primals_32, primals_33, primals_34,
            primals_35, primals_36, primals_37, primals_38, primals_39,
            primals_40, primals_41])
        return output[0]


Measured latency_ms: 0.033488