diff --git "a/sequoia/Llama-2-7b-hf_chunk9.mlmodelc/model.mil" "b/sequoia/Llama-2-7b-hf_chunk9.mlmodelc/model.mil" --- "a/sequoia/Llama-2-7b-hf_chunk9.mlmodelc/model.mil" +++ "b/sequoia/Llama-2-7b-hf_chunk9.mlmodelc/model.mil" @@ -1,7 +1,7 @@ program(1.3) -[buildInfo = dict({{"coremlc-component-MIL", "3400.34.1"}, {"coremlc-version", "3400.42.1"}})] +[buildInfo = dict({{"coremlc-component-MIL", "3400.42.1"}, {"coremlc-version", "3400.51.1"}})] { - func input_1_context_512(tensor cos, tensor k_cache_0, tensor k_cache_1, tensor k_cache_2, tensor mask, tensor sin, tensor v_cache_0, tensor v_cache_1, tensor v_cache_2, tensor x) [CoreML_InputDefaultValues = dict({{"k_cache_0", 0}, {"k_cache_1", 0}, {"k_cache_2", 0}, {"v_cache_0", 0}, {"v_cache_1", 0}, {"v_cache_2", 0}})] { + func input_1_context_512(tensor cos, tensor k_cache_0, tensor k_cache_1, tensor k_cache_2, tensor mask, tensor sin, tensor v_cache_0, tensor v_cache_1, tensor v_cache_2, tensor x) [CoreML_InputDefaultValues = dict({{"k_cache_0", 0}, {"k_cache_1", 0}, {"k_cache_2", 0}, {"v_cache_0", 0}, {"v_cache_1", 0}, {"v_cache_2", 0}})] { tensor blocks_0_attn_q_proj_weight_palettized_cast_fp16 = constexpr_lut_to_dense(indices = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64))), lut = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303873792))))[name = string("blocks_0_attn_q_proj_weight_palettized_cast_fp16")]; tensor blocks_0_attn_k_proj_weight_palettized_cast_fp16 = constexpr_lut_to_dense(indices = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8388864))), lut = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303873920))))[name = string("blocks_0_attn_k_proj_weight_palettized_cast_fp16")]; tensor blocks_0_attn_v_proj_weight_palettized_cast_fp16 = constexpr_lut_to_dense(indices = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16777664))), lut = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303874048))))[name = string("blocks_0_attn_v_proj_weight_palettized_cast_fp16")]; @@ -29,438 +29,450 @@ program(1.3) int32 var_34 = const()[name = string("op_34"), val = int32(-2)]; bool var_35 = const()[name = string("op_35"), val = bool(true)]; tensor var_53_axes_0 = const()[name = string("op_53_axes_0"), val = tensor([-2])]; - tensor var_53_cast_fp16 = squeeze(axes = var_53_axes_0, x = x)[name = string("op_53_cast_fp16")]; + tensor var_53_cast_fp16 = squeeze(axes = var_53_axes_0, x = x)[name = string("op_53_cast_fp16")]; bool var_55_interleave_0 = const()[name = string("op_55_interleave_0"), val = bool(false)]; - tensor eps_chan_1_to_fp16 = const()[name = string("eps_chan_1_to_fp16"), val = tensor([[[0x1.9e8p-3]]])]; - tensor var_55_cast_fp16 = concat(axis = var_31, interleave = var_55_interleave_0, values = (var_53_cast_fp16, eps_chan_1_to_fp16))[name = string("op_55_cast_fp16")]; + tensor eps_chan_1_to_fp16 = const()[name = string("eps_chan_1_to_fp16"), val = tensor([[[0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3]]])]; + tensor var_55_cast_fp16 = concat(axis = var_31, interleave = var_55_interleave_0, values = (var_53_cast_fp16, eps_chan_1_to_fp16))[name = string("op_55_cast_fp16")]; tensor x_eps_1_axes_0 = const()[name = string("x_eps_1_axes_0"), val = tensor([-2])]; - tensor x_eps_1_cast_fp16 = expand_dims(axes = x_eps_1_axes_0, x = var_55_cast_fp16)[name = string("x_eps_1_cast_fp16")]; + tensor x_eps_1_cast_fp16 = expand_dims(axes = x_eps_1_axes_0, x = var_55_cast_fp16)[name = string("x_eps_1_cast_fp16")]; tensor norm_x_1_axes_0 = const()[name = string("norm_x_1_axes_0"), val = tensor([1])]; - tensor norm_x_1_cast_fp16 = reduce_l2_norm(axes = norm_x_1_axes_0, keep_dims = var_35, x = x_eps_1_cast_fp16)[name = string("norm_x_1_cast_fp16")]; - tensor x_normed_1_cast_fp16 = real_div(x = x, y = norm_x_1_cast_fp16)[name = string("x_normed_1_cast_fp16")]; + tensor norm_x_1_cast_fp16 = reduce_l2_norm(axes = norm_x_1_axes_0, keep_dims = var_35, x = x_eps_1_cast_fp16)[name = string("norm_x_1_cast_fp16")]; + tensor x_normed_1_cast_fp16 = real_div(x = x, y = norm_x_1_cast_fp16)[name = string("x_normed_1_cast_fp16")]; fp16 var_60_to_fp16 = const()[name = string("op_60_to_fp16"), val = fp16(0x1p+6)]; - tensor x_normed_3_cast_fp16 = mul(x = x_normed_1_cast_fp16, y = var_60_to_fp16)[name = string("x_normed_3_cast_fp16")]; + tensor x_normed_3_cast_fp16 = mul(x = x_normed_1_cast_fp16, y = var_60_to_fp16)[name = string("x_normed_3_cast_fp16")]; tensor blocks_0_norm_1_weight_to_fp16 = const()[name = string("blocks_0_norm_1_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303567936)))]; - tensor x_5_cast_fp16 = mul(x = x_normed_3_cast_fp16, y = blocks_0_norm_1_weight_to_fp16)[name = string("x_5_cast_fp16")]; - tensor var_72 = const()[name = string("op_72"), val = tensor([1, 1])]; - tensor var_74 = const()[name = string("op_74"), val = tensor([1, 1])]; - string var_76_pad_type_0 = const()[name = string("op_76_pad_type_0"), val = string("custom")]; - tensor var_76_pad_0 = const()[name = string("op_76_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_76_cast_fp16 = conv(dilations = var_74, groups = var_31, pad = var_76_pad_0, pad_type = var_76_pad_type_0, strides = var_72, weight = blocks_0_attn_q_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_76_cast_fp16")]; + tensor x_5_cast_fp16 = mul(x = x_normed_3_cast_fp16, y = blocks_0_norm_1_weight_to_fp16)[name = string("x_5_cast_fp16")]; + tensor var_73 = const()[name = string("op_73"), val = tensor([1, 1])]; + tensor var_75 = const()[name = string("op_75"), val = tensor([1, 1])]; + string var_77_pad_type_0 = const()[name = string("op_77_pad_type_0"), val = string("custom")]; + tensor var_77_pad_0 = const()[name = string("op_77_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_77_cast_fp16 = conv(dilations = var_75, groups = var_31, pad = var_77_pad_0, pad_type = var_77_pad_type_0, strides = var_73, weight = blocks_0_attn_q_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_77_cast_fp16")]; tensor blocks_0_attn_q_proj_output_scales_to_fp16 = const()[name = string("blocks_0_attn_q_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303576192)))]; - tensor q_1_cast_fp16 = mul(x = var_76_cast_fp16, y = blocks_0_attn_q_proj_output_scales_to_fp16)[name = string("q_1_cast_fp16")]; - tensor var_80 = const()[name = string("op_80"), val = tensor([1, 1])]; - tensor var_82 = const()[name = string("op_82"), val = tensor([1, 1])]; - string var_84_pad_type_0 = const()[name = string("op_84_pad_type_0"), val = string("custom")]; - tensor var_84_pad_0 = const()[name = string("op_84_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_84_cast_fp16 = conv(dilations = var_82, groups = var_31, pad = var_84_pad_0, pad_type = var_84_pad_type_0, strides = var_80, weight = blocks_0_attn_k_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_84_cast_fp16")]; + tensor q_1_cast_fp16 = mul(x = var_77_cast_fp16, y = blocks_0_attn_q_proj_output_scales_to_fp16)[name = string("q_1_cast_fp16")]; + tensor var_81 = const()[name = string("op_81"), val = tensor([1, 1])]; + tensor var_83 = const()[name = string("op_83"), val = tensor([1, 1])]; + string var_85_pad_type_0 = const()[name = string("op_85_pad_type_0"), val = string("custom")]; + tensor var_85_pad_0 = const()[name = string("op_85_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_85_cast_fp16 = conv(dilations = var_83, groups = var_31, pad = var_85_pad_0, pad_type = var_85_pad_type_0, strides = var_81, weight = blocks_0_attn_k_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_85_cast_fp16")]; tensor blocks_0_attn_k_proj_output_scales_to_fp16 = const()[name = string("blocks_0_attn_k_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303584448)))]; - tensor k_1_cast_fp16 = mul(x = var_84_cast_fp16, y = blocks_0_attn_k_proj_output_scales_to_fp16)[name = string("k_1_cast_fp16")]; - tensor var_88 = const()[name = string("op_88"), val = tensor([1, 1])]; - tensor var_90 = const()[name = string("op_90"), val = tensor([1, 1])]; - string var_92_pad_type_0 = const()[name = string("op_92_pad_type_0"), val = string("custom")]; - tensor var_92_pad_0 = const()[name = string("op_92_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_92_cast_fp16 = conv(dilations = var_90, groups = var_31, pad = var_92_pad_0, pad_type = var_92_pad_type_0, strides = var_88, weight = blocks_0_attn_v_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_92_cast_fp16")]; + tensor k_1_cast_fp16 = mul(x = var_85_cast_fp16, y = blocks_0_attn_k_proj_output_scales_to_fp16)[name = string("k_1_cast_fp16")]; + tensor var_89 = const()[name = string("op_89"), val = tensor([1, 1])]; + tensor var_91 = const()[name = string("op_91"), val = tensor([1, 1])]; + string var_93_pad_type_0 = const()[name = string("op_93_pad_type_0"), val = string("custom")]; + tensor var_93_pad_0 = const()[name = string("op_93_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_93_cast_fp16 = conv(dilations = var_91, groups = var_31, pad = var_93_pad_0, pad_type = var_93_pad_type_0, strides = var_89, weight = blocks_0_attn_v_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_93_cast_fp16")]; tensor blocks_0_attn_v_proj_output_scales_to_fp16 = const()[name = string("blocks_0_attn_v_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303592704)))]; - tensor v_1_cast_fp16 = mul(x = var_92_cast_fp16, y = blocks_0_attn_v_proj_output_scales_to_fp16)[name = string("v_1_cast_fp16")]; - tensor var_94 = const()[name = string("op_94"), val = tensor([1, 32, 128, 1])]; - tensor q_3_cast_fp16 = reshape(shape = var_94, x = q_1_cast_fp16)[name = string("q_3_cast_fp16")]; - tensor var_96 = const()[name = string("op_96"), val = tensor([1, 32, 128, 1])]; - tensor k_3_cast_fp16 = reshape(shape = var_96, x = k_1_cast_fp16)[name = string("k_3_cast_fp16")]; - tensor var_98 = const()[name = string("op_98"), val = tensor([1, 32, 128, 1])]; - tensor v_3_cast_fp16 = reshape(shape = var_98, x = v_1_cast_fp16)[name = string("v_3_cast_fp16")]; - tensor var_110_begin_0 = const()[name = string("op_110_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_110_end_0 = const()[name = string("op_110_end_0"), val = tensor([1, 32, 64, 1])]; - tensor var_110_end_mask_0 = const()[name = string("op_110_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_110_cast_fp16 = slice_by_index(begin = var_110_begin_0, end = var_110_end_0, end_mask = var_110_end_mask_0, x = q_3_cast_fp16)[name = string("op_110_cast_fp16")]; - tensor var_116_begin_0 = const()[name = string("op_116_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_116_end_0 = const()[name = string("op_116_end_0"), val = tensor([1, 32, 128, 1])]; - tensor var_116_end_mask_0 = const()[name = string("op_116_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_116_cast_fp16 = slice_by_index(begin = var_116_begin_0, end = var_116_end_0, end_mask = var_116_end_mask_0, x = q_3_cast_fp16)[name = string("op_116_cast_fp16")]; + tensor v_1_cast_fp16 = mul(x = var_93_cast_fp16, y = blocks_0_attn_v_proj_output_scales_to_fp16)[name = string("v_1_cast_fp16")]; + tensor var_95 = const()[name = string("op_95"), val = tensor([1, 32, 128, 4])]; + tensor q_3_cast_fp16 = reshape(shape = var_95, x = q_1_cast_fp16)[name = string("q_3_cast_fp16")]; + tensor var_97 = const()[name = string("op_97"), val = tensor([1, 32, 128, 4])]; + tensor k_3_cast_fp16 = reshape(shape = var_97, x = k_1_cast_fp16)[name = string("k_3_cast_fp16")]; + tensor var_99 = const()[name = string("op_99"), val = tensor([1, 32, 128, 4])]; + tensor v_3_cast_fp16 = reshape(shape = var_99, x = v_1_cast_fp16)[name = string("v_3_cast_fp16")]; + tensor var_111_begin_0 = const()[name = string("op_111_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_111_end_0 = const()[name = string("op_111_end_0"), val = tensor([1, 32, 64, 4])]; + tensor var_111_end_mask_0 = const()[name = string("op_111_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_111_cast_fp16 = slice_by_index(begin = var_111_begin_0, end = var_111_end_0, end_mask = var_111_end_mask_0, x = q_3_cast_fp16)[name = string("op_111_cast_fp16")]; + tensor var_117_begin_0 = const()[name = string("op_117_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_117_end_0 = const()[name = string("op_117_end_0"), val = tensor([1, 32, 128, 4])]; + tensor var_117_end_mask_0 = const()[name = string("op_117_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_117_cast_fp16 = slice_by_index(begin = var_117_begin_0, end = var_117_end_0, end_mask = var_117_end_mask_0, x = q_3_cast_fp16)[name = string("op_117_cast_fp16")]; fp16 const_6_promoted_to_fp16 = const()[name = string("const_6_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_118_cast_fp16 = mul(x = var_116_cast_fp16, y = const_6_promoted_to_fp16)[name = string("op_118_cast_fp16")]; + tensor var_119_cast_fp16 = mul(x = var_117_cast_fp16, y = const_6_promoted_to_fp16)[name = string("op_119_cast_fp16")]; bool rotated_1_interleave_0 = const()[name = string("rotated_1_interleave_0"), val = bool(false)]; - tensor rotated_1_cast_fp16 = concat(axis = var_34, interleave = rotated_1_interleave_0, values = (var_118_cast_fp16, var_110_cast_fp16))[name = string("rotated_1_cast_fp16")]; - tensor var_121_cast_fp16 = mul(x = q_3_cast_fp16, y = cos)[name = string("op_121_cast_fp16")]; - tensor var_122_cast_fp16 = mul(x = rotated_1_cast_fp16, y = sin)[name = string("op_122_cast_fp16")]; - tensor roped_1_cast_fp16 = add(x = var_121_cast_fp16, y = var_122_cast_fp16)[name = string("roped_1_cast_fp16")]; - tensor var_135_begin_0 = const()[name = string("op_135_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_135_end_0 = const()[name = string("op_135_end_0"), val = tensor([1, 32, 64, 1])]; - tensor var_135_end_mask_0 = const()[name = string("op_135_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_135_cast_fp16 = slice_by_index(begin = var_135_begin_0, end = var_135_end_0, end_mask = var_135_end_mask_0, x = k_3_cast_fp16)[name = string("op_135_cast_fp16")]; - tensor var_141_begin_0 = const()[name = string("op_141_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_141_end_0 = const()[name = string("op_141_end_0"), val = tensor([1, 32, 128, 1])]; - tensor var_141_end_mask_0 = const()[name = string("op_141_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_141_cast_fp16 = slice_by_index(begin = var_141_begin_0, end = var_141_end_0, end_mask = var_141_end_mask_0, x = k_3_cast_fp16)[name = string("op_141_cast_fp16")]; + tensor rotated_1_cast_fp16 = concat(axis = var_34, interleave = rotated_1_interleave_0, values = (var_119_cast_fp16, var_111_cast_fp16))[name = string("rotated_1_cast_fp16")]; + tensor var_122_cast_fp16 = mul(x = q_3_cast_fp16, y = cos)[name = string("op_122_cast_fp16")]; + tensor var_123_cast_fp16 = mul(x = rotated_1_cast_fp16, y = sin)[name = string("op_123_cast_fp16")]; + tensor roped_1_cast_fp16 = add(x = var_122_cast_fp16, y = var_123_cast_fp16)[name = string("roped_1_cast_fp16")]; + tensor var_136_begin_0 = const()[name = string("op_136_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_136_end_0 = const()[name = string("op_136_end_0"), val = tensor([1, 32, 64, 4])]; + tensor var_136_end_mask_0 = const()[name = string("op_136_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_136_cast_fp16 = slice_by_index(begin = var_136_begin_0, end = var_136_end_0, end_mask = var_136_end_mask_0, x = k_3_cast_fp16)[name = string("op_136_cast_fp16")]; + tensor var_142_begin_0 = const()[name = string("op_142_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_142_end_0 = const()[name = string("op_142_end_0"), val = tensor([1, 32, 128, 4])]; + tensor var_142_end_mask_0 = const()[name = string("op_142_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_142_cast_fp16 = slice_by_index(begin = var_142_begin_0, end = var_142_end_0, end_mask = var_142_end_mask_0, x = k_3_cast_fp16)[name = string("op_142_cast_fp16")]; fp16 const_8_promoted_to_fp16 = const()[name = string("const_8_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_143_cast_fp16 = mul(x = var_141_cast_fp16, y = const_8_promoted_to_fp16)[name = string("op_143_cast_fp16")]; + tensor var_144_cast_fp16 = mul(x = var_142_cast_fp16, y = const_8_promoted_to_fp16)[name = string("op_144_cast_fp16")]; bool rotated_3_interleave_0 = const()[name = string("rotated_3_interleave_0"), val = bool(false)]; - tensor rotated_3_cast_fp16 = concat(axis = var_34, interleave = rotated_3_interleave_0, values = (var_143_cast_fp16, var_135_cast_fp16))[name = string("rotated_3_cast_fp16")]; - tensor var_146_cast_fp16 = mul(x = k_3_cast_fp16, y = cos)[name = string("op_146_cast_fp16")]; - tensor var_147_cast_fp16 = mul(x = rotated_3_cast_fp16, y = sin)[name = string("op_147_cast_fp16")]; - tensor roped_3_cast_fp16 = add(x = var_146_cast_fp16, y = var_147_cast_fp16)[name = string("roped_3_cast_fp16")]; + tensor rotated_3_cast_fp16 = concat(axis = var_34, interleave = rotated_3_interleave_0, values = (var_144_cast_fp16, var_136_cast_fp16))[name = string("rotated_3_cast_fp16")]; + tensor var_147_cast_fp16 = mul(x = k_3_cast_fp16, y = cos)[name = string("op_147_cast_fp16")]; + tensor var_148_cast_fp16 = mul(x = rotated_3_cast_fp16, y = sin)[name = string("op_148_cast_fp16")]; + tensor roped_3_cast_fp16 = add(x = var_147_cast_fp16, y = var_148_cast_fp16)[name = string("roped_3_cast_fp16")]; + tensor v_5_perm_0 = const()[name = string("v_5_perm_0"), val = tensor([0, 1, -1, -2])]; bool k_7_interleave_0 = const()[name = string("k_7_interleave_0"), val = bool(false)]; tensor k_7_cast_fp16 = concat(axis = var_22, interleave = k_7_interleave_0, values = (k_cache_0, roped_3_cast_fp16))[name = string("k_7_cast_fp16")]; - bool v_5_interleave_0 = const()[name = string("v_5_interleave_0"), val = bool(false)]; - tensor v_5_cast_fp16 = concat(axis = var_22, interleave = v_5_interleave_0, values = (v_cache_0, v_3_cast_fp16))[name = string("v_5_cast_fp16")]; - tensor var_154_begin_0 = const()[name = string("op_154_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_154_end_0 = const()[name = string("op_154_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_154_end_mask_0 = const()[name = string("op_154_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_k_cache_0 = slice_by_index(begin = var_154_begin_0, end = var_154_end_0, end_mask = var_154_end_mask_0, x = k_7_cast_fp16)[name = string("op_154_cast_fp16")]; - tensor var_155_begin_0 = const()[name = string("op_155_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_155_end_0 = const()[name = string("op_155_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_155_end_mask_0 = const()[name = string("op_155_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_v_cache_0 = slice_by_index(begin = var_155_begin_0, end = var_155_end_0, end_mask = var_155_end_mask_0, x = v_5_cast_fp16)[name = string("op_155_cast_fp16")]; - fp16 var_159_to_fp16 = const()[name = string("op_159_to_fp16"), val = fp16(0x1.6ap-4)]; - tensor var_160_cast_fp16 = mul(x = roped_1_cast_fp16, y = var_159_to_fp16)[name = string("op_160_cast_fp16")]; + bool v_7_interleave_0 = const()[name = string("v_7_interleave_0"), val = bool(false)]; + tensor v_5_cast_fp16 = transpose(perm = v_5_perm_0, x = v_3_cast_fp16)[name = string("transpose_5")]; + tensor v_7_cast_fp16 = concat(axis = var_34, interleave = v_7_interleave_0, values = (v_cache_0, v_5_cast_fp16))[name = string("v_7_cast_fp16")]; + tensor var_159_begin_0 = const()[name = string("op_159_begin_0"), val = tensor([0, 0, 0, 1])]; + tensor var_159_end_0 = const()[name = string("op_159_end_0"), val = tensor([1, 32, 128, 509])]; + tensor var_159_end_mask_0 = const()[name = string("op_159_end_mask_0"), val = tensor([true, true, true, false])]; + tensor new_k_cache_0 = slice_by_index(begin = var_159_begin_0, end = var_159_end_0, end_mask = var_159_end_mask_0, x = k_7_cast_fp16)[name = string("op_159_cast_fp16")]; + tensor var_160_begin_0 = const()[name = string("op_160_begin_0"), val = tensor([0, 0, 1, 0])]; + tensor var_160_end_0 = const()[name = string("op_160_end_0"), val = tensor([1, 32, 509, 128])]; + tensor var_160_end_mask_0 = const()[name = string("op_160_end_mask_0"), val = tensor([true, true, false, true])]; + tensor new_v_cache_0 = slice_by_index(begin = var_160_begin_0, end = var_160_end_0, end_mask = var_160_end_mask_0, x = v_7_cast_fp16)[name = string("op_160_cast_fp16")]; + fp16 var_165_to_fp16 = const()[name = string("op_165_to_fp16"), val = fp16(0x1.6ap-4)]; + tensor var_166_cast_fp16 = mul(x = roped_1_cast_fp16, y = var_165_to_fp16)[name = string("op_166_cast_fp16")]; bool attn_weights_1_transpose_x_0 = const()[name = string("attn_weights_1_transpose_x_0"), val = bool(true)]; bool attn_weights_1_transpose_y_0 = const()[name = string("attn_weights_1_transpose_y_0"), val = bool(false)]; - tensor attn_weights_1_cast_fp16 = matmul(transpose_x = attn_weights_1_transpose_x_0, transpose_y = attn_weights_1_transpose_y_0, x = var_160_cast_fp16, y = k_7_cast_fp16)[name = string("attn_weights_1_cast_fp16")]; - tensor attn_weights_3_cast_fp16 = add(x = attn_weights_1_cast_fp16, y = mask)[name = string("attn_weights_3_cast_fp16")]; - tensor var_168_cast_fp16 = softmax(axis = var_30, x = attn_weights_3_cast_fp16)[name = string("op_168_cast_fp16")]; - bool attn_1_transpose_x_0 = const()[name = string("attn_1_transpose_x_0"), val = bool(false)]; - bool attn_1_transpose_y_0 = const()[name = string("attn_1_transpose_y_0"), val = bool(true)]; - tensor attn_1_cast_fp16 = matmul(transpose_x = attn_1_transpose_x_0, transpose_y = attn_1_transpose_y_0, x = v_5_cast_fp16, y = var_168_cast_fp16)[name = string("attn_1_cast_fp16")]; - tensor var_172 = const()[name = string("op_172"), val = tensor([1, 4096, 1, -1])]; - tensor input_1_cast_fp16 = reshape(shape = var_172, x = attn_1_cast_fp16)[name = string("input_1_cast_fp16")]; - tensor var_176 = const()[name = string("op_176"), val = tensor([1, 1])]; - tensor var_178 = const()[name = string("op_178"), val = tensor([1, 1])]; - string var_180_pad_type_0 = const()[name = string("op_180_pad_type_0"), val = string("custom")]; - tensor var_180_pad_0 = const()[name = string("op_180_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_180_cast_fp16 = conv(dilations = var_178, groups = var_31, pad = var_180_pad_0, pad_type = var_180_pad_type_0, strides = var_176, weight = blocks_0_attn_proj_weight_palettized_cast_fp16, x = input_1_cast_fp16)[name = string("op_180_cast_fp16")]; + tensor attn_weights_1_cast_fp16 = matmul(transpose_x = attn_weights_1_transpose_x_0, transpose_y = attn_weights_1_transpose_y_0, x = var_166_cast_fp16, y = k_7_cast_fp16)[name = string("attn_weights_1_cast_fp16")]; + tensor attn_weights_3_cast_fp16 = add(x = attn_weights_1_cast_fp16, y = mask)[name = string("attn_weights_3_cast_fp16")]; + tensor attn_weights_5_cast_fp16 = softmax(axis = var_30, x = attn_weights_3_cast_fp16)[name = string("attn_weights_5_cast_fp16")]; + bool var_175_transpose_x_0 = const()[name = string("op_175_transpose_x_0"), val = bool(false)]; + bool var_175_transpose_y_0 = const()[name = string("op_175_transpose_y_0"), val = bool(false)]; + tensor var_175_cast_fp16 = matmul(transpose_x = var_175_transpose_x_0, transpose_y = var_175_transpose_y_0, x = attn_weights_5_cast_fp16, y = v_7_cast_fp16)[name = string("op_175_cast_fp16")]; + tensor attn_1_perm_0 = const()[name = string("attn_1_perm_0"), val = tensor([0, 1, -1, -2])]; + tensor var_178 = const()[name = string("op_178"), val = tensor([1, 4096, 1, -1])]; + tensor attn_1_cast_fp16 = transpose(perm = attn_1_perm_0, x = var_175_cast_fp16)[name = string("transpose_4")]; + tensor input_1_cast_fp16 = reshape(shape = var_178, x = attn_1_cast_fp16)[name = string("input_1_cast_fp16")]; + tensor var_182 = const()[name = string("op_182"), val = tensor([1, 1])]; + tensor var_184 = const()[name = string("op_184"), val = tensor([1, 1])]; + string var_186_pad_type_0 = const()[name = string("op_186_pad_type_0"), val = string("custom")]; + tensor var_186_pad_0 = const()[name = string("op_186_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_186_cast_fp16 = conv(dilations = var_184, groups = var_31, pad = var_186_pad_0, pad_type = var_186_pad_type_0, strides = var_182, weight = blocks_0_attn_proj_weight_palettized_cast_fp16, x = input_1_cast_fp16)[name = string("op_186_cast_fp16")]; tensor blocks_0_attn_proj_output_scales_to_fp16 = const()[name = string("blocks_0_attn_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303600960)))]; - tensor attention_output_1_cast_fp16 = mul(x = var_180_cast_fp16, y = blocks_0_attn_proj_output_scales_to_fp16)[name = string("attention_output_1_cast_fp16")]; - tensor x_11_cast_fp16 = add(x = attention_output_1_cast_fp16, y = x)[name = string("x_11_cast_fp16")]; - tensor var_199_axes_0 = const()[name = string("op_199_axes_0"), val = tensor([-2])]; - tensor var_199_cast_fp16 = squeeze(axes = var_199_axes_0, x = x_11_cast_fp16)[name = string("op_199_cast_fp16")]; - bool var_201_interleave_0 = const()[name = string("op_201_interleave_0"), val = bool(false)]; - tensor eps_chan_3_to_fp16 = const()[name = string("eps_chan_3_to_fp16"), val = tensor([[[0x1.9e8p-3]]])]; - tensor var_201_cast_fp16 = concat(axis = var_31, interleave = var_201_interleave_0, values = (var_199_cast_fp16, eps_chan_3_to_fp16))[name = string("op_201_cast_fp16")]; + tensor attention_output_1_cast_fp16 = mul(x = var_186_cast_fp16, y = blocks_0_attn_proj_output_scales_to_fp16)[name = string("attention_output_1_cast_fp16")]; + tensor x_11_cast_fp16 = add(x = attention_output_1_cast_fp16, y = x)[name = string("x_11_cast_fp16")]; + tensor var_205_axes_0 = const()[name = string("op_205_axes_0"), val = tensor([-2])]; + tensor var_205_cast_fp16 = squeeze(axes = var_205_axes_0, x = x_11_cast_fp16)[name = string("op_205_cast_fp16")]; + bool var_207_interleave_0 = const()[name = string("op_207_interleave_0"), val = bool(false)]; + tensor eps_chan_3_to_fp16 = const()[name = string("eps_chan_3_to_fp16"), val = tensor([[[0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3]]])]; + tensor var_207_cast_fp16 = concat(axis = var_31, interleave = var_207_interleave_0, values = (var_205_cast_fp16, eps_chan_3_to_fp16))[name = string("op_207_cast_fp16")]; tensor x_eps_3_axes_0 = const()[name = string("x_eps_3_axes_0"), val = tensor([-2])]; - tensor x_eps_3_cast_fp16 = expand_dims(axes = x_eps_3_axes_0, x = var_201_cast_fp16)[name = string("x_eps_3_cast_fp16")]; + tensor x_eps_3_cast_fp16 = expand_dims(axes = x_eps_3_axes_0, x = var_207_cast_fp16)[name = string("x_eps_3_cast_fp16")]; tensor norm_x_3_axes_0 = const()[name = string("norm_x_3_axes_0"), val = tensor([1])]; - tensor norm_x_3_cast_fp16 = reduce_l2_norm(axes = norm_x_3_axes_0, keep_dims = var_35, x = x_eps_3_cast_fp16)[name = string("norm_x_3_cast_fp16")]; - tensor x_normed_7_cast_fp16 = real_div(x = x_11_cast_fp16, y = norm_x_3_cast_fp16)[name = string("x_normed_7_cast_fp16")]; - fp16 var_206_to_fp16 = const()[name = string("op_206_to_fp16"), val = fp16(0x1p+6)]; - tensor x_normed_9_cast_fp16 = mul(x = x_normed_7_cast_fp16, y = var_206_to_fp16)[name = string("x_normed_9_cast_fp16")]; + tensor norm_x_3_cast_fp16 = reduce_l2_norm(axes = norm_x_3_axes_0, keep_dims = var_35, x = x_eps_3_cast_fp16)[name = string("norm_x_3_cast_fp16")]; + tensor x_normed_7_cast_fp16 = real_div(x = x_11_cast_fp16, y = norm_x_3_cast_fp16)[name = string("x_normed_7_cast_fp16")]; + fp16 var_212_to_fp16 = const()[name = string("op_212_to_fp16"), val = fp16(0x1p+6)]; + tensor x_normed_9_cast_fp16 = mul(x = x_normed_7_cast_fp16, y = var_212_to_fp16)[name = string("x_normed_9_cast_fp16")]; tensor blocks_0_norm_2_weight_to_fp16 = const()[name = string("blocks_0_norm_2_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303609216)))]; - tensor input_3_cast_fp16 = mul(x = x_normed_9_cast_fp16, y = blocks_0_norm_2_weight_to_fp16)[name = string("input_3_cast_fp16")]; - tensor var_218 = const()[name = string("op_218"), val = tensor([1, 1])]; - tensor var_220 = const()[name = string("op_220"), val = tensor([1, 1])]; - string var_222_pad_type_0 = const()[name = string("op_222_pad_type_0"), val = string("custom")]; - tensor var_222_pad_0 = const()[name = string("op_222_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_222_cast_fp16 = conv(dilations = var_220, groups = var_31, pad = var_222_pad_0, pad_type = var_222_pad_type_0, strides = var_218, weight = blocks_0_mlp_fc_1_weight_palettized_cast_fp16, x = input_3_cast_fp16)[name = string("op_222_cast_fp16")]; - tensor blocks_0_mlp_fc_1_output_scales_to_fp16 = const()[name = string("blocks_0_mlp_fc_1_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303617472)))]; - tensor input_5_cast_fp16 = mul(x = var_222_cast_fp16, y = blocks_0_mlp_fc_1_output_scales_to_fp16)[name = string("input_5_cast_fp16")]; + tensor input_3_cast_fp16 = mul(x = x_normed_9_cast_fp16, y = blocks_0_norm_2_weight_to_fp16)[name = string("input_3_cast_fp16")]; + tensor var_224 = const()[name = string("op_224"), val = tensor([1, 1])]; tensor var_226 = const()[name = string("op_226"), val = tensor([1, 1])]; - tensor var_228 = const()[name = string("op_228"), val = tensor([1, 1])]; - string var_230_pad_type_0 = const()[name = string("op_230_pad_type_0"), val = string("custom")]; - tensor var_230_pad_0 = const()[name = string("op_230_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_230_cast_fp16 = conv(dilations = var_228, groups = var_31, pad = var_230_pad_0, pad_type = var_230_pad_type_0, strides = var_226, weight = blocks_0_mlp_fc_2_weight_palettized_cast_fp16, x = input_3_cast_fp16)[name = string("op_230_cast_fp16")]; + string var_228_pad_type_0 = const()[name = string("op_228_pad_type_0"), val = string("custom")]; + tensor var_228_pad_0 = const()[name = string("op_228_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_228_cast_fp16 = conv(dilations = var_226, groups = var_31, pad = var_228_pad_0, pad_type = var_228_pad_type_0, strides = var_224, weight = blocks_0_mlp_fc_1_weight_palettized_cast_fp16, x = input_3_cast_fp16)[name = string("op_228_cast_fp16")]; + tensor blocks_0_mlp_fc_1_output_scales_to_fp16 = const()[name = string("blocks_0_mlp_fc_1_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303617472)))]; + tensor input_5_cast_fp16 = mul(x = var_228_cast_fp16, y = blocks_0_mlp_fc_1_output_scales_to_fp16)[name = string("input_5_cast_fp16")]; + tensor var_232 = const()[name = string("op_232"), val = tensor([1, 1])]; + tensor var_234 = const()[name = string("op_234"), val = tensor([1, 1])]; + string var_236_pad_type_0 = const()[name = string("op_236_pad_type_0"), val = string("custom")]; + tensor var_236_pad_0 = const()[name = string("op_236_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_236_cast_fp16 = conv(dilations = var_234, groups = var_31, pad = var_236_pad_0, pad_type = var_236_pad_type_0, strides = var_232, weight = blocks_0_mlp_fc_2_weight_palettized_cast_fp16, x = input_3_cast_fp16)[name = string("op_236_cast_fp16")]; tensor blocks_0_mlp_fc_2_output_scales_to_fp16 = const()[name = string("blocks_0_mlp_fc_2_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303639552)))]; - tensor x_fc_2_1_cast_fp16 = mul(x = var_230_cast_fp16, y = blocks_0_mlp_fc_2_output_scales_to_fp16)[name = string("x_fc_2_1_cast_fp16")]; - tensor var_232_cast_fp16 = silu(x = input_5_cast_fp16)[name = string("op_232_cast_fp16")]; - tensor input_7_cast_fp16 = mul(x = var_232_cast_fp16, y = x_fc_2_1_cast_fp16)[name = string("input_7_cast_fp16")]; - tensor var_236 = const()[name = string("op_236"), val = tensor([1, 1])]; - tensor var_238 = const()[name = string("op_238"), val = tensor([1, 1])]; - string var_240_pad_type_0 = const()[name = string("op_240_pad_type_0"), val = string("custom")]; - tensor var_240_pad_0 = const()[name = string("op_240_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_240_cast_fp16 = conv(dilations = var_238, groups = var_31, pad = var_240_pad_0, pad_type = var_240_pad_type_0, strides = var_236, weight = blocks_0_mlp_proj_weight_palettized_cast_fp16, x = input_7_cast_fp16)[name = string("op_240_cast_fp16")]; + tensor x_fc_2_1_cast_fp16 = mul(x = var_236_cast_fp16, y = blocks_0_mlp_fc_2_output_scales_to_fp16)[name = string("x_fc_2_1_cast_fp16")]; + tensor var_238_cast_fp16 = silu(x = input_5_cast_fp16)[name = string("op_238_cast_fp16")]; + tensor input_7_cast_fp16 = mul(x = var_238_cast_fp16, y = x_fc_2_1_cast_fp16)[name = string("input_7_cast_fp16")]; + tensor var_242 = const()[name = string("op_242"), val = tensor([1, 1])]; + tensor var_244 = const()[name = string("op_244"), val = tensor([1, 1])]; + string var_246_pad_type_0 = const()[name = string("op_246_pad_type_0"), val = string("custom")]; + tensor var_246_pad_0 = const()[name = string("op_246_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_246_cast_fp16 = conv(dilations = var_244, groups = var_31, pad = var_246_pad_0, pad_type = var_246_pad_type_0, strides = var_242, weight = blocks_0_mlp_proj_weight_palettized_cast_fp16, x = input_7_cast_fp16)[name = string("op_246_cast_fp16")]; tensor blocks_0_mlp_proj_output_scales_to_fp16 = const()[name = string("blocks_0_mlp_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303661632)))]; - tensor var_241_cast_fp16 = mul(x = var_240_cast_fp16, y = blocks_0_mlp_proj_output_scales_to_fp16)[name = string("op_241_cast_fp16")]; - tensor x_15_cast_fp16 = add(x = var_241_cast_fp16, y = x_11_cast_fp16)[name = string("x_15_cast_fp16")]; - int32 var_252 = const()[name = string("op_252"), val = int32(-1)]; - int32 var_260 = const()[name = string("op_260"), val = int32(3)]; - int32 var_261 = const()[name = string("op_261"), val = int32(1)]; - int32 var_264 = const()[name = string("op_264"), val = int32(-2)]; - bool var_265 = const()[name = string("op_265"), val = bool(true)]; - tensor var_282_axes_0 = const()[name = string("op_282_axes_0"), val = tensor([-2])]; - tensor var_282_cast_fp16 = squeeze(axes = var_282_axes_0, x = x_15_cast_fp16)[name = string("op_282_cast_fp16")]; - bool var_284_interleave_0 = const()[name = string("op_284_interleave_0"), val = bool(false)]; - tensor eps_chan_5_to_fp16 = const()[name = string("eps_chan_5_to_fp16"), val = tensor([[[0x1.9e8p-3]]])]; - tensor var_284_cast_fp16 = concat(axis = var_261, interleave = var_284_interleave_0, values = (var_282_cast_fp16, eps_chan_5_to_fp16))[name = string("op_284_cast_fp16")]; + tensor var_247_cast_fp16 = mul(x = var_246_cast_fp16, y = blocks_0_mlp_proj_output_scales_to_fp16)[name = string("op_247_cast_fp16")]; + tensor x_15_cast_fp16 = add(x = var_247_cast_fp16, y = x_11_cast_fp16)[name = string("x_15_cast_fp16")]; + int32 var_258 = const()[name = string("op_258"), val = int32(-1)]; + int32 var_266 = const()[name = string("op_266"), val = int32(3)]; + int32 var_267 = const()[name = string("op_267"), val = int32(1)]; + int32 var_270 = const()[name = string("op_270"), val = int32(-2)]; + bool var_271 = const()[name = string("op_271"), val = bool(true)]; + tensor var_288_axes_0 = const()[name = string("op_288_axes_0"), val = tensor([-2])]; + tensor var_288_cast_fp16 = squeeze(axes = var_288_axes_0, x = x_15_cast_fp16)[name = string("op_288_cast_fp16")]; + bool var_290_interleave_0 = const()[name = string("op_290_interleave_0"), val = bool(false)]; + tensor eps_chan_5_to_fp16 = const()[name = string("eps_chan_5_to_fp16"), val = tensor([[[0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3]]])]; + tensor var_290_cast_fp16 = concat(axis = var_267, interleave = var_290_interleave_0, values = (var_288_cast_fp16, eps_chan_5_to_fp16))[name = string("op_290_cast_fp16")]; tensor x_eps_5_axes_0 = const()[name = string("x_eps_5_axes_0"), val = tensor([-2])]; - tensor x_eps_5_cast_fp16 = expand_dims(axes = x_eps_5_axes_0, x = var_284_cast_fp16)[name = string("x_eps_5_cast_fp16")]; + tensor x_eps_5_cast_fp16 = expand_dims(axes = x_eps_5_axes_0, x = var_290_cast_fp16)[name = string("x_eps_5_cast_fp16")]; tensor norm_x_5_axes_0 = const()[name = string("norm_x_5_axes_0"), val = tensor([1])]; - tensor norm_x_5_cast_fp16 = reduce_l2_norm(axes = norm_x_5_axes_0, keep_dims = var_265, x = x_eps_5_cast_fp16)[name = string("norm_x_5_cast_fp16")]; - tensor x_normed_13_cast_fp16 = real_div(x = x_15_cast_fp16, y = norm_x_5_cast_fp16)[name = string("x_normed_13_cast_fp16")]; - fp16 var_289_to_fp16 = const()[name = string("op_289_to_fp16"), val = fp16(0x1p+6)]; - tensor x_normed_15_cast_fp16 = mul(x = x_normed_13_cast_fp16, y = var_289_to_fp16)[name = string("x_normed_15_cast_fp16")]; + tensor norm_x_5_cast_fp16 = reduce_l2_norm(axes = norm_x_5_axes_0, keep_dims = var_271, x = x_eps_5_cast_fp16)[name = string("norm_x_5_cast_fp16")]; + tensor x_normed_13_cast_fp16 = real_div(x = x_15_cast_fp16, y = norm_x_5_cast_fp16)[name = string("x_normed_13_cast_fp16")]; + fp16 var_295_to_fp16 = const()[name = string("op_295_to_fp16"), val = fp16(0x1p+6)]; + tensor x_normed_15_cast_fp16 = mul(x = x_normed_13_cast_fp16, y = var_295_to_fp16)[name = string("x_normed_15_cast_fp16")]; tensor blocks_1_norm_1_weight_to_fp16 = const()[name = string("blocks_1_norm_1_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303669888)))]; - tensor x_19_cast_fp16 = mul(x = x_normed_15_cast_fp16, y = blocks_1_norm_1_weight_to_fp16)[name = string("x_19_cast_fp16")]; - tensor var_304 = const()[name = string("op_304"), val = tensor([1, 1])]; - tensor var_306 = const()[name = string("op_306"), val = tensor([1, 1])]; - string var_308_pad_type_0 = const()[name = string("op_308_pad_type_0"), val = string("custom")]; - tensor var_308_pad_0 = const()[name = string("op_308_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_308_cast_fp16 = conv(dilations = var_306, groups = var_261, pad = var_308_pad_0, pad_type = var_308_pad_type_0, strides = var_304, weight = blocks_1_attn_q_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_308_cast_fp16")]; + tensor x_19_cast_fp16 = mul(x = x_normed_15_cast_fp16, y = blocks_1_norm_1_weight_to_fp16)[name = string("x_19_cast_fp16")]; + tensor var_311 = const()[name = string("op_311"), val = tensor([1, 1])]; + tensor var_313 = const()[name = string("op_313"), val = tensor([1, 1])]; + string var_315_pad_type_0 = const()[name = string("op_315_pad_type_0"), val = string("custom")]; + tensor var_315_pad_0 = const()[name = string("op_315_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_315_cast_fp16 = conv(dilations = var_313, groups = var_267, pad = var_315_pad_0, pad_type = var_315_pad_type_0, strides = var_311, weight = blocks_1_attn_q_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_315_cast_fp16")]; tensor blocks_1_attn_q_proj_output_scales_to_fp16 = const()[name = string("blocks_1_attn_q_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303678144)))]; - tensor q_7_cast_fp16 = mul(x = var_308_cast_fp16, y = blocks_1_attn_q_proj_output_scales_to_fp16)[name = string("q_7_cast_fp16")]; - tensor var_312 = const()[name = string("op_312"), val = tensor([1, 1])]; - tensor var_314 = const()[name = string("op_314"), val = tensor([1, 1])]; - string var_316_pad_type_0 = const()[name = string("op_316_pad_type_0"), val = string("custom")]; - tensor var_316_pad_0 = const()[name = string("op_316_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_316_cast_fp16 = conv(dilations = var_314, groups = var_261, pad = var_316_pad_0, pad_type = var_316_pad_type_0, strides = var_312, weight = blocks_1_attn_k_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_316_cast_fp16")]; + tensor q_7_cast_fp16 = mul(x = var_315_cast_fp16, y = blocks_1_attn_q_proj_output_scales_to_fp16)[name = string("q_7_cast_fp16")]; + tensor var_319 = const()[name = string("op_319"), val = tensor([1, 1])]; + tensor var_321 = const()[name = string("op_321"), val = tensor([1, 1])]; + string var_323_pad_type_0 = const()[name = string("op_323_pad_type_0"), val = string("custom")]; + tensor var_323_pad_0 = const()[name = string("op_323_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_323_cast_fp16 = conv(dilations = var_321, groups = var_267, pad = var_323_pad_0, pad_type = var_323_pad_type_0, strides = var_319, weight = blocks_1_attn_k_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_323_cast_fp16")]; tensor blocks_1_attn_k_proj_output_scales_to_fp16 = const()[name = string("blocks_1_attn_k_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303686400)))]; - tensor k_9_cast_fp16 = mul(x = var_316_cast_fp16, y = blocks_1_attn_k_proj_output_scales_to_fp16)[name = string("k_9_cast_fp16")]; - tensor var_320 = const()[name = string("op_320"), val = tensor([1, 1])]; - tensor var_322 = const()[name = string("op_322"), val = tensor([1, 1])]; - string var_324_pad_type_0 = const()[name = string("op_324_pad_type_0"), val = string("custom")]; - tensor var_324_pad_0 = const()[name = string("op_324_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_324_cast_fp16 = conv(dilations = var_322, groups = var_261, pad = var_324_pad_0, pad_type = var_324_pad_type_0, strides = var_320, weight = blocks_1_attn_v_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_324_cast_fp16")]; + tensor k_9_cast_fp16 = mul(x = var_323_cast_fp16, y = blocks_1_attn_k_proj_output_scales_to_fp16)[name = string("k_9_cast_fp16")]; + tensor var_327 = const()[name = string("op_327"), val = tensor([1, 1])]; + tensor var_329 = const()[name = string("op_329"), val = tensor([1, 1])]; + string var_331_pad_type_0 = const()[name = string("op_331_pad_type_0"), val = string("custom")]; + tensor var_331_pad_0 = const()[name = string("op_331_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_331_cast_fp16 = conv(dilations = var_329, groups = var_267, pad = var_331_pad_0, pad_type = var_331_pad_type_0, strides = var_327, weight = blocks_1_attn_v_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_331_cast_fp16")]; tensor blocks_1_attn_v_proj_output_scales_to_fp16 = const()[name = string("blocks_1_attn_v_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303694656)))]; - tensor v_7_cast_fp16 = mul(x = var_324_cast_fp16, y = blocks_1_attn_v_proj_output_scales_to_fp16)[name = string("v_7_cast_fp16")]; - tensor var_326 = const()[name = string("op_326"), val = tensor([1, 32, 128, 1])]; - tensor q_9_cast_fp16 = reshape(shape = var_326, x = q_7_cast_fp16)[name = string("q_9_cast_fp16")]; - tensor var_328 = const()[name = string("op_328"), val = tensor([1, 32, 128, 1])]; - tensor k_11_cast_fp16 = reshape(shape = var_328, x = k_9_cast_fp16)[name = string("k_11_cast_fp16")]; - tensor var_330 = const()[name = string("op_330"), val = tensor([1, 32, 128, 1])]; - tensor v_9_cast_fp16 = reshape(shape = var_330, x = v_7_cast_fp16)[name = string("v_9_cast_fp16")]; - tensor var_342_begin_0 = const()[name = string("op_342_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_342_end_0 = const()[name = string("op_342_end_0"), val = tensor([1, 32, 64, 1])]; - tensor var_342_end_mask_0 = const()[name = string("op_342_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_342_cast_fp16 = slice_by_index(begin = var_342_begin_0, end = var_342_end_0, end_mask = var_342_end_mask_0, x = q_9_cast_fp16)[name = string("op_342_cast_fp16")]; - tensor var_348_begin_0 = const()[name = string("op_348_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_348_end_0 = const()[name = string("op_348_end_0"), val = tensor([1, 32, 128, 1])]; - tensor var_348_end_mask_0 = const()[name = string("op_348_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_348_cast_fp16 = slice_by_index(begin = var_348_begin_0, end = var_348_end_0, end_mask = var_348_end_mask_0, x = q_9_cast_fp16)[name = string("op_348_cast_fp16")]; + tensor v_9_cast_fp16 = mul(x = var_331_cast_fp16, y = blocks_1_attn_v_proj_output_scales_to_fp16)[name = string("v_9_cast_fp16")]; + tensor var_333 = const()[name = string("op_333"), val = tensor([1, 32, 128, 4])]; + tensor q_9_cast_fp16 = reshape(shape = var_333, x = q_7_cast_fp16)[name = string("q_9_cast_fp16")]; + tensor var_335 = const()[name = string("op_335"), val = tensor([1, 32, 128, 4])]; + tensor k_11_cast_fp16 = reshape(shape = var_335, x = k_9_cast_fp16)[name = string("k_11_cast_fp16")]; + tensor var_337 = const()[name = string("op_337"), val = tensor([1, 32, 128, 4])]; + tensor v_11_cast_fp16 = reshape(shape = var_337, x = v_9_cast_fp16)[name = string("v_11_cast_fp16")]; + tensor var_349_begin_0 = const()[name = string("op_349_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_349_end_0 = const()[name = string("op_349_end_0"), val = tensor([1, 32, 64, 4])]; + tensor var_349_end_mask_0 = const()[name = string("op_349_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_349_cast_fp16 = slice_by_index(begin = var_349_begin_0, end = var_349_end_0, end_mask = var_349_end_mask_0, x = q_9_cast_fp16)[name = string("op_349_cast_fp16")]; + tensor var_355_begin_0 = const()[name = string("op_355_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_355_end_0 = const()[name = string("op_355_end_0"), val = tensor([1, 32, 128, 4])]; + tensor var_355_end_mask_0 = const()[name = string("op_355_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_355_cast_fp16 = slice_by_index(begin = var_355_begin_0, end = var_355_end_0, end_mask = var_355_end_mask_0, x = q_9_cast_fp16)[name = string("op_355_cast_fp16")]; fp16 const_19_promoted_to_fp16 = const()[name = string("const_19_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_350_cast_fp16 = mul(x = var_348_cast_fp16, y = const_19_promoted_to_fp16)[name = string("op_350_cast_fp16")]; + tensor var_357_cast_fp16 = mul(x = var_355_cast_fp16, y = const_19_promoted_to_fp16)[name = string("op_357_cast_fp16")]; bool rotated_5_interleave_0 = const()[name = string("rotated_5_interleave_0"), val = bool(false)]; - tensor rotated_5_cast_fp16 = concat(axis = var_264, interleave = rotated_5_interleave_0, values = (var_350_cast_fp16, var_342_cast_fp16))[name = string("rotated_5_cast_fp16")]; - tensor var_353_cast_fp16 = mul(x = q_9_cast_fp16, y = cos)[name = string("op_353_cast_fp16")]; - tensor var_354_cast_fp16 = mul(x = rotated_5_cast_fp16, y = sin)[name = string("op_354_cast_fp16")]; - tensor roped_5_cast_fp16 = add(x = var_353_cast_fp16, y = var_354_cast_fp16)[name = string("roped_5_cast_fp16")]; - tensor var_367_begin_0 = const()[name = string("op_367_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_367_end_0 = const()[name = string("op_367_end_0"), val = tensor([1, 32, 64, 1])]; - tensor var_367_end_mask_0 = const()[name = string("op_367_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_367_cast_fp16 = slice_by_index(begin = var_367_begin_0, end = var_367_end_0, end_mask = var_367_end_mask_0, x = k_11_cast_fp16)[name = string("op_367_cast_fp16")]; - tensor var_373_begin_0 = const()[name = string("op_373_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_373_end_0 = const()[name = string("op_373_end_0"), val = tensor([1, 32, 128, 1])]; - tensor var_373_end_mask_0 = const()[name = string("op_373_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_373_cast_fp16 = slice_by_index(begin = var_373_begin_0, end = var_373_end_0, end_mask = var_373_end_mask_0, x = k_11_cast_fp16)[name = string("op_373_cast_fp16")]; + tensor rotated_5_cast_fp16 = concat(axis = var_270, interleave = rotated_5_interleave_0, values = (var_357_cast_fp16, var_349_cast_fp16))[name = string("rotated_5_cast_fp16")]; + tensor var_360_cast_fp16 = mul(x = q_9_cast_fp16, y = cos)[name = string("op_360_cast_fp16")]; + tensor var_361_cast_fp16 = mul(x = rotated_5_cast_fp16, y = sin)[name = string("op_361_cast_fp16")]; + tensor roped_5_cast_fp16 = add(x = var_360_cast_fp16, y = var_361_cast_fp16)[name = string("roped_5_cast_fp16")]; + tensor var_374_begin_0 = const()[name = string("op_374_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_374_end_0 = const()[name = string("op_374_end_0"), val = tensor([1, 32, 64, 4])]; + tensor var_374_end_mask_0 = const()[name = string("op_374_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_374_cast_fp16 = slice_by_index(begin = var_374_begin_0, end = var_374_end_0, end_mask = var_374_end_mask_0, x = k_11_cast_fp16)[name = string("op_374_cast_fp16")]; + tensor var_380_begin_0 = const()[name = string("op_380_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_380_end_0 = const()[name = string("op_380_end_0"), val = tensor([1, 32, 128, 4])]; + tensor var_380_end_mask_0 = const()[name = string("op_380_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_380_cast_fp16 = slice_by_index(begin = var_380_begin_0, end = var_380_end_0, end_mask = var_380_end_mask_0, x = k_11_cast_fp16)[name = string("op_380_cast_fp16")]; fp16 const_21_promoted_to_fp16 = const()[name = string("const_21_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_375_cast_fp16 = mul(x = var_373_cast_fp16, y = const_21_promoted_to_fp16)[name = string("op_375_cast_fp16")]; + tensor var_382_cast_fp16 = mul(x = var_380_cast_fp16, y = const_21_promoted_to_fp16)[name = string("op_382_cast_fp16")]; bool rotated_7_interleave_0 = const()[name = string("rotated_7_interleave_0"), val = bool(false)]; - tensor rotated_7_cast_fp16 = concat(axis = var_264, interleave = rotated_7_interleave_0, values = (var_375_cast_fp16, var_367_cast_fp16))[name = string("rotated_7_cast_fp16")]; - tensor var_378_cast_fp16 = mul(x = k_11_cast_fp16, y = cos)[name = string("op_378_cast_fp16")]; - tensor var_379_cast_fp16 = mul(x = rotated_7_cast_fp16, y = sin)[name = string("op_379_cast_fp16")]; - tensor roped_7_cast_fp16 = add(x = var_378_cast_fp16, y = var_379_cast_fp16)[name = string("roped_7_cast_fp16")]; + tensor rotated_7_cast_fp16 = concat(axis = var_270, interleave = rotated_7_interleave_0, values = (var_382_cast_fp16, var_374_cast_fp16))[name = string("rotated_7_cast_fp16")]; + tensor var_385_cast_fp16 = mul(x = k_11_cast_fp16, y = cos)[name = string("op_385_cast_fp16")]; + tensor var_386_cast_fp16 = mul(x = rotated_7_cast_fp16, y = sin)[name = string("op_386_cast_fp16")]; + tensor roped_7_cast_fp16 = add(x = var_385_cast_fp16, y = var_386_cast_fp16)[name = string("roped_7_cast_fp16")]; + tensor v_13_perm_0 = const()[name = string("v_13_perm_0"), val = tensor([0, 1, -1, -2])]; bool k_15_interleave_0 = const()[name = string("k_15_interleave_0"), val = bool(false)]; - tensor k_15_cast_fp16 = concat(axis = var_252, interleave = k_15_interleave_0, values = (k_cache_1, roped_7_cast_fp16))[name = string("k_15_cast_fp16")]; - bool v_11_interleave_0 = const()[name = string("v_11_interleave_0"), val = bool(false)]; - tensor v_11_cast_fp16 = concat(axis = var_252, interleave = v_11_interleave_0, values = (v_cache_1, v_9_cast_fp16))[name = string("v_11_cast_fp16")]; - tensor var_386_begin_0 = const()[name = string("op_386_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_386_end_0 = const()[name = string("op_386_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_386_end_mask_0 = const()[name = string("op_386_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_k_cache_1 = slice_by_index(begin = var_386_begin_0, end = var_386_end_0, end_mask = var_386_end_mask_0, x = k_15_cast_fp16)[name = string("op_386_cast_fp16")]; - tensor var_387_begin_0 = const()[name = string("op_387_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_387_end_0 = const()[name = string("op_387_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_387_end_mask_0 = const()[name = string("op_387_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_v_cache_1 = slice_by_index(begin = var_387_begin_0, end = var_387_end_0, end_mask = var_387_end_mask_0, x = v_11_cast_fp16)[name = string("op_387_cast_fp16")]; - fp16 var_391_to_fp16 = const()[name = string("op_391_to_fp16"), val = fp16(0x1.6ap-4)]; - tensor var_392_cast_fp16 = mul(x = roped_5_cast_fp16, y = var_391_to_fp16)[name = string("op_392_cast_fp16")]; - bool attn_weights_5_transpose_x_0 = const()[name = string("attn_weights_5_transpose_x_0"), val = bool(true)]; - bool attn_weights_5_transpose_y_0 = const()[name = string("attn_weights_5_transpose_y_0"), val = bool(false)]; - tensor attn_weights_5_cast_fp16 = matmul(transpose_x = attn_weights_5_transpose_x_0, transpose_y = attn_weights_5_transpose_y_0, x = var_392_cast_fp16, y = k_15_cast_fp16)[name = string("attn_weights_5_cast_fp16")]; - tensor attn_weights_7_cast_fp16 = add(x = attn_weights_5_cast_fp16, y = mask)[name = string("attn_weights_7_cast_fp16")]; - tensor var_400_cast_fp16 = softmax(axis = var_260, x = attn_weights_7_cast_fp16)[name = string("op_400_cast_fp16")]; - bool attn_5_transpose_x_0 = const()[name = string("attn_5_transpose_x_0"), val = bool(false)]; - bool attn_5_transpose_y_0 = const()[name = string("attn_5_transpose_y_0"), val = bool(true)]; - tensor attn_5_cast_fp16 = matmul(transpose_x = attn_5_transpose_x_0, transpose_y = attn_5_transpose_y_0, x = v_11_cast_fp16, y = var_400_cast_fp16)[name = string("attn_5_cast_fp16")]; - tensor var_404 = const()[name = string("op_404"), val = tensor([1, 4096, 1, -1])]; - tensor input_9_cast_fp16 = reshape(shape = var_404, x = attn_5_cast_fp16)[name = string("input_9_cast_fp16")]; - tensor var_408 = const()[name = string("op_408"), val = tensor([1, 1])]; - tensor var_410 = const()[name = string("op_410"), val = tensor([1, 1])]; - string var_412_pad_type_0 = const()[name = string("op_412_pad_type_0"), val = string("custom")]; - tensor var_412_pad_0 = const()[name = string("op_412_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_412_cast_fp16 = conv(dilations = var_410, groups = var_261, pad = var_412_pad_0, pad_type = var_412_pad_type_0, strides = var_408, weight = blocks_1_attn_proj_weight_palettized_cast_fp16, x = input_9_cast_fp16)[name = string("op_412_cast_fp16")]; + tensor k_15_cast_fp16 = concat(axis = var_258, interleave = k_15_interleave_0, values = (k_cache_1, roped_7_cast_fp16))[name = string("k_15_cast_fp16")]; + bool v_15_interleave_0 = const()[name = string("v_15_interleave_0"), val = bool(false)]; + tensor v_13_cast_fp16 = transpose(perm = v_13_perm_0, x = v_11_cast_fp16)[name = string("transpose_3")]; + tensor v_15_cast_fp16 = concat(axis = var_270, interleave = v_15_interleave_0, values = (v_cache_1, v_13_cast_fp16))[name = string("v_15_cast_fp16")]; + tensor var_397_begin_0 = const()[name = string("op_397_begin_0"), val = tensor([0, 0, 0, 1])]; + tensor var_397_end_0 = const()[name = string("op_397_end_0"), val = tensor([1, 32, 128, 509])]; + tensor var_397_end_mask_0 = const()[name = string("op_397_end_mask_0"), val = tensor([true, true, true, false])]; + tensor new_k_cache_1 = slice_by_index(begin = var_397_begin_0, end = var_397_end_0, end_mask = var_397_end_mask_0, x = k_15_cast_fp16)[name = string("op_397_cast_fp16")]; + tensor var_398_begin_0 = const()[name = string("op_398_begin_0"), val = tensor([0, 0, 1, 0])]; + tensor var_398_end_0 = const()[name = string("op_398_end_0"), val = tensor([1, 32, 509, 128])]; + tensor var_398_end_mask_0 = const()[name = string("op_398_end_mask_0"), val = tensor([true, true, false, true])]; + tensor new_v_cache_1 = slice_by_index(begin = var_398_begin_0, end = var_398_end_0, end_mask = var_398_end_mask_0, x = v_15_cast_fp16)[name = string("op_398_cast_fp16")]; + fp16 var_403_to_fp16 = const()[name = string("op_403_to_fp16"), val = fp16(0x1.6ap-4)]; + tensor var_404_cast_fp16 = mul(x = roped_5_cast_fp16, y = var_403_to_fp16)[name = string("op_404_cast_fp16")]; + bool attn_weights_7_transpose_x_0 = const()[name = string("attn_weights_7_transpose_x_0"), val = bool(true)]; + bool attn_weights_7_transpose_y_0 = const()[name = string("attn_weights_7_transpose_y_0"), val = bool(false)]; + tensor attn_weights_7_cast_fp16 = matmul(transpose_x = attn_weights_7_transpose_x_0, transpose_y = attn_weights_7_transpose_y_0, x = var_404_cast_fp16, y = k_15_cast_fp16)[name = string("attn_weights_7_cast_fp16")]; + tensor attn_weights_9_cast_fp16 = add(x = attn_weights_7_cast_fp16, y = mask)[name = string("attn_weights_9_cast_fp16")]; + tensor attn_weights_11_cast_fp16 = softmax(axis = var_266, x = attn_weights_9_cast_fp16)[name = string("attn_weights_11_cast_fp16")]; + bool var_413_transpose_x_0 = const()[name = string("op_413_transpose_x_0"), val = bool(false)]; + bool var_413_transpose_y_0 = const()[name = string("op_413_transpose_y_0"), val = bool(false)]; + tensor var_413_cast_fp16 = matmul(transpose_x = var_413_transpose_x_0, transpose_y = var_413_transpose_y_0, x = attn_weights_11_cast_fp16, y = v_15_cast_fp16)[name = string("op_413_cast_fp16")]; + tensor attn_3_perm_0 = const()[name = string("attn_3_perm_0"), val = tensor([0, 1, -1, -2])]; + tensor var_416 = const()[name = string("op_416"), val = tensor([1, 4096, 1, -1])]; + tensor attn_3_cast_fp16 = transpose(perm = attn_3_perm_0, x = var_413_cast_fp16)[name = string("transpose_2")]; + tensor input_9_cast_fp16 = reshape(shape = var_416, x = attn_3_cast_fp16)[name = string("input_9_cast_fp16")]; + tensor var_420 = const()[name = string("op_420"), val = tensor([1, 1])]; + tensor var_422 = const()[name = string("op_422"), val = tensor([1, 1])]; + string var_424_pad_type_0 = const()[name = string("op_424_pad_type_0"), val = string("custom")]; + tensor var_424_pad_0 = const()[name = string("op_424_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_424_cast_fp16 = conv(dilations = var_422, groups = var_267, pad = var_424_pad_0, pad_type = var_424_pad_type_0, strides = var_420, weight = blocks_1_attn_proj_weight_palettized_cast_fp16, x = input_9_cast_fp16)[name = string("op_424_cast_fp16")]; tensor blocks_1_attn_proj_output_scales_to_fp16 = const()[name = string("blocks_1_attn_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303702912)))]; - tensor attention_output_3_cast_fp16 = mul(x = var_412_cast_fp16, y = blocks_1_attn_proj_output_scales_to_fp16)[name = string("attention_output_3_cast_fp16")]; - tensor x_25_cast_fp16 = add(x = attention_output_3_cast_fp16, y = x_15_cast_fp16)[name = string("x_25_cast_fp16")]; - tensor var_431_axes_0 = const()[name = string("op_431_axes_0"), val = tensor([-2])]; - tensor var_431_cast_fp16 = squeeze(axes = var_431_axes_0, x = x_25_cast_fp16)[name = string("op_431_cast_fp16")]; - bool var_433_interleave_0 = const()[name = string("op_433_interleave_0"), val = bool(false)]; - tensor eps_chan_7_to_fp16 = const()[name = string("eps_chan_7_to_fp16"), val = tensor([[[0x1.9e8p-3]]])]; - tensor var_433_cast_fp16 = concat(axis = var_261, interleave = var_433_interleave_0, values = (var_431_cast_fp16, eps_chan_7_to_fp16))[name = string("op_433_cast_fp16")]; + tensor attention_output_3_cast_fp16 = mul(x = var_424_cast_fp16, y = blocks_1_attn_proj_output_scales_to_fp16)[name = string("attention_output_3_cast_fp16")]; + tensor x_25_cast_fp16 = add(x = attention_output_3_cast_fp16, y = x_15_cast_fp16)[name = string("x_25_cast_fp16")]; + tensor var_443_axes_0 = const()[name = string("op_443_axes_0"), val = tensor([-2])]; + tensor var_443_cast_fp16 = squeeze(axes = var_443_axes_0, x = x_25_cast_fp16)[name = string("op_443_cast_fp16")]; + bool var_445_interleave_0 = const()[name = string("op_445_interleave_0"), val = bool(false)]; + tensor eps_chan_7_to_fp16 = const()[name = string("eps_chan_7_to_fp16"), val = tensor([[[0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3]]])]; + tensor var_445_cast_fp16 = concat(axis = var_267, interleave = var_445_interleave_0, values = (var_443_cast_fp16, eps_chan_7_to_fp16))[name = string("op_445_cast_fp16")]; tensor x_eps_7_axes_0 = const()[name = string("x_eps_7_axes_0"), val = tensor([-2])]; - tensor x_eps_7_cast_fp16 = expand_dims(axes = x_eps_7_axes_0, x = var_433_cast_fp16)[name = string("x_eps_7_cast_fp16")]; + tensor x_eps_7_cast_fp16 = expand_dims(axes = x_eps_7_axes_0, x = var_445_cast_fp16)[name = string("x_eps_7_cast_fp16")]; tensor norm_x_7_axes_0 = const()[name = string("norm_x_7_axes_0"), val = tensor([1])]; - tensor norm_x_7_cast_fp16 = reduce_l2_norm(axes = norm_x_7_axes_0, keep_dims = var_265, x = x_eps_7_cast_fp16)[name = string("norm_x_7_cast_fp16")]; - tensor x_normed_19_cast_fp16 = real_div(x = x_25_cast_fp16, y = norm_x_7_cast_fp16)[name = string("x_normed_19_cast_fp16")]; - fp16 var_438_to_fp16 = const()[name = string("op_438_to_fp16"), val = fp16(0x1p+6)]; - tensor x_normed_21_cast_fp16 = mul(x = x_normed_19_cast_fp16, y = var_438_to_fp16)[name = string("x_normed_21_cast_fp16")]; + tensor norm_x_7_cast_fp16 = reduce_l2_norm(axes = norm_x_7_axes_0, keep_dims = var_271, x = x_eps_7_cast_fp16)[name = string("norm_x_7_cast_fp16")]; + tensor x_normed_19_cast_fp16 = real_div(x = x_25_cast_fp16, y = norm_x_7_cast_fp16)[name = string("x_normed_19_cast_fp16")]; + fp16 var_450_to_fp16 = const()[name = string("op_450_to_fp16"), val = fp16(0x1p+6)]; + tensor x_normed_21_cast_fp16 = mul(x = x_normed_19_cast_fp16, y = var_450_to_fp16)[name = string("x_normed_21_cast_fp16")]; tensor blocks_1_norm_2_weight_to_fp16 = const()[name = string("blocks_1_norm_2_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303711168)))]; - tensor input_11_cast_fp16 = mul(x = x_normed_21_cast_fp16, y = blocks_1_norm_2_weight_to_fp16)[name = string("input_11_cast_fp16")]; - tensor var_450 = const()[name = string("op_450"), val = tensor([1, 1])]; - tensor var_452 = const()[name = string("op_452"), val = tensor([1, 1])]; - string var_454_pad_type_0 = const()[name = string("op_454_pad_type_0"), val = string("custom")]; - tensor var_454_pad_0 = const()[name = string("op_454_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_454_cast_fp16 = conv(dilations = var_452, groups = var_261, pad = var_454_pad_0, pad_type = var_454_pad_type_0, strides = var_450, weight = blocks_1_mlp_fc_1_weight_palettized_cast_fp16, x = input_11_cast_fp16)[name = string("op_454_cast_fp16")]; + tensor input_11_cast_fp16 = mul(x = x_normed_21_cast_fp16, y = blocks_1_norm_2_weight_to_fp16)[name = string("input_11_cast_fp16")]; + tensor var_462 = const()[name = string("op_462"), val = tensor([1, 1])]; + tensor var_464 = const()[name = string("op_464"), val = tensor([1, 1])]; + string var_466_pad_type_0 = const()[name = string("op_466_pad_type_0"), val = string("custom")]; + tensor var_466_pad_0 = const()[name = string("op_466_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_466_cast_fp16 = conv(dilations = var_464, groups = var_267, pad = var_466_pad_0, pad_type = var_466_pad_type_0, strides = var_462, weight = blocks_1_mlp_fc_1_weight_palettized_cast_fp16, x = input_11_cast_fp16)[name = string("op_466_cast_fp16")]; tensor blocks_1_mlp_fc_1_output_scales_to_fp16 = const()[name = string("blocks_1_mlp_fc_1_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303719424)))]; - tensor input_13_cast_fp16 = mul(x = var_454_cast_fp16, y = blocks_1_mlp_fc_1_output_scales_to_fp16)[name = string("input_13_cast_fp16")]; - tensor var_458 = const()[name = string("op_458"), val = tensor([1, 1])]; - tensor var_460 = const()[name = string("op_460"), val = tensor([1, 1])]; - string var_462_pad_type_0 = const()[name = string("op_462_pad_type_0"), val = string("custom")]; - tensor var_462_pad_0 = const()[name = string("op_462_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_462_cast_fp16 = conv(dilations = var_460, groups = var_261, pad = var_462_pad_0, pad_type = var_462_pad_type_0, strides = var_458, weight = blocks_1_mlp_fc_2_weight_palettized_cast_fp16, x = input_11_cast_fp16)[name = string("op_462_cast_fp16")]; - tensor blocks_1_mlp_fc_2_output_scales_to_fp16 = const()[name = string("blocks_1_mlp_fc_2_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303741504)))]; - tensor x_fc_2_3_cast_fp16 = mul(x = var_462_cast_fp16, y = blocks_1_mlp_fc_2_output_scales_to_fp16)[name = string("x_fc_2_3_cast_fp16")]; - tensor var_464_cast_fp16 = silu(x = input_13_cast_fp16)[name = string("op_464_cast_fp16")]; - tensor input_15_cast_fp16 = mul(x = var_464_cast_fp16, y = x_fc_2_3_cast_fp16)[name = string("input_15_cast_fp16")]; - tensor var_468 = const()[name = string("op_468"), val = tensor([1, 1])]; + tensor input_13_cast_fp16 = mul(x = var_466_cast_fp16, y = blocks_1_mlp_fc_1_output_scales_to_fp16)[name = string("input_13_cast_fp16")]; tensor var_470 = const()[name = string("op_470"), val = tensor([1, 1])]; - string var_472_pad_type_0 = const()[name = string("op_472_pad_type_0"), val = string("custom")]; - tensor var_472_pad_0 = const()[name = string("op_472_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_472_cast_fp16 = conv(dilations = var_470, groups = var_261, pad = var_472_pad_0, pad_type = var_472_pad_type_0, strides = var_468, weight = blocks_1_mlp_proj_weight_palettized_cast_fp16, x = input_15_cast_fp16)[name = string("op_472_cast_fp16")]; + tensor var_472 = const()[name = string("op_472"), val = tensor([1, 1])]; + string var_474_pad_type_0 = const()[name = string("op_474_pad_type_0"), val = string("custom")]; + tensor var_474_pad_0 = const()[name = string("op_474_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_474_cast_fp16 = conv(dilations = var_472, groups = var_267, pad = var_474_pad_0, pad_type = var_474_pad_type_0, strides = var_470, weight = blocks_1_mlp_fc_2_weight_palettized_cast_fp16, x = input_11_cast_fp16)[name = string("op_474_cast_fp16")]; + tensor blocks_1_mlp_fc_2_output_scales_to_fp16 = const()[name = string("blocks_1_mlp_fc_2_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303741504)))]; + tensor x_fc_2_3_cast_fp16 = mul(x = var_474_cast_fp16, y = blocks_1_mlp_fc_2_output_scales_to_fp16)[name = string("x_fc_2_3_cast_fp16")]; + tensor var_476_cast_fp16 = silu(x = input_13_cast_fp16)[name = string("op_476_cast_fp16")]; + tensor input_15_cast_fp16 = mul(x = var_476_cast_fp16, y = x_fc_2_3_cast_fp16)[name = string("input_15_cast_fp16")]; + tensor var_480 = const()[name = string("op_480"), val = tensor([1, 1])]; + tensor var_482 = const()[name = string("op_482"), val = tensor([1, 1])]; + string var_484_pad_type_0 = const()[name = string("op_484_pad_type_0"), val = string("custom")]; + tensor var_484_pad_0 = const()[name = string("op_484_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_484_cast_fp16 = conv(dilations = var_482, groups = var_267, pad = var_484_pad_0, pad_type = var_484_pad_type_0, strides = var_480, weight = blocks_1_mlp_proj_weight_palettized_cast_fp16, x = input_15_cast_fp16)[name = string("op_484_cast_fp16")]; tensor blocks_1_mlp_proj_output_scales_to_fp16 = const()[name = string("blocks_1_mlp_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303763584)))]; - tensor var_473_cast_fp16 = mul(x = var_472_cast_fp16, y = blocks_1_mlp_proj_output_scales_to_fp16)[name = string("op_473_cast_fp16")]; - tensor x_29_cast_fp16 = add(x = var_473_cast_fp16, y = x_25_cast_fp16)[name = string("x_29_cast_fp16")]; - int32 var_484 = const()[name = string("op_484"), val = int32(-1)]; - int32 var_492 = const()[name = string("op_492"), val = int32(3)]; - int32 var_493 = const()[name = string("op_493"), val = int32(1)]; - int32 var_496 = const()[name = string("op_496"), val = int32(-2)]; - bool var_497 = const()[name = string("op_497"), val = bool(true)]; - tensor var_514_axes_0 = const()[name = string("op_514_axes_0"), val = tensor([-2])]; - tensor var_514_cast_fp16 = squeeze(axes = var_514_axes_0, x = x_29_cast_fp16)[name = string("op_514_cast_fp16")]; - bool var_516_interleave_0 = const()[name = string("op_516_interleave_0"), val = bool(false)]; - tensor eps_chan_9_to_fp16 = const()[name = string("eps_chan_9_to_fp16"), val = tensor([[[0x1.9e8p-3]]])]; - tensor var_516_cast_fp16 = concat(axis = var_493, interleave = var_516_interleave_0, values = (var_514_cast_fp16, eps_chan_9_to_fp16))[name = string("op_516_cast_fp16")]; + tensor var_485_cast_fp16 = mul(x = var_484_cast_fp16, y = blocks_1_mlp_proj_output_scales_to_fp16)[name = string("op_485_cast_fp16")]; + tensor x_29_cast_fp16 = add(x = var_485_cast_fp16, y = x_25_cast_fp16)[name = string("x_29_cast_fp16")]; + int32 var_496 = const()[name = string("op_496"), val = int32(-1)]; + int32 var_504 = const()[name = string("op_504"), val = int32(3)]; + int32 var_505 = const()[name = string("op_505"), val = int32(1)]; + int32 var_508 = const()[name = string("op_508"), val = int32(-2)]; + bool var_509 = const()[name = string("op_509"), val = bool(true)]; + tensor var_526_axes_0 = const()[name = string("op_526_axes_0"), val = tensor([-2])]; + tensor var_526_cast_fp16 = squeeze(axes = var_526_axes_0, x = x_29_cast_fp16)[name = string("op_526_cast_fp16")]; + bool var_528_interleave_0 = const()[name = string("op_528_interleave_0"), val = bool(false)]; + tensor eps_chan_9_to_fp16 = const()[name = string("eps_chan_9_to_fp16"), val = tensor([[[0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3]]])]; + tensor var_528_cast_fp16 = concat(axis = var_505, interleave = var_528_interleave_0, values = (var_526_cast_fp16, eps_chan_9_to_fp16))[name = string("op_528_cast_fp16")]; tensor x_eps_9_axes_0 = const()[name = string("x_eps_9_axes_0"), val = tensor([-2])]; - tensor x_eps_9_cast_fp16 = expand_dims(axes = x_eps_9_axes_0, x = var_516_cast_fp16)[name = string("x_eps_9_cast_fp16")]; + tensor x_eps_9_cast_fp16 = expand_dims(axes = x_eps_9_axes_0, x = var_528_cast_fp16)[name = string("x_eps_9_cast_fp16")]; tensor norm_x_9_axes_0 = const()[name = string("norm_x_9_axes_0"), val = tensor([1])]; - tensor norm_x_9_cast_fp16 = reduce_l2_norm(axes = norm_x_9_axes_0, keep_dims = var_497, x = x_eps_9_cast_fp16)[name = string("norm_x_9_cast_fp16")]; - tensor x_normed_25_cast_fp16 = real_div(x = x_29_cast_fp16, y = norm_x_9_cast_fp16)[name = string("x_normed_25_cast_fp16")]; - fp16 var_521_to_fp16 = const()[name = string("op_521_to_fp16"), val = fp16(0x1p+6)]; - tensor x_normed_27_cast_fp16 = mul(x = x_normed_25_cast_fp16, y = var_521_to_fp16)[name = string("x_normed_27_cast_fp16")]; + tensor norm_x_9_cast_fp16 = reduce_l2_norm(axes = norm_x_9_axes_0, keep_dims = var_509, x = x_eps_9_cast_fp16)[name = string("norm_x_9_cast_fp16")]; + tensor x_normed_25_cast_fp16 = real_div(x = x_29_cast_fp16, y = norm_x_9_cast_fp16)[name = string("x_normed_25_cast_fp16")]; + fp16 var_533_to_fp16 = const()[name = string("op_533_to_fp16"), val = fp16(0x1p+6)]; + tensor x_normed_27_cast_fp16 = mul(x = x_normed_25_cast_fp16, y = var_533_to_fp16)[name = string("x_normed_27_cast_fp16")]; tensor blocks_2_norm_1_weight_to_fp16 = const()[name = string("blocks_2_norm_1_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303771840)))]; - tensor x_33_cast_fp16 = mul(x = x_normed_27_cast_fp16, y = blocks_2_norm_1_weight_to_fp16)[name = string("x_33_cast_fp16")]; - tensor var_536 = const()[name = string("op_536"), val = tensor([1, 1])]; - tensor var_538 = const()[name = string("op_538"), val = tensor([1, 1])]; - string var_540_pad_type_0 = const()[name = string("op_540_pad_type_0"), val = string("custom")]; - tensor var_540_pad_0 = const()[name = string("op_540_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_540_cast_fp16 = conv(dilations = var_538, groups = var_493, pad = var_540_pad_0, pad_type = var_540_pad_type_0, strides = var_536, weight = blocks_2_attn_q_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_540_cast_fp16")]; + tensor x_33_cast_fp16 = mul(x = x_normed_27_cast_fp16, y = blocks_2_norm_1_weight_to_fp16)[name = string("x_33_cast_fp16")]; + tensor var_549 = const()[name = string("op_549"), val = tensor([1, 1])]; + tensor var_551 = const()[name = string("op_551"), val = tensor([1, 1])]; + string var_553_pad_type_0 = const()[name = string("op_553_pad_type_0"), val = string("custom")]; + tensor var_553_pad_0 = const()[name = string("op_553_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_553_cast_fp16 = conv(dilations = var_551, groups = var_505, pad = var_553_pad_0, pad_type = var_553_pad_type_0, strides = var_549, weight = blocks_2_attn_q_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_553_cast_fp16")]; tensor blocks_2_attn_q_proj_output_scales_to_fp16 = const()[name = string("blocks_2_attn_q_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303780096)))]; - tensor q_13_cast_fp16 = mul(x = var_540_cast_fp16, y = blocks_2_attn_q_proj_output_scales_to_fp16)[name = string("q_13_cast_fp16")]; - tensor var_544 = const()[name = string("op_544"), val = tensor([1, 1])]; - tensor var_546 = const()[name = string("op_546"), val = tensor([1, 1])]; - string var_548_pad_type_0 = const()[name = string("op_548_pad_type_0"), val = string("custom")]; - tensor var_548_pad_0 = const()[name = string("op_548_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_548_cast_fp16 = conv(dilations = var_546, groups = var_493, pad = var_548_pad_0, pad_type = var_548_pad_type_0, strides = var_544, weight = blocks_2_attn_k_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_548_cast_fp16")]; + tensor q_13_cast_fp16 = mul(x = var_553_cast_fp16, y = blocks_2_attn_q_proj_output_scales_to_fp16)[name = string("q_13_cast_fp16")]; + tensor var_557 = const()[name = string("op_557"), val = tensor([1, 1])]; + tensor var_559 = const()[name = string("op_559"), val = tensor([1, 1])]; + string var_561_pad_type_0 = const()[name = string("op_561_pad_type_0"), val = string("custom")]; + tensor var_561_pad_0 = const()[name = string("op_561_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_561_cast_fp16 = conv(dilations = var_559, groups = var_505, pad = var_561_pad_0, pad_type = var_561_pad_type_0, strides = var_557, weight = blocks_2_attn_k_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_561_cast_fp16")]; tensor blocks_2_attn_k_proj_output_scales_to_fp16 = const()[name = string("blocks_2_attn_k_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303788352)))]; - tensor k_17_cast_fp16 = mul(x = var_548_cast_fp16, y = blocks_2_attn_k_proj_output_scales_to_fp16)[name = string("k_17_cast_fp16")]; - tensor var_552 = const()[name = string("op_552"), val = tensor([1, 1])]; - tensor var_554 = const()[name = string("op_554"), val = tensor([1, 1])]; - string var_556_pad_type_0 = const()[name = string("op_556_pad_type_0"), val = string("custom")]; - tensor var_556_pad_0 = const()[name = string("op_556_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_556_cast_fp16 = conv(dilations = var_554, groups = var_493, pad = var_556_pad_0, pad_type = var_556_pad_type_0, strides = var_552, weight = blocks_2_attn_v_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_556_cast_fp16")]; + tensor k_17_cast_fp16 = mul(x = var_561_cast_fp16, y = blocks_2_attn_k_proj_output_scales_to_fp16)[name = string("k_17_cast_fp16")]; + tensor var_565 = const()[name = string("op_565"), val = tensor([1, 1])]; + tensor var_567 = const()[name = string("op_567"), val = tensor([1, 1])]; + string var_569_pad_type_0 = const()[name = string("op_569_pad_type_0"), val = string("custom")]; + tensor var_569_pad_0 = const()[name = string("op_569_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_569_cast_fp16 = conv(dilations = var_567, groups = var_505, pad = var_569_pad_0, pad_type = var_569_pad_type_0, strides = var_565, weight = blocks_2_attn_v_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_569_cast_fp16")]; tensor blocks_2_attn_v_proj_output_scales_to_fp16 = const()[name = string("blocks_2_attn_v_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303796608)))]; - tensor v_13_cast_fp16 = mul(x = var_556_cast_fp16, y = blocks_2_attn_v_proj_output_scales_to_fp16)[name = string("v_13_cast_fp16")]; - tensor var_558 = const()[name = string("op_558"), val = tensor([1, 32, 128, 1])]; - tensor q_15_cast_fp16 = reshape(shape = var_558, x = q_13_cast_fp16)[name = string("q_15_cast_fp16")]; - tensor var_560 = const()[name = string("op_560"), val = tensor([1, 32, 128, 1])]; - tensor k_19_cast_fp16 = reshape(shape = var_560, x = k_17_cast_fp16)[name = string("k_19_cast_fp16")]; - tensor var_562 = const()[name = string("op_562"), val = tensor([1, 32, 128, 1])]; - tensor v_15_cast_fp16 = reshape(shape = var_562, x = v_13_cast_fp16)[name = string("v_15_cast_fp16")]; - tensor var_574_begin_0 = const()[name = string("op_574_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_574_end_0 = const()[name = string("op_574_end_0"), val = tensor([1, 32, 64, 1])]; - tensor var_574_end_mask_0 = const()[name = string("op_574_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_574_cast_fp16 = slice_by_index(begin = var_574_begin_0, end = var_574_end_0, end_mask = var_574_end_mask_0, x = q_15_cast_fp16)[name = string("op_574_cast_fp16")]; - tensor var_580_begin_0 = const()[name = string("op_580_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_580_end_0 = const()[name = string("op_580_end_0"), val = tensor([1, 32, 128, 1])]; - tensor var_580_end_mask_0 = const()[name = string("op_580_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_580_cast_fp16 = slice_by_index(begin = var_580_begin_0, end = var_580_end_0, end_mask = var_580_end_mask_0, x = q_15_cast_fp16)[name = string("op_580_cast_fp16")]; + tensor v_17_cast_fp16 = mul(x = var_569_cast_fp16, y = blocks_2_attn_v_proj_output_scales_to_fp16)[name = string("v_17_cast_fp16")]; + tensor var_571 = const()[name = string("op_571"), val = tensor([1, 32, 128, 4])]; + tensor q_15_cast_fp16 = reshape(shape = var_571, x = q_13_cast_fp16)[name = string("q_15_cast_fp16")]; + tensor var_573 = const()[name = string("op_573"), val = tensor([1, 32, 128, 4])]; + tensor k_19_cast_fp16 = reshape(shape = var_573, x = k_17_cast_fp16)[name = string("k_19_cast_fp16")]; + tensor var_575 = const()[name = string("op_575"), val = tensor([1, 32, 128, 4])]; + tensor v_19_cast_fp16 = reshape(shape = var_575, x = v_17_cast_fp16)[name = string("v_19_cast_fp16")]; + tensor var_587_begin_0 = const()[name = string("op_587_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_587_end_0 = const()[name = string("op_587_end_0"), val = tensor([1, 32, 64, 4])]; + tensor var_587_end_mask_0 = const()[name = string("op_587_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_587_cast_fp16 = slice_by_index(begin = var_587_begin_0, end = var_587_end_0, end_mask = var_587_end_mask_0, x = q_15_cast_fp16)[name = string("op_587_cast_fp16")]; + tensor var_593_begin_0 = const()[name = string("op_593_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_593_end_0 = const()[name = string("op_593_end_0"), val = tensor([1, 32, 128, 4])]; + tensor var_593_end_mask_0 = const()[name = string("op_593_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_593_cast_fp16 = slice_by_index(begin = var_593_begin_0, end = var_593_end_0, end_mask = var_593_end_mask_0, x = q_15_cast_fp16)[name = string("op_593_cast_fp16")]; fp16 const_32_promoted_to_fp16 = const()[name = string("const_32_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_582_cast_fp16 = mul(x = var_580_cast_fp16, y = const_32_promoted_to_fp16)[name = string("op_582_cast_fp16")]; + tensor var_595_cast_fp16 = mul(x = var_593_cast_fp16, y = const_32_promoted_to_fp16)[name = string("op_595_cast_fp16")]; bool rotated_9_interleave_0 = const()[name = string("rotated_9_interleave_0"), val = bool(false)]; - tensor rotated_9_cast_fp16 = concat(axis = var_496, interleave = rotated_9_interleave_0, values = (var_582_cast_fp16, var_574_cast_fp16))[name = string("rotated_9_cast_fp16")]; - tensor var_585_cast_fp16 = mul(x = q_15_cast_fp16, y = cos)[name = string("op_585_cast_fp16")]; - tensor var_586_cast_fp16 = mul(x = rotated_9_cast_fp16, y = sin)[name = string("op_586_cast_fp16")]; - tensor roped_9_cast_fp16 = add(x = var_585_cast_fp16, y = var_586_cast_fp16)[name = string("roped_9_cast_fp16")]; - tensor var_599_begin_0 = const()[name = string("op_599_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_599_end_0 = const()[name = string("op_599_end_0"), val = tensor([1, 32, 64, 1])]; - tensor var_599_end_mask_0 = const()[name = string("op_599_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_599_cast_fp16 = slice_by_index(begin = var_599_begin_0, end = var_599_end_0, end_mask = var_599_end_mask_0, x = k_19_cast_fp16)[name = string("op_599_cast_fp16")]; - tensor var_605_begin_0 = const()[name = string("op_605_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_605_end_0 = const()[name = string("op_605_end_0"), val = tensor([1, 32, 128, 1])]; - tensor var_605_end_mask_0 = const()[name = string("op_605_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_605_cast_fp16 = slice_by_index(begin = var_605_begin_0, end = var_605_end_0, end_mask = var_605_end_mask_0, x = k_19_cast_fp16)[name = string("op_605_cast_fp16")]; + tensor rotated_9_cast_fp16 = concat(axis = var_508, interleave = rotated_9_interleave_0, values = (var_595_cast_fp16, var_587_cast_fp16))[name = string("rotated_9_cast_fp16")]; + tensor var_598_cast_fp16 = mul(x = q_15_cast_fp16, y = cos)[name = string("op_598_cast_fp16")]; + tensor var_599_cast_fp16 = mul(x = rotated_9_cast_fp16, y = sin)[name = string("op_599_cast_fp16")]; + tensor roped_9_cast_fp16 = add(x = var_598_cast_fp16, y = var_599_cast_fp16)[name = string("roped_9_cast_fp16")]; + tensor var_612_begin_0 = const()[name = string("op_612_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_612_end_0 = const()[name = string("op_612_end_0"), val = tensor([1, 32, 64, 4])]; + tensor var_612_end_mask_0 = const()[name = string("op_612_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_612_cast_fp16 = slice_by_index(begin = var_612_begin_0, end = var_612_end_0, end_mask = var_612_end_mask_0, x = k_19_cast_fp16)[name = string("op_612_cast_fp16")]; + tensor var_618_begin_0 = const()[name = string("op_618_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_618_end_0 = const()[name = string("op_618_end_0"), val = tensor([1, 32, 128, 4])]; + tensor var_618_end_mask_0 = const()[name = string("op_618_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_618_cast_fp16 = slice_by_index(begin = var_618_begin_0, end = var_618_end_0, end_mask = var_618_end_mask_0, x = k_19_cast_fp16)[name = string("op_618_cast_fp16")]; fp16 const_34_promoted_to_fp16 = const()[name = string("const_34_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_607_cast_fp16 = mul(x = var_605_cast_fp16, y = const_34_promoted_to_fp16)[name = string("op_607_cast_fp16")]; + tensor var_620_cast_fp16 = mul(x = var_618_cast_fp16, y = const_34_promoted_to_fp16)[name = string("op_620_cast_fp16")]; bool rotated_interleave_0 = const()[name = string("rotated_interleave_0"), val = bool(false)]; - tensor rotated_cast_fp16 = concat(axis = var_496, interleave = rotated_interleave_0, values = (var_607_cast_fp16, var_599_cast_fp16))[name = string("rotated_cast_fp16")]; - tensor var_610_cast_fp16 = mul(x = k_19_cast_fp16, y = cos)[name = string("op_610_cast_fp16")]; - tensor var_611_cast_fp16 = mul(x = rotated_cast_fp16, y = sin)[name = string("op_611_cast_fp16")]; - tensor roped_cast_fp16 = add(x = var_610_cast_fp16, y = var_611_cast_fp16)[name = string("roped_cast_fp16")]; + tensor rotated_cast_fp16 = concat(axis = var_508, interleave = rotated_interleave_0, values = (var_620_cast_fp16, var_612_cast_fp16))[name = string("rotated_cast_fp16")]; + tensor var_623_cast_fp16 = mul(x = k_19_cast_fp16, y = cos)[name = string("op_623_cast_fp16")]; + tensor var_624_cast_fp16 = mul(x = rotated_cast_fp16, y = sin)[name = string("op_624_cast_fp16")]; + tensor roped_cast_fp16 = add(x = var_623_cast_fp16, y = var_624_cast_fp16)[name = string("roped_cast_fp16")]; + tensor v_21_perm_0 = const()[name = string("v_21_perm_0"), val = tensor([0, 1, -1, -2])]; bool k_interleave_0 = const()[name = string("k_interleave_0"), val = bool(false)]; - tensor k_cast_fp16 = concat(axis = var_484, interleave = k_interleave_0, values = (k_cache_2, roped_cast_fp16))[name = string("k_cast_fp16")]; + tensor k_cast_fp16 = concat(axis = var_496, interleave = k_interleave_0, values = (k_cache_2, roped_cast_fp16))[name = string("k_cast_fp16")]; bool v_interleave_0 = const()[name = string("v_interleave_0"), val = bool(false)]; - tensor v_cast_fp16 = concat(axis = var_484, interleave = v_interleave_0, values = (v_cache_2, v_15_cast_fp16))[name = string("v_cast_fp16")]; - tensor var_618_begin_0 = const()[name = string("op_618_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_618_end_0 = const()[name = string("op_618_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_618_end_mask_0 = const()[name = string("op_618_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_k_cache_2 = slice_by_index(begin = var_618_begin_0, end = var_618_end_0, end_mask = var_618_end_mask_0, x = k_cast_fp16)[name = string("op_618_cast_fp16")]; - tensor var_619_begin_0 = const()[name = string("op_619_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_619_end_0 = const()[name = string("op_619_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_619_end_mask_0 = const()[name = string("op_619_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_v_cache_2 = slice_by_index(begin = var_619_begin_0, end = var_619_end_0, end_mask = var_619_end_mask_0, x = v_cast_fp16)[name = string("op_619_cast_fp16")]; - fp16 var_623_to_fp16 = const()[name = string("op_623_to_fp16"), val = fp16(0x1.6ap-4)]; - tensor var_624_cast_fp16 = mul(x = roped_9_cast_fp16, y = var_623_to_fp16)[name = string("op_624_cast_fp16")]; - bool attn_weights_9_transpose_x_0 = const()[name = string("attn_weights_9_transpose_x_0"), val = bool(true)]; - bool attn_weights_9_transpose_y_0 = const()[name = string("attn_weights_9_transpose_y_0"), val = bool(false)]; - tensor attn_weights_9_cast_fp16 = matmul(transpose_x = attn_weights_9_transpose_x_0, transpose_y = attn_weights_9_transpose_y_0, x = var_624_cast_fp16, y = k_cast_fp16)[name = string("attn_weights_9_cast_fp16")]; - tensor attn_weights_cast_fp16 = add(x = attn_weights_9_cast_fp16, y = mask)[name = string("attn_weights_cast_fp16")]; - tensor var_632_cast_fp16 = softmax(axis = var_492, x = attn_weights_cast_fp16)[name = string("op_632_cast_fp16")]; - bool attn_9_transpose_x_0 = const()[name = string("attn_9_transpose_x_0"), val = bool(false)]; - bool attn_9_transpose_y_0 = const()[name = string("attn_9_transpose_y_0"), val = bool(true)]; - tensor attn_9_cast_fp16 = matmul(transpose_x = attn_9_transpose_x_0, transpose_y = attn_9_transpose_y_0, x = v_cast_fp16, y = var_632_cast_fp16)[name = string("attn_9_cast_fp16")]; - tensor var_636 = const()[name = string("op_636"), val = tensor([1, 4096, 1, -1])]; - tensor input_17_cast_fp16 = reshape(shape = var_636, x = attn_9_cast_fp16)[name = string("input_17_cast_fp16")]; - tensor var_640 = const()[name = string("op_640"), val = tensor([1, 1])]; - tensor var_642 = const()[name = string("op_642"), val = tensor([1, 1])]; - string var_644_pad_type_0 = const()[name = string("op_644_pad_type_0"), val = string("custom")]; - tensor var_644_pad_0 = const()[name = string("op_644_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_644_cast_fp16 = conv(dilations = var_642, groups = var_493, pad = var_644_pad_0, pad_type = var_644_pad_type_0, strides = var_640, weight = blocks_2_attn_proj_weight_palettized_cast_fp16, x = input_17_cast_fp16)[name = string("op_644_cast_fp16")]; + tensor v_21_cast_fp16 = transpose(perm = v_21_perm_0, x = v_19_cast_fp16)[name = string("transpose_1")]; + tensor v_cast_fp16 = concat(axis = var_508, interleave = v_interleave_0, values = (v_cache_2, v_21_cast_fp16))[name = string("v_cast_fp16")]; + tensor var_635_begin_0 = const()[name = string("op_635_begin_0"), val = tensor([0, 0, 0, 1])]; + tensor var_635_end_0 = const()[name = string("op_635_end_0"), val = tensor([1, 32, 128, 509])]; + tensor var_635_end_mask_0 = const()[name = string("op_635_end_mask_0"), val = tensor([true, true, true, false])]; + tensor new_k_cache_2 = slice_by_index(begin = var_635_begin_0, end = var_635_end_0, end_mask = var_635_end_mask_0, x = k_cast_fp16)[name = string("op_635_cast_fp16")]; + tensor var_636_begin_0 = const()[name = string("op_636_begin_0"), val = tensor([0, 0, 1, 0])]; + tensor var_636_end_0 = const()[name = string("op_636_end_0"), val = tensor([1, 32, 509, 128])]; + tensor var_636_end_mask_0 = const()[name = string("op_636_end_mask_0"), val = tensor([true, true, false, true])]; + tensor new_v_cache_2 = slice_by_index(begin = var_636_begin_0, end = var_636_end_0, end_mask = var_636_end_mask_0, x = v_cast_fp16)[name = string("op_636_cast_fp16")]; + fp16 var_641_to_fp16 = const()[name = string("op_641_to_fp16"), val = fp16(0x1.6ap-4)]; + tensor var_642_cast_fp16 = mul(x = roped_9_cast_fp16, y = var_641_to_fp16)[name = string("op_642_cast_fp16")]; + bool attn_weights_13_transpose_x_0 = const()[name = string("attn_weights_13_transpose_x_0"), val = bool(true)]; + bool attn_weights_13_transpose_y_0 = const()[name = string("attn_weights_13_transpose_y_0"), val = bool(false)]; + tensor attn_weights_13_cast_fp16 = matmul(transpose_x = attn_weights_13_transpose_x_0, transpose_y = attn_weights_13_transpose_y_0, x = var_642_cast_fp16, y = k_cast_fp16)[name = string("attn_weights_13_cast_fp16")]; + tensor attn_weights_15_cast_fp16 = add(x = attn_weights_13_cast_fp16, y = mask)[name = string("attn_weights_15_cast_fp16")]; + tensor attn_weights_cast_fp16 = softmax(axis = var_504, x = attn_weights_15_cast_fp16)[name = string("attn_weights_cast_fp16")]; + bool var_651_transpose_x_0 = const()[name = string("op_651_transpose_x_0"), val = bool(false)]; + bool var_651_transpose_y_0 = const()[name = string("op_651_transpose_y_0"), val = bool(false)]; + tensor var_651_cast_fp16 = matmul(transpose_x = var_651_transpose_x_0, transpose_y = var_651_transpose_y_0, x = attn_weights_cast_fp16, y = v_cast_fp16)[name = string("op_651_cast_fp16")]; + tensor attn_5_perm_0 = const()[name = string("attn_5_perm_0"), val = tensor([0, 1, -1, -2])]; + tensor var_654 = const()[name = string("op_654"), val = tensor([1, 4096, 1, -1])]; + tensor attn_5_cast_fp16 = transpose(perm = attn_5_perm_0, x = var_651_cast_fp16)[name = string("transpose_0")]; + tensor input_17_cast_fp16 = reshape(shape = var_654, x = attn_5_cast_fp16)[name = string("input_17_cast_fp16")]; + tensor var_658 = const()[name = string("op_658"), val = tensor([1, 1])]; + tensor var_660 = const()[name = string("op_660"), val = tensor([1, 1])]; + string var_662_pad_type_0 = const()[name = string("op_662_pad_type_0"), val = string("custom")]; + tensor var_662_pad_0 = const()[name = string("op_662_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_662_cast_fp16 = conv(dilations = var_660, groups = var_505, pad = var_662_pad_0, pad_type = var_662_pad_type_0, strides = var_658, weight = blocks_2_attn_proj_weight_palettized_cast_fp16, x = input_17_cast_fp16)[name = string("op_662_cast_fp16")]; tensor blocks_2_attn_proj_output_scales_to_fp16 = const()[name = string("blocks_2_attn_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303804864)))]; - tensor attention_output_cast_fp16 = mul(x = var_644_cast_fp16, y = blocks_2_attn_proj_output_scales_to_fp16)[name = string("attention_output_cast_fp16")]; - tensor x_39_cast_fp16 = add(x = attention_output_cast_fp16, y = x_29_cast_fp16)[name = string("x_39_cast_fp16")]; - tensor var_663_axes_0 = const()[name = string("op_663_axes_0"), val = tensor([-2])]; - tensor var_663_cast_fp16 = squeeze(axes = var_663_axes_0, x = x_39_cast_fp16)[name = string("op_663_cast_fp16")]; - bool var_665_interleave_0 = const()[name = string("op_665_interleave_0"), val = bool(false)]; - tensor eps_chan_to_fp16 = const()[name = string("eps_chan_to_fp16"), val = tensor([[[0x1.9e8p-3]]])]; - tensor var_665_cast_fp16 = concat(axis = var_493, interleave = var_665_interleave_0, values = (var_663_cast_fp16, eps_chan_to_fp16))[name = string("op_665_cast_fp16")]; + tensor attention_output_cast_fp16 = mul(x = var_662_cast_fp16, y = blocks_2_attn_proj_output_scales_to_fp16)[name = string("attention_output_cast_fp16")]; + tensor x_39_cast_fp16 = add(x = attention_output_cast_fp16, y = x_29_cast_fp16)[name = string("x_39_cast_fp16")]; + tensor var_681_axes_0 = const()[name = string("op_681_axes_0"), val = tensor([-2])]; + tensor var_681_cast_fp16 = squeeze(axes = var_681_axes_0, x = x_39_cast_fp16)[name = string("op_681_cast_fp16")]; + bool var_683_interleave_0 = const()[name = string("op_683_interleave_0"), val = bool(false)]; + tensor eps_chan_to_fp16 = const()[name = string("eps_chan_to_fp16"), val = tensor([[[0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3, 0x1.9e8p-3]]])]; + tensor var_683_cast_fp16 = concat(axis = var_505, interleave = var_683_interleave_0, values = (var_681_cast_fp16, eps_chan_to_fp16))[name = string("op_683_cast_fp16")]; tensor x_eps_axes_0 = const()[name = string("x_eps_axes_0"), val = tensor([-2])]; - tensor x_eps_cast_fp16 = expand_dims(axes = x_eps_axes_0, x = var_665_cast_fp16)[name = string("x_eps_cast_fp16")]; + tensor x_eps_cast_fp16 = expand_dims(axes = x_eps_axes_0, x = var_683_cast_fp16)[name = string("x_eps_cast_fp16")]; tensor norm_x_axes_0 = const()[name = string("norm_x_axes_0"), val = tensor([1])]; - tensor norm_x_cast_fp16 = reduce_l2_norm(axes = norm_x_axes_0, keep_dims = var_497, x = x_eps_cast_fp16)[name = string("norm_x_cast_fp16")]; - tensor x_normed_31_cast_fp16 = real_div(x = x_39_cast_fp16, y = norm_x_cast_fp16)[name = string("x_normed_31_cast_fp16")]; - fp16 var_670_to_fp16 = const()[name = string("op_670_to_fp16"), val = fp16(0x1p+6)]; - tensor x_normed_33_cast_fp16 = mul(x = x_normed_31_cast_fp16, y = var_670_to_fp16)[name = string("x_normed_33_cast_fp16")]; + tensor norm_x_cast_fp16 = reduce_l2_norm(axes = norm_x_axes_0, keep_dims = var_509, x = x_eps_cast_fp16)[name = string("norm_x_cast_fp16")]; + tensor x_normed_31_cast_fp16 = real_div(x = x_39_cast_fp16, y = norm_x_cast_fp16)[name = string("x_normed_31_cast_fp16")]; + fp16 var_688_to_fp16 = const()[name = string("op_688_to_fp16"), val = fp16(0x1p+6)]; + tensor x_normed_33_cast_fp16 = mul(x = x_normed_31_cast_fp16, y = var_688_to_fp16)[name = string("x_normed_33_cast_fp16")]; tensor blocks_2_norm_2_weight_to_fp16 = const()[name = string("blocks_2_norm_2_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303813120)))]; - tensor input_19_cast_fp16 = mul(x = x_normed_33_cast_fp16, y = blocks_2_norm_2_weight_to_fp16)[name = string("input_19_cast_fp16")]; - tensor var_682 = const()[name = string("op_682"), val = tensor([1, 1])]; - tensor var_684 = const()[name = string("op_684"), val = tensor([1, 1])]; - string var_686_pad_type_0 = const()[name = string("op_686_pad_type_0"), val = string("custom")]; - tensor var_686_pad_0 = const()[name = string("op_686_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_686_cast_fp16 = conv(dilations = var_684, groups = var_493, pad = var_686_pad_0, pad_type = var_686_pad_type_0, strides = var_682, weight = blocks_2_mlp_fc_1_weight_palettized_cast_fp16, x = input_19_cast_fp16)[name = string("op_686_cast_fp16")]; - tensor blocks_2_mlp_fc_1_output_scales_to_fp16 = const()[name = string("blocks_2_mlp_fc_1_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303821376)))]; - tensor input_21_cast_fp16 = mul(x = var_686_cast_fp16, y = blocks_2_mlp_fc_1_output_scales_to_fp16)[name = string("input_21_cast_fp16")]; - tensor var_690 = const()[name = string("op_690"), val = tensor([1, 1])]; - tensor var_692 = const()[name = string("op_692"), val = tensor([1, 1])]; - string var_694_pad_type_0 = const()[name = string("op_694_pad_type_0"), val = string("custom")]; - tensor var_694_pad_0 = const()[name = string("op_694_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_694_cast_fp16 = conv(dilations = var_692, groups = var_493, pad = var_694_pad_0, pad_type = var_694_pad_type_0, strides = var_690, weight = blocks_2_mlp_fc_2_weight_palettized_cast_fp16, x = input_19_cast_fp16)[name = string("op_694_cast_fp16")]; - tensor blocks_2_mlp_fc_2_output_scales_to_fp16 = const()[name = string("blocks_2_mlp_fc_2_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303843456)))]; - tensor x_fc_2_cast_fp16 = mul(x = var_694_cast_fp16, y = blocks_2_mlp_fc_2_output_scales_to_fp16)[name = string("x_fc_2_cast_fp16")]; - tensor var_696_cast_fp16 = silu(x = input_21_cast_fp16)[name = string("op_696_cast_fp16")]; - tensor input_cast_fp16 = mul(x = var_696_cast_fp16, y = x_fc_2_cast_fp16)[name = string("input_cast_fp16")]; + tensor input_19_cast_fp16 = mul(x = x_normed_33_cast_fp16, y = blocks_2_norm_2_weight_to_fp16)[name = string("input_19_cast_fp16")]; tensor var_700 = const()[name = string("op_700"), val = tensor([1, 1])]; tensor var_702 = const()[name = string("op_702"), val = tensor([1, 1])]; string var_704_pad_type_0 = const()[name = string("op_704_pad_type_0"), val = string("custom")]; tensor var_704_pad_0 = const()[name = string("op_704_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_704_cast_fp16 = conv(dilations = var_702, groups = var_493, pad = var_704_pad_0, pad_type = var_704_pad_type_0, strides = var_700, weight = blocks_2_mlp_proj_weight_palettized_cast_fp16, x = input_cast_fp16)[name = string("op_704_cast_fp16")]; + tensor var_704_cast_fp16 = conv(dilations = var_702, groups = var_505, pad = var_704_pad_0, pad_type = var_704_pad_type_0, strides = var_700, weight = blocks_2_mlp_fc_1_weight_palettized_cast_fp16, x = input_19_cast_fp16)[name = string("op_704_cast_fp16")]; + tensor blocks_2_mlp_fc_1_output_scales_to_fp16 = const()[name = string("blocks_2_mlp_fc_1_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303821376)))]; + tensor input_21_cast_fp16 = mul(x = var_704_cast_fp16, y = blocks_2_mlp_fc_1_output_scales_to_fp16)[name = string("input_21_cast_fp16")]; + tensor var_708 = const()[name = string("op_708"), val = tensor([1, 1])]; + tensor var_710 = const()[name = string("op_710"), val = tensor([1, 1])]; + string var_712_pad_type_0 = const()[name = string("op_712_pad_type_0"), val = string("custom")]; + tensor var_712_pad_0 = const()[name = string("op_712_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_712_cast_fp16 = conv(dilations = var_710, groups = var_505, pad = var_712_pad_0, pad_type = var_712_pad_type_0, strides = var_708, weight = blocks_2_mlp_fc_2_weight_palettized_cast_fp16, x = input_19_cast_fp16)[name = string("op_712_cast_fp16")]; + tensor blocks_2_mlp_fc_2_output_scales_to_fp16 = const()[name = string("blocks_2_mlp_fc_2_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303843456)))]; + tensor x_fc_2_cast_fp16 = mul(x = var_712_cast_fp16, y = blocks_2_mlp_fc_2_output_scales_to_fp16)[name = string("x_fc_2_cast_fp16")]; + tensor var_714_cast_fp16 = silu(x = input_21_cast_fp16)[name = string("op_714_cast_fp16")]; + tensor input_cast_fp16 = mul(x = var_714_cast_fp16, y = x_fc_2_cast_fp16)[name = string("input_cast_fp16")]; + tensor var_718 = const()[name = string("op_718"), val = tensor([1, 1])]; + tensor var_720 = const()[name = string("op_720"), val = tensor([1, 1])]; + string var_722_pad_type_0 = const()[name = string("op_722_pad_type_0"), val = string("custom")]; + tensor var_722_pad_0 = const()[name = string("op_722_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_722_cast_fp16 = conv(dilations = var_720, groups = var_505, pad = var_722_pad_0, pad_type = var_722_pad_type_0, strides = var_718, weight = blocks_2_mlp_proj_weight_palettized_cast_fp16, x = input_cast_fp16)[name = string("op_722_cast_fp16")]; tensor blocks_2_mlp_proj_output_scales_to_fp16 = const()[name = string("blocks_2_mlp_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303865536)))]; - tensor var_705_cast_fp16 = mul(x = var_704_cast_fp16, y = blocks_2_mlp_proj_output_scales_to_fp16)[name = string("op_705_cast_fp16")]; - tensor new_x = add(x = var_705_cast_fp16, y = x_39_cast_fp16)[name = string("op_706_cast_fp16")]; + tensor var_723_cast_fp16 = mul(x = var_722_cast_fp16, y = blocks_2_mlp_proj_output_scales_to_fp16)[name = string("op_723_cast_fp16")]; + tensor new_x = add(x = var_723_cast_fp16, y = x_39_cast_fp16)[name = string("op_724_cast_fp16")]; } -> (new_x, new_k_cache_0, new_k_cache_1, new_k_cache_2, new_v_cache_0, new_v_cache_1, new_v_cache_2); func input_512_context_512(tensor cos, tensor mask, tensor sin, tensor x) { tensor blocks_0_attn_q_proj_weight_palettized_cast_fp16 = constexpr_lut_to_dense(indices = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64))), lut = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8388736))))[name = string("blocks_0_attn_q_proj_weight_palettized_cast_fp16")]; @@ -502,86 +514,86 @@ program(1.3) tensor x_normed_3_cast_fp16 = mul(x = x_normed_1_cast_fp16, y = var_54_to_fp16)[name = string("x_normed_3_cast_fp16")]; tensor blocks_0_norm_1_weight_to_fp16 = const()[name = string("blocks_0_norm_1_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303567936)))]; tensor x_5_cast_fp16 = mul(x = x_normed_3_cast_fp16, y = blocks_0_norm_1_weight_to_fp16)[name = string("x_5_cast_fp16")]; - tensor var_66 = const()[name = string("op_66"), val = tensor([1, 1])]; - tensor var_68 = const()[name = string("op_68"), val = tensor([1, 1])]; - string var_70_pad_type_0 = const()[name = string("op_70_pad_type_0"), val = string("custom")]; - tensor var_70_pad_0 = const()[name = string("op_70_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_70_cast_fp16 = conv(dilations = var_68, groups = var_25, pad = var_70_pad_0, pad_type = var_70_pad_type_0, strides = var_66, weight = blocks_0_attn_q_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_70_cast_fp16")]; + tensor var_67 = const()[name = string("op_67"), val = tensor([1, 1])]; + tensor var_69 = const()[name = string("op_69"), val = tensor([1, 1])]; + string var_71_pad_type_0 = const()[name = string("op_71_pad_type_0"), val = string("custom")]; + tensor var_71_pad_0 = const()[name = string("op_71_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_71_cast_fp16 = conv(dilations = var_69, groups = var_25, pad = var_71_pad_0, pad_type = var_71_pad_type_0, strides = var_67, weight = blocks_0_attn_q_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_71_cast_fp16")]; tensor blocks_0_attn_q_proj_output_scales_to_fp16 = const()[name = string("blocks_0_attn_q_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303576192)))]; - tensor q_1_cast_fp16 = mul(x = var_70_cast_fp16, y = blocks_0_attn_q_proj_output_scales_to_fp16)[name = string("q_1_cast_fp16")]; - tensor var_74 = const()[name = string("op_74"), val = tensor([1, 1])]; - tensor var_76 = const()[name = string("op_76"), val = tensor([1, 1])]; - string var_78_pad_type_0 = const()[name = string("op_78_pad_type_0"), val = string("custom")]; - tensor var_78_pad_0 = const()[name = string("op_78_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_78_cast_fp16 = conv(dilations = var_76, groups = var_25, pad = var_78_pad_0, pad_type = var_78_pad_type_0, strides = var_74, weight = blocks_0_attn_k_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_78_cast_fp16")]; + tensor q_1_cast_fp16 = mul(x = var_71_cast_fp16, y = blocks_0_attn_q_proj_output_scales_to_fp16)[name = string("q_1_cast_fp16")]; + tensor var_75 = const()[name = string("op_75"), val = tensor([1, 1])]; + tensor var_77 = const()[name = string("op_77"), val = tensor([1, 1])]; + string var_79_pad_type_0 = const()[name = string("op_79_pad_type_0"), val = string("custom")]; + tensor var_79_pad_0 = const()[name = string("op_79_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_79_cast_fp16 = conv(dilations = var_77, groups = var_25, pad = var_79_pad_0, pad_type = var_79_pad_type_0, strides = var_75, weight = blocks_0_attn_k_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_79_cast_fp16")]; tensor blocks_0_attn_k_proj_output_scales_to_fp16 = const()[name = string("blocks_0_attn_k_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303584448)))]; - tensor k_1_cast_fp16 = mul(x = var_78_cast_fp16, y = blocks_0_attn_k_proj_output_scales_to_fp16)[name = string("k_1_cast_fp16")]; - tensor var_82 = const()[name = string("op_82"), val = tensor([1, 1])]; - tensor var_84 = const()[name = string("op_84"), val = tensor([1, 1])]; - string var_86_pad_type_0 = const()[name = string("op_86_pad_type_0"), val = string("custom")]; - tensor var_86_pad_0 = const()[name = string("op_86_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_86_cast_fp16 = conv(dilations = var_84, groups = var_25, pad = var_86_pad_0, pad_type = var_86_pad_type_0, strides = var_82, weight = blocks_0_attn_v_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_86_cast_fp16")]; + tensor k_1_cast_fp16 = mul(x = var_79_cast_fp16, y = blocks_0_attn_k_proj_output_scales_to_fp16)[name = string("k_1_cast_fp16")]; + tensor var_83 = const()[name = string("op_83"), val = tensor([1, 1])]; + tensor var_85 = const()[name = string("op_85"), val = tensor([1, 1])]; + string var_87_pad_type_0 = const()[name = string("op_87_pad_type_0"), val = string("custom")]; + tensor var_87_pad_0 = const()[name = string("op_87_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_87_cast_fp16 = conv(dilations = var_85, groups = var_25, pad = var_87_pad_0, pad_type = var_87_pad_type_0, strides = var_83, weight = blocks_0_attn_v_proj_weight_palettized_cast_fp16, x = x_5_cast_fp16)[name = string("op_87_cast_fp16")]; tensor blocks_0_attn_v_proj_output_scales_to_fp16 = const()[name = string("blocks_0_attn_v_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303592704)))]; - tensor v_1_cast_fp16 = mul(x = var_86_cast_fp16, y = blocks_0_attn_v_proj_output_scales_to_fp16)[name = string("v_1_cast_fp16")]; - tensor var_88 = const()[name = string("op_88"), val = tensor([1, 32, 128, 512])]; - tensor q_3_cast_fp16 = reshape(shape = var_88, x = q_1_cast_fp16)[name = string("q_3_cast_fp16")]; - tensor var_90 = const()[name = string("op_90"), val = tensor([1, 32, 128, 512])]; - tensor k_3_cast_fp16 = reshape(shape = var_90, x = k_1_cast_fp16)[name = string("k_3_cast_fp16")]; - tensor var_92 = const()[name = string("op_92"), val = tensor([1, 32, 128, 512])]; - tensor v_3_cast_fp16 = reshape(shape = var_92, x = v_1_cast_fp16)[name = string("v_3_cast_fp16")]; - tensor var_104_begin_0 = const()[name = string("op_104_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_104_end_0 = const()[name = string("op_104_end_0"), val = tensor([1, 32, 64, 512])]; - tensor var_104_end_mask_0 = const()[name = string("op_104_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_104_cast_fp16 = slice_by_index(begin = var_104_begin_0, end = var_104_end_0, end_mask = var_104_end_mask_0, x = q_3_cast_fp16)[name = string("op_104_cast_fp16")]; - tensor var_110_begin_0 = const()[name = string("op_110_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_110_end_0 = const()[name = string("op_110_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_110_end_mask_0 = const()[name = string("op_110_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_110_cast_fp16 = slice_by_index(begin = var_110_begin_0, end = var_110_end_0, end_mask = var_110_end_mask_0, x = q_3_cast_fp16)[name = string("op_110_cast_fp16")]; + tensor v_1_cast_fp16 = mul(x = var_87_cast_fp16, y = blocks_0_attn_v_proj_output_scales_to_fp16)[name = string("v_1_cast_fp16")]; + tensor var_89 = const()[name = string("op_89"), val = tensor([1, 32, 128, 512])]; + tensor q_3_cast_fp16 = reshape(shape = var_89, x = q_1_cast_fp16)[name = string("q_3_cast_fp16")]; + tensor var_91 = const()[name = string("op_91"), val = tensor([1, 32, 128, 512])]; + tensor k_3_cast_fp16 = reshape(shape = var_91, x = k_1_cast_fp16)[name = string("k_3_cast_fp16")]; + tensor var_93 = const()[name = string("op_93"), val = tensor([1, 32, 128, 512])]; + tensor v_3_cast_fp16 = reshape(shape = var_93, x = v_1_cast_fp16)[name = string("v_3_cast_fp16")]; + tensor var_105_begin_0 = const()[name = string("op_105_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_105_end_0 = const()[name = string("op_105_end_0"), val = tensor([1, 32, 64, 512])]; + tensor var_105_end_mask_0 = const()[name = string("op_105_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_105_cast_fp16 = slice_by_index(begin = var_105_begin_0, end = var_105_end_0, end_mask = var_105_end_mask_0, x = q_3_cast_fp16)[name = string("op_105_cast_fp16")]; + tensor var_111_begin_0 = const()[name = string("op_111_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_111_end_0 = const()[name = string("op_111_end_0"), val = tensor([1, 32, 128, 512])]; + tensor var_111_end_mask_0 = const()[name = string("op_111_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_111_cast_fp16 = slice_by_index(begin = var_111_begin_0, end = var_111_end_0, end_mask = var_111_end_mask_0, x = q_3_cast_fp16)[name = string("op_111_cast_fp16")]; fp16 const_6_promoted_to_fp16 = const()[name = string("const_6_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_112_cast_fp16 = mul(x = var_110_cast_fp16, y = const_6_promoted_to_fp16)[name = string("op_112_cast_fp16")]; + tensor var_113_cast_fp16 = mul(x = var_111_cast_fp16, y = const_6_promoted_to_fp16)[name = string("op_113_cast_fp16")]; bool rotated_1_interleave_0 = const()[name = string("rotated_1_interleave_0"), val = bool(false)]; - tensor rotated_1_cast_fp16 = concat(axis = var_28, interleave = rotated_1_interleave_0, values = (var_112_cast_fp16, var_104_cast_fp16))[name = string("rotated_1_cast_fp16")]; - tensor var_115_cast_fp16 = mul(x = q_3_cast_fp16, y = cos)[name = string("op_115_cast_fp16")]; - tensor var_116_cast_fp16 = mul(x = rotated_1_cast_fp16, y = sin)[name = string("op_116_cast_fp16")]; - tensor roped_1_cast_fp16 = add(x = var_115_cast_fp16, y = var_116_cast_fp16)[name = string("roped_1_cast_fp16")]; - tensor var_129_begin_0 = const()[name = string("op_129_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_129_end_0 = const()[name = string("op_129_end_0"), val = tensor([1, 32, 64, 512])]; - tensor var_129_end_mask_0 = const()[name = string("op_129_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_129_cast_fp16 = slice_by_index(begin = var_129_begin_0, end = var_129_end_0, end_mask = var_129_end_mask_0, x = k_3_cast_fp16)[name = string("op_129_cast_fp16")]; - tensor var_135_begin_0 = const()[name = string("op_135_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_135_end_0 = const()[name = string("op_135_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_135_end_mask_0 = const()[name = string("op_135_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_135_cast_fp16 = slice_by_index(begin = var_135_begin_0, end = var_135_end_0, end_mask = var_135_end_mask_0, x = k_3_cast_fp16)[name = string("op_135_cast_fp16")]; + tensor rotated_1_cast_fp16 = concat(axis = var_28, interleave = rotated_1_interleave_0, values = (var_113_cast_fp16, var_105_cast_fp16))[name = string("rotated_1_cast_fp16")]; + tensor var_116_cast_fp16 = mul(x = q_3_cast_fp16, y = cos)[name = string("op_116_cast_fp16")]; + tensor var_117_cast_fp16 = mul(x = rotated_1_cast_fp16, y = sin)[name = string("op_117_cast_fp16")]; + tensor roped_1_cast_fp16 = add(x = var_116_cast_fp16, y = var_117_cast_fp16)[name = string("roped_1_cast_fp16")]; + tensor var_130_begin_0 = const()[name = string("op_130_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_130_end_0 = const()[name = string("op_130_end_0"), val = tensor([1, 32, 64, 512])]; + tensor var_130_end_mask_0 = const()[name = string("op_130_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_130_cast_fp16 = slice_by_index(begin = var_130_begin_0, end = var_130_end_0, end_mask = var_130_end_mask_0, x = k_3_cast_fp16)[name = string("op_130_cast_fp16")]; + tensor var_136_begin_0 = const()[name = string("op_136_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_136_end_0 = const()[name = string("op_136_end_0"), val = tensor([1, 32, 128, 512])]; + tensor var_136_end_mask_0 = const()[name = string("op_136_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_136_cast_fp16 = slice_by_index(begin = var_136_begin_0, end = var_136_end_0, end_mask = var_136_end_mask_0, x = k_3_cast_fp16)[name = string("op_136_cast_fp16")]; fp16 const_8_promoted_to_fp16 = const()[name = string("const_8_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_137_cast_fp16 = mul(x = var_135_cast_fp16, y = const_8_promoted_to_fp16)[name = string("op_137_cast_fp16")]; + tensor var_138_cast_fp16 = mul(x = var_136_cast_fp16, y = const_8_promoted_to_fp16)[name = string("op_138_cast_fp16")]; bool rotated_3_interleave_0 = const()[name = string("rotated_3_interleave_0"), val = bool(false)]; - tensor rotated_3_cast_fp16 = concat(axis = var_28, interleave = rotated_3_interleave_0, values = (var_137_cast_fp16, var_129_cast_fp16))[name = string("rotated_3_cast_fp16")]; - tensor var_140_cast_fp16 = mul(x = k_3_cast_fp16, y = cos)[name = string("op_140_cast_fp16")]; - tensor var_141_cast_fp16 = mul(x = rotated_3_cast_fp16, y = sin)[name = string("op_141_cast_fp16")]; - tensor roped_3_cast_fp16 = add(x = var_140_cast_fp16, y = var_141_cast_fp16)[name = string("roped_3_cast_fp16")]; - bool q_5_interleave_0 = const()[name = string("q_5_interleave_0"), val = bool(false)]; - tensor q_5_cast_fp16 = concat(axis = var_28, interleave = q_5_interleave_0, values = roped_1_cast_fp16)[name = string("q_5_cast_fp16")]; - bool k_5_interleave_0 = const()[name = string("k_5_interleave_0"), val = bool(false)]; - tensor k_5_cast_fp16 = concat(axis = var_28, interleave = k_5_interleave_0, values = roped_3_cast_fp16)[name = string("k_5_cast_fp16")]; - tensor var_156_begin_0 = const()[name = string("op_156_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_156_end_0 = const()[name = string("op_156_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_156_end_mask_0 = const()[name = string("op_156_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_k_cache_0 = slice_by_index(begin = var_156_begin_0, end = var_156_end_0, end_mask = var_156_end_mask_0, x = k_5_cast_fp16)[name = string("op_156_cast_fp16")]; - tensor var_157_begin_0 = const()[name = string("op_157_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_157_end_0 = const()[name = string("op_157_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_157_end_mask_0 = const()[name = string("op_157_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_v_cache_0 = slice_by_index(begin = var_157_begin_0, end = var_157_end_0, end_mask = var_157_end_mask_0, x = v_3_cast_fp16)[name = string("op_157_cast_fp16")]; + tensor rotated_3_cast_fp16 = concat(axis = var_28, interleave = rotated_3_interleave_0, values = (var_138_cast_fp16, var_130_cast_fp16))[name = string("rotated_3_cast_fp16")]; + tensor var_141_cast_fp16 = mul(x = k_3_cast_fp16, y = cos)[name = string("op_141_cast_fp16")]; + tensor var_142_cast_fp16 = mul(x = rotated_3_cast_fp16, y = sin)[name = string("op_142_cast_fp16")]; + tensor roped_3_cast_fp16 = add(x = var_141_cast_fp16, y = var_142_cast_fp16)[name = string("roped_3_cast_fp16")]; + tensor v_5_perm_0 = const()[name = string("v_5_perm_0"), val = tensor([0, 1, -1, -2])]; + tensor var_146_begin_0 = const()[name = string("op_146_begin_0"), val = tensor([0, 0, 0, 1])]; + tensor var_146_end_0 = const()[name = string("op_146_end_0"), val = tensor([1, 32, 128, 509])]; + tensor var_146_end_mask_0 = const()[name = string("op_146_end_mask_0"), val = tensor([true, true, true, false])]; + tensor new_k_cache_0 = slice_by_index(begin = var_146_begin_0, end = var_146_end_0, end_mask = var_146_end_mask_0, x = roped_3_cast_fp16)[name = string("op_146_cast_fp16")]; + tensor var_147_begin_0 = const()[name = string("op_147_begin_0"), val = tensor([0, 0, 1, 0])]; + tensor var_147_end_0 = const()[name = string("op_147_end_0"), val = tensor([1, 32, 509, 128])]; + tensor var_147_end_mask_0 = const()[name = string("op_147_end_mask_0"), val = tensor([true, true, false, true])]; + tensor v_5_cast_fp16 = transpose(perm = v_5_perm_0, x = v_3_cast_fp16)[name = string("transpose_5")]; + tensor new_v_cache_0 = slice_by_index(begin = var_147_begin_0, end = var_147_end_0, end_mask = var_147_end_mask_0, x = v_5_cast_fp16)[name = string("op_147_cast_fp16")]; fp16 var_161_to_fp16 = const()[name = string("op_161_to_fp16"), val = fp16(0x1.6ap-4)]; - tensor var_162_cast_fp16 = mul(x = q_5_cast_fp16, y = var_161_to_fp16)[name = string("op_162_cast_fp16")]; + tensor var_162_cast_fp16 = mul(x = roped_1_cast_fp16, y = var_161_to_fp16)[name = string("op_162_cast_fp16")]; bool attn_weights_1_transpose_x_0 = const()[name = string("attn_weights_1_transpose_x_0"), val = bool(true)]; bool attn_weights_1_transpose_y_0 = const()[name = string("attn_weights_1_transpose_y_0"), val = bool(false)]; - tensor attn_weights_1_cast_fp16 = matmul(transpose_x = attn_weights_1_transpose_x_0, transpose_y = attn_weights_1_transpose_y_0, x = var_162_cast_fp16, y = k_5_cast_fp16)[name = string("attn_weights_1_cast_fp16")]; + tensor attn_weights_1_cast_fp16 = matmul(transpose_x = attn_weights_1_transpose_x_0, transpose_y = attn_weights_1_transpose_y_0, x = var_162_cast_fp16, y = roped_3_cast_fp16)[name = string("attn_weights_1_cast_fp16")]; tensor attn_weights_3_cast_fp16 = add(x = attn_weights_1_cast_fp16, y = mask)[name = string("attn_weights_3_cast_fp16")]; - tensor var_170_cast_fp16 = softmax(axis = var_24, x = attn_weights_3_cast_fp16)[name = string("op_170_cast_fp16")]; - bool attn_1_transpose_x_0 = const()[name = string("attn_1_transpose_x_0"), val = bool(false)]; - bool attn_1_transpose_y_0 = const()[name = string("attn_1_transpose_y_0"), val = bool(true)]; - tensor attn_1_cast_fp16 = matmul(transpose_x = attn_1_transpose_x_0, transpose_y = attn_1_transpose_y_0, x = v_3_cast_fp16, y = var_170_cast_fp16)[name = string("attn_1_cast_fp16")]; + tensor attn_weights_5_cast_fp16 = softmax(axis = var_24, x = attn_weights_3_cast_fp16)[name = string("attn_weights_5_cast_fp16")]; + bool var_171_transpose_x_1 = const()[name = string("op_171_transpose_x_1"), val = bool(false)]; + bool var_171_transpose_y_1 = const()[name = string("op_171_transpose_y_1"), val = bool(true)]; + tensor var_171_cast_fp16 = matmul(transpose_x = var_171_transpose_x_1, transpose_y = var_171_transpose_y_1, x = attn_weights_5_cast_fp16, y = v_3_cast_fp16)[name = string("op_171_cast_fp16")]; + tensor attn_1_perm_0 = const()[name = string("attn_1_perm_0"), val = tensor([0, 1, -1, -2])]; tensor var_174 = const()[name = string("op_174"), val = tensor([1, 4096, 1, -1])]; + tensor attn_1_cast_fp16 = transpose(perm = attn_1_perm_0, x = var_171_cast_fp16)[name = string("transpose_4")]; tensor input_1_cast_fp16 = reshape(shape = var_174, x = attn_1_cast_fp16)[name = string("input_1_cast_fp16")]; tensor var_178 = const()[name = string("op_178"), val = tensor([1, 1])]; tensor var_180 = const()[name = string("op_180"), val = tensor([1, 1])]; @@ -645,86 +657,86 @@ program(1.3) tensor x_normed_15_cast_fp16 = mul(x = x_normed_13_cast_fp16, y = var_291_to_fp16)[name = string("x_normed_15_cast_fp16")]; tensor blocks_1_norm_1_weight_to_fp16 = const()[name = string("blocks_1_norm_1_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303669888)))]; tensor x_19_cast_fp16 = mul(x = x_normed_15_cast_fp16, y = blocks_1_norm_1_weight_to_fp16)[name = string("x_19_cast_fp16")]; - tensor var_306 = const()[name = string("op_306"), val = tensor([1, 1])]; - tensor var_308 = const()[name = string("op_308"), val = tensor([1, 1])]; - string var_310_pad_type_0 = const()[name = string("op_310_pad_type_0"), val = string("custom")]; - tensor var_310_pad_0 = const()[name = string("op_310_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_310_cast_fp16 = conv(dilations = var_308, groups = var_263, pad = var_310_pad_0, pad_type = var_310_pad_type_0, strides = var_306, weight = blocks_1_attn_q_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_310_cast_fp16")]; + tensor var_307 = const()[name = string("op_307"), val = tensor([1, 1])]; + tensor var_309 = const()[name = string("op_309"), val = tensor([1, 1])]; + string var_311_pad_type_0 = const()[name = string("op_311_pad_type_0"), val = string("custom")]; + tensor var_311_pad_0 = const()[name = string("op_311_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_311_cast_fp16 = conv(dilations = var_309, groups = var_263, pad = var_311_pad_0, pad_type = var_311_pad_type_0, strides = var_307, weight = blocks_1_attn_q_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_311_cast_fp16")]; tensor blocks_1_attn_q_proj_output_scales_to_fp16 = const()[name = string("blocks_1_attn_q_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303678144)))]; - tensor q_7_cast_fp16 = mul(x = var_310_cast_fp16, y = blocks_1_attn_q_proj_output_scales_to_fp16)[name = string("q_7_cast_fp16")]; - tensor var_314 = const()[name = string("op_314"), val = tensor([1, 1])]; - tensor var_316 = const()[name = string("op_316"), val = tensor([1, 1])]; - string var_318_pad_type_0 = const()[name = string("op_318_pad_type_0"), val = string("custom")]; - tensor var_318_pad_0 = const()[name = string("op_318_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_318_cast_fp16 = conv(dilations = var_316, groups = var_263, pad = var_318_pad_0, pad_type = var_318_pad_type_0, strides = var_314, weight = blocks_1_attn_k_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_318_cast_fp16")]; + tensor q_7_cast_fp16 = mul(x = var_311_cast_fp16, y = blocks_1_attn_q_proj_output_scales_to_fp16)[name = string("q_7_cast_fp16")]; + tensor var_315 = const()[name = string("op_315"), val = tensor([1, 1])]; + tensor var_317 = const()[name = string("op_317"), val = tensor([1, 1])]; + string var_319_pad_type_0 = const()[name = string("op_319_pad_type_0"), val = string("custom")]; + tensor var_319_pad_0 = const()[name = string("op_319_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_319_cast_fp16 = conv(dilations = var_317, groups = var_263, pad = var_319_pad_0, pad_type = var_319_pad_type_0, strides = var_315, weight = blocks_1_attn_k_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_319_cast_fp16")]; tensor blocks_1_attn_k_proj_output_scales_to_fp16 = const()[name = string("blocks_1_attn_k_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303686400)))]; - tensor k_7_cast_fp16 = mul(x = var_318_cast_fp16, y = blocks_1_attn_k_proj_output_scales_to_fp16)[name = string("k_7_cast_fp16")]; - tensor var_322 = const()[name = string("op_322"), val = tensor([1, 1])]; - tensor var_324 = const()[name = string("op_324"), val = tensor([1, 1])]; - string var_326_pad_type_0 = const()[name = string("op_326_pad_type_0"), val = string("custom")]; - tensor var_326_pad_0 = const()[name = string("op_326_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_326_cast_fp16 = conv(dilations = var_324, groups = var_263, pad = var_326_pad_0, pad_type = var_326_pad_type_0, strides = var_322, weight = blocks_1_attn_v_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_326_cast_fp16")]; + tensor k_7_cast_fp16 = mul(x = var_319_cast_fp16, y = blocks_1_attn_k_proj_output_scales_to_fp16)[name = string("k_7_cast_fp16")]; + tensor var_323 = const()[name = string("op_323"), val = tensor([1, 1])]; + tensor var_325 = const()[name = string("op_325"), val = tensor([1, 1])]; + string var_327_pad_type_0 = const()[name = string("op_327_pad_type_0"), val = string("custom")]; + tensor var_327_pad_0 = const()[name = string("op_327_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_327_cast_fp16 = conv(dilations = var_325, groups = var_263, pad = var_327_pad_0, pad_type = var_327_pad_type_0, strides = var_323, weight = blocks_1_attn_v_proj_weight_palettized_cast_fp16, x = x_19_cast_fp16)[name = string("op_327_cast_fp16")]; tensor blocks_1_attn_v_proj_output_scales_to_fp16 = const()[name = string("blocks_1_attn_v_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303694656)))]; - tensor v_5_cast_fp16 = mul(x = var_326_cast_fp16, y = blocks_1_attn_v_proj_output_scales_to_fp16)[name = string("v_5_cast_fp16")]; - tensor var_328 = const()[name = string("op_328"), val = tensor([1, 32, 128, 512])]; - tensor q_9_cast_fp16 = reshape(shape = var_328, x = q_7_cast_fp16)[name = string("q_9_cast_fp16")]; - tensor var_330 = const()[name = string("op_330"), val = tensor([1, 32, 128, 512])]; - tensor k_9_cast_fp16 = reshape(shape = var_330, x = k_7_cast_fp16)[name = string("k_9_cast_fp16")]; - tensor var_332 = const()[name = string("op_332"), val = tensor([1, 32, 128, 512])]; - tensor v_7_cast_fp16 = reshape(shape = var_332, x = v_5_cast_fp16)[name = string("v_7_cast_fp16")]; - tensor var_344_begin_0 = const()[name = string("op_344_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_344_end_0 = const()[name = string("op_344_end_0"), val = tensor([1, 32, 64, 512])]; - tensor var_344_end_mask_0 = const()[name = string("op_344_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_344_cast_fp16 = slice_by_index(begin = var_344_begin_0, end = var_344_end_0, end_mask = var_344_end_mask_0, x = q_9_cast_fp16)[name = string("op_344_cast_fp16")]; - tensor var_350_begin_0 = const()[name = string("op_350_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_350_end_0 = const()[name = string("op_350_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_350_end_mask_0 = const()[name = string("op_350_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_350_cast_fp16 = slice_by_index(begin = var_350_begin_0, end = var_350_end_0, end_mask = var_350_end_mask_0, x = q_9_cast_fp16)[name = string("op_350_cast_fp16")]; + tensor v_7_cast_fp16 = mul(x = var_327_cast_fp16, y = blocks_1_attn_v_proj_output_scales_to_fp16)[name = string("v_7_cast_fp16")]; + tensor var_329 = const()[name = string("op_329"), val = tensor([1, 32, 128, 512])]; + tensor q_9_cast_fp16 = reshape(shape = var_329, x = q_7_cast_fp16)[name = string("q_9_cast_fp16")]; + tensor var_331 = const()[name = string("op_331"), val = tensor([1, 32, 128, 512])]; + tensor k_9_cast_fp16 = reshape(shape = var_331, x = k_7_cast_fp16)[name = string("k_9_cast_fp16")]; + tensor var_333 = const()[name = string("op_333"), val = tensor([1, 32, 128, 512])]; + tensor v_9_cast_fp16 = reshape(shape = var_333, x = v_7_cast_fp16)[name = string("v_9_cast_fp16")]; + tensor var_345_begin_0 = const()[name = string("op_345_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_345_end_0 = const()[name = string("op_345_end_0"), val = tensor([1, 32, 64, 512])]; + tensor var_345_end_mask_0 = const()[name = string("op_345_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_345_cast_fp16 = slice_by_index(begin = var_345_begin_0, end = var_345_end_0, end_mask = var_345_end_mask_0, x = q_9_cast_fp16)[name = string("op_345_cast_fp16")]; + tensor var_351_begin_0 = const()[name = string("op_351_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_351_end_0 = const()[name = string("op_351_end_0"), val = tensor([1, 32, 128, 512])]; + tensor var_351_end_mask_0 = const()[name = string("op_351_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_351_cast_fp16 = slice_by_index(begin = var_351_begin_0, end = var_351_end_0, end_mask = var_351_end_mask_0, x = q_9_cast_fp16)[name = string("op_351_cast_fp16")]; fp16 const_19_promoted_to_fp16 = const()[name = string("const_19_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_352_cast_fp16 = mul(x = var_350_cast_fp16, y = const_19_promoted_to_fp16)[name = string("op_352_cast_fp16")]; + tensor var_353_cast_fp16 = mul(x = var_351_cast_fp16, y = const_19_promoted_to_fp16)[name = string("op_353_cast_fp16")]; bool rotated_5_interleave_0 = const()[name = string("rotated_5_interleave_0"), val = bool(false)]; - tensor rotated_5_cast_fp16 = concat(axis = var_266, interleave = rotated_5_interleave_0, values = (var_352_cast_fp16, var_344_cast_fp16))[name = string("rotated_5_cast_fp16")]; - tensor var_355_cast_fp16 = mul(x = q_9_cast_fp16, y = cos)[name = string("op_355_cast_fp16")]; - tensor var_356_cast_fp16 = mul(x = rotated_5_cast_fp16, y = sin)[name = string("op_356_cast_fp16")]; - tensor roped_5_cast_fp16 = add(x = var_355_cast_fp16, y = var_356_cast_fp16)[name = string("roped_5_cast_fp16")]; - tensor var_369_begin_0 = const()[name = string("op_369_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_369_end_0 = const()[name = string("op_369_end_0"), val = tensor([1, 32, 64, 512])]; - tensor var_369_end_mask_0 = const()[name = string("op_369_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_369_cast_fp16 = slice_by_index(begin = var_369_begin_0, end = var_369_end_0, end_mask = var_369_end_mask_0, x = k_9_cast_fp16)[name = string("op_369_cast_fp16")]; - tensor var_375_begin_0 = const()[name = string("op_375_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_375_end_0 = const()[name = string("op_375_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_375_end_mask_0 = const()[name = string("op_375_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_375_cast_fp16 = slice_by_index(begin = var_375_begin_0, end = var_375_end_0, end_mask = var_375_end_mask_0, x = k_9_cast_fp16)[name = string("op_375_cast_fp16")]; + tensor rotated_5_cast_fp16 = concat(axis = var_266, interleave = rotated_5_interleave_0, values = (var_353_cast_fp16, var_345_cast_fp16))[name = string("rotated_5_cast_fp16")]; + tensor var_356_cast_fp16 = mul(x = q_9_cast_fp16, y = cos)[name = string("op_356_cast_fp16")]; + tensor var_357_cast_fp16 = mul(x = rotated_5_cast_fp16, y = sin)[name = string("op_357_cast_fp16")]; + tensor roped_5_cast_fp16 = add(x = var_356_cast_fp16, y = var_357_cast_fp16)[name = string("roped_5_cast_fp16")]; + tensor var_370_begin_0 = const()[name = string("op_370_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_370_end_0 = const()[name = string("op_370_end_0"), val = tensor([1, 32, 64, 512])]; + tensor var_370_end_mask_0 = const()[name = string("op_370_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_370_cast_fp16 = slice_by_index(begin = var_370_begin_0, end = var_370_end_0, end_mask = var_370_end_mask_0, x = k_9_cast_fp16)[name = string("op_370_cast_fp16")]; + tensor var_376_begin_0 = const()[name = string("op_376_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_376_end_0 = const()[name = string("op_376_end_0"), val = tensor([1, 32, 128, 512])]; + tensor var_376_end_mask_0 = const()[name = string("op_376_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_376_cast_fp16 = slice_by_index(begin = var_376_begin_0, end = var_376_end_0, end_mask = var_376_end_mask_0, x = k_9_cast_fp16)[name = string("op_376_cast_fp16")]; fp16 const_21_promoted_to_fp16 = const()[name = string("const_21_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_377_cast_fp16 = mul(x = var_375_cast_fp16, y = const_21_promoted_to_fp16)[name = string("op_377_cast_fp16")]; + tensor var_378_cast_fp16 = mul(x = var_376_cast_fp16, y = const_21_promoted_to_fp16)[name = string("op_378_cast_fp16")]; bool rotated_7_interleave_0 = const()[name = string("rotated_7_interleave_0"), val = bool(false)]; - tensor rotated_7_cast_fp16 = concat(axis = var_266, interleave = rotated_7_interleave_0, values = (var_377_cast_fp16, var_369_cast_fp16))[name = string("rotated_7_cast_fp16")]; - tensor var_380_cast_fp16 = mul(x = k_9_cast_fp16, y = cos)[name = string("op_380_cast_fp16")]; - tensor var_381_cast_fp16 = mul(x = rotated_7_cast_fp16, y = sin)[name = string("op_381_cast_fp16")]; - tensor roped_7_cast_fp16 = add(x = var_380_cast_fp16, y = var_381_cast_fp16)[name = string("roped_7_cast_fp16")]; - bool q_11_interleave_0 = const()[name = string("q_11_interleave_0"), val = bool(false)]; - tensor q_11_cast_fp16 = concat(axis = var_266, interleave = q_11_interleave_0, values = roped_5_cast_fp16)[name = string("q_11_cast_fp16")]; - bool k_11_interleave_0 = const()[name = string("k_11_interleave_0"), val = bool(false)]; - tensor k_11_cast_fp16 = concat(axis = var_266, interleave = k_11_interleave_0, values = roped_7_cast_fp16)[name = string("k_11_cast_fp16")]; - tensor var_396_begin_0 = const()[name = string("op_396_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_396_end_0 = const()[name = string("op_396_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_396_end_mask_0 = const()[name = string("op_396_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_k_cache_1 = slice_by_index(begin = var_396_begin_0, end = var_396_end_0, end_mask = var_396_end_mask_0, x = k_11_cast_fp16)[name = string("op_396_cast_fp16")]; - tensor var_397_begin_0 = const()[name = string("op_397_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_397_end_0 = const()[name = string("op_397_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_397_end_mask_0 = const()[name = string("op_397_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_v_cache_1 = slice_by_index(begin = var_397_begin_0, end = var_397_end_0, end_mask = var_397_end_mask_0, x = v_7_cast_fp16)[name = string("op_397_cast_fp16")]; + tensor rotated_7_cast_fp16 = concat(axis = var_266, interleave = rotated_7_interleave_0, values = (var_378_cast_fp16, var_370_cast_fp16))[name = string("rotated_7_cast_fp16")]; + tensor var_381_cast_fp16 = mul(x = k_9_cast_fp16, y = cos)[name = string("op_381_cast_fp16")]; + tensor var_382_cast_fp16 = mul(x = rotated_7_cast_fp16, y = sin)[name = string("op_382_cast_fp16")]; + tensor roped_7_cast_fp16 = add(x = var_381_cast_fp16, y = var_382_cast_fp16)[name = string("roped_7_cast_fp16")]; + tensor v_11_perm_0 = const()[name = string("v_11_perm_0"), val = tensor([0, 1, -1, -2])]; + tensor var_386_begin_0 = const()[name = string("op_386_begin_0"), val = tensor([0, 0, 0, 1])]; + tensor var_386_end_0 = const()[name = string("op_386_end_0"), val = tensor([1, 32, 128, 509])]; + tensor var_386_end_mask_0 = const()[name = string("op_386_end_mask_0"), val = tensor([true, true, true, false])]; + tensor new_k_cache_1 = slice_by_index(begin = var_386_begin_0, end = var_386_end_0, end_mask = var_386_end_mask_0, x = roped_7_cast_fp16)[name = string("op_386_cast_fp16")]; + tensor var_387_begin_0 = const()[name = string("op_387_begin_0"), val = tensor([0, 0, 1, 0])]; + tensor var_387_end_0 = const()[name = string("op_387_end_0"), val = tensor([1, 32, 509, 128])]; + tensor var_387_end_mask_0 = const()[name = string("op_387_end_mask_0"), val = tensor([true, true, false, true])]; + tensor v_11_cast_fp16 = transpose(perm = v_11_perm_0, x = v_9_cast_fp16)[name = string("transpose_3")]; + tensor new_v_cache_1 = slice_by_index(begin = var_387_begin_0, end = var_387_end_0, end_mask = var_387_end_mask_0, x = v_11_cast_fp16)[name = string("op_387_cast_fp16")]; fp16 var_401_to_fp16 = const()[name = string("op_401_to_fp16"), val = fp16(0x1.6ap-4)]; - tensor var_402_cast_fp16 = mul(x = q_11_cast_fp16, y = var_401_to_fp16)[name = string("op_402_cast_fp16")]; - bool attn_weights_5_transpose_x_0 = const()[name = string("attn_weights_5_transpose_x_0"), val = bool(true)]; - bool attn_weights_5_transpose_y_0 = const()[name = string("attn_weights_5_transpose_y_0"), val = bool(false)]; - tensor attn_weights_5_cast_fp16 = matmul(transpose_x = attn_weights_5_transpose_x_0, transpose_y = attn_weights_5_transpose_y_0, x = var_402_cast_fp16, y = k_11_cast_fp16)[name = string("attn_weights_5_cast_fp16")]; - tensor attn_weights_7_cast_fp16 = add(x = attn_weights_5_cast_fp16, y = mask)[name = string("attn_weights_7_cast_fp16")]; - tensor var_410_cast_fp16 = softmax(axis = var_262, x = attn_weights_7_cast_fp16)[name = string("op_410_cast_fp16")]; - bool attn_3_transpose_x_0 = const()[name = string("attn_3_transpose_x_0"), val = bool(false)]; - bool attn_3_transpose_y_0 = const()[name = string("attn_3_transpose_y_0"), val = bool(true)]; - tensor attn_3_cast_fp16 = matmul(transpose_x = attn_3_transpose_x_0, transpose_y = attn_3_transpose_y_0, x = v_7_cast_fp16, y = var_410_cast_fp16)[name = string("attn_3_cast_fp16")]; + tensor var_402_cast_fp16 = mul(x = roped_5_cast_fp16, y = var_401_to_fp16)[name = string("op_402_cast_fp16")]; + bool attn_weights_7_transpose_x_0 = const()[name = string("attn_weights_7_transpose_x_0"), val = bool(true)]; + bool attn_weights_7_transpose_y_0 = const()[name = string("attn_weights_7_transpose_y_0"), val = bool(false)]; + tensor attn_weights_7_cast_fp16 = matmul(transpose_x = attn_weights_7_transpose_x_0, transpose_y = attn_weights_7_transpose_y_0, x = var_402_cast_fp16, y = roped_7_cast_fp16)[name = string("attn_weights_7_cast_fp16")]; + tensor attn_weights_9_cast_fp16 = add(x = attn_weights_7_cast_fp16, y = mask)[name = string("attn_weights_9_cast_fp16")]; + tensor attn_weights_11_cast_fp16 = softmax(axis = var_262, x = attn_weights_9_cast_fp16)[name = string("attn_weights_11_cast_fp16")]; + bool var_411_transpose_x_1 = const()[name = string("op_411_transpose_x_1"), val = bool(false)]; + bool var_411_transpose_y_1 = const()[name = string("op_411_transpose_y_1"), val = bool(true)]; + tensor var_411_cast_fp16 = matmul(transpose_x = var_411_transpose_x_1, transpose_y = var_411_transpose_y_1, x = attn_weights_11_cast_fp16, y = v_9_cast_fp16)[name = string("op_411_cast_fp16")]; + tensor attn_3_perm_0 = const()[name = string("attn_3_perm_0"), val = tensor([0, 1, -1, -2])]; tensor var_414 = const()[name = string("op_414"), val = tensor([1, 4096, 1, -1])]; + tensor attn_3_cast_fp16 = transpose(perm = attn_3_perm_0, x = var_411_cast_fp16)[name = string("transpose_2")]; tensor input_9_cast_fp16 = reshape(shape = var_414, x = attn_3_cast_fp16)[name = string("input_9_cast_fp16")]; tensor var_418 = const()[name = string("op_418"), val = tensor([1, 1])]; tensor var_420 = const()[name = string("op_420"), val = tensor([1, 1])]; @@ -788,86 +800,86 @@ program(1.3) tensor x_normed_27_cast_fp16 = mul(x = x_normed_25_cast_fp16, y = var_531_to_fp16)[name = string("x_normed_27_cast_fp16")]; tensor blocks_2_norm_1_weight_to_fp16 = const()[name = string("blocks_2_norm_1_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303771840)))]; tensor x_33_cast_fp16 = mul(x = x_normed_27_cast_fp16, y = blocks_2_norm_1_weight_to_fp16)[name = string("x_33_cast_fp16")]; - tensor var_546 = const()[name = string("op_546"), val = tensor([1, 1])]; - tensor var_548 = const()[name = string("op_548"), val = tensor([1, 1])]; - string var_550_pad_type_0 = const()[name = string("op_550_pad_type_0"), val = string("custom")]; - tensor var_550_pad_0 = const()[name = string("op_550_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_550_cast_fp16 = conv(dilations = var_548, groups = var_503, pad = var_550_pad_0, pad_type = var_550_pad_type_0, strides = var_546, weight = blocks_2_attn_q_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_550_cast_fp16")]; + tensor var_547 = const()[name = string("op_547"), val = tensor([1, 1])]; + tensor var_549 = const()[name = string("op_549"), val = tensor([1, 1])]; + string var_551_pad_type_0 = const()[name = string("op_551_pad_type_0"), val = string("custom")]; + tensor var_551_pad_0 = const()[name = string("op_551_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_551_cast_fp16 = conv(dilations = var_549, groups = var_503, pad = var_551_pad_0, pad_type = var_551_pad_type_0, strides = var_547, weight = blocks_2_attn_q_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_551_cast_fp16")]; tensor blocks_2_attn_q_proj_output_scales_to_fp16 = const()[name = string("blocks_2_attn_q_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303780096)))]; - tensor q_13_cast_fp16 = mul(x = var_550_cast_fp16, y = blocks_2_attn_q_proj_output_scales_to_fp16)[name = string("q_13_cast_fp16")]; - tensor var_554 = const()[name = string("op_554"), val = tensor([1, 1])]; - tensor var_556 = const()[name = string("op_556"), val = tensor([1, 1])]; - string var_558_pad_type_0 = const()[name = string("op_558_pad_type_0"), val = string("custom")]; - tensor var_558_pad_0 = const()[name = string("op_558_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_558_cast_fp16 = conv(dilations = var_556, groups = var_503, pad = var_558_pad_0, pad_type = var_558_pad_type_0, strides = var_554, weight = blocks_2_attn_k_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_558_cast_fp16")]; + tensor q_13_cast_fp16 = mul(x = var_551_cast_fp16, y = blocks_2_attn_q_proj_output_scales_to_fp16)[name = string("q_13_cast_fp16")]; + tensor var_555 = const()[name = string("op_555"), val = tensor([1, 1])]; + tensor var_557 = const()[name = string("op_557"), val = tensor([1, 1])]; + string var_559_pad_type_0 = const()[name = string("op_559_pad_type_0"), val = string("custom")]; + tensor var_559_pad_0 = const()[name = string("op_559_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_559_cast_fp16 = conv(dilations = var_557, groups = var_503, pad = var_559_pad_0, pad_type = var_559_pad_type_0, strides = var_555, weight = blocks_2_attn_k_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_559_cast_fp16")]; tensor blocks_2_attn_k_proj_output_scales_to_fp16 = const()[name = string("blocks_2_attn_k_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303788352)))]; - tensor k_13_cast_fp16 = mul(x = var_558_cast_fp16, y = blocks_2_attn_k_proj_output_scales_to_fp16)[name = string("k_13_cast_fp16")]; - tensor var_562 = const()[name = string("op_562"), val = tensor([1, 1])]; - tensor var_564 = const()[name = string("op_564"), val = tensor([1, 1])]; - string var_566_pad_type_0 = const()[name = string("op_566_pad_type_0"), val = string("custom")]; - tensor var_566_pad_0 = const()[name = string("op_566_pad_0"), val = tensor([0, 0, 0, 0])]; - tensor var_566_cast_fp16 = conv(dilations = var_564, groups = var_503, pad = var_566_pad_0, pad_type = var_566_pad_type_0, strides = var_562, weight = blocks_2_attn_v_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_566_cast_fp16")]; + tensor k_13_cast_fp16 = mul(x = var_559_cast_fp16, y = blocks_2_attn_k_proj_output_scales_to_fp16)[name = string("k_13_cast_fp16")]; + tensor var_563 = const()[name = string("op_563"), val = tensor([1, 1])]; + tensor var_565 = const()[name = string("op_565"), val = tensor([1, 1])]; + string var_567_pad_type_0 = const()[name = string("op_567_pad_type_0"), val = string("custom")]; + tensor var_567_pad_0 = const()[name = string("op_567_pad_0"), val = tensor([0, 0, 0, 0])]; + tensor var_567_cast_fp16 = conv(dilations = var_565, groups = var_503, pad = var_567_pad_0, pad_type = var_567_pad_type_0, strides = var_563, weight = blocks_2_attn_v_proj_weight_palettized_cast_fp16, x = x_33_cast_fp16)[name = string("op_567_cast_fp16")]; tensor blocks_2_attn_v_proj_output_scales_to_fp16 = const()[name = string("blocks_2_attn_v_proj_output_scales_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(303796608)))]; - tensor v_9_cast_fp16 = mul(x = var_566_cast_fp16, y = blocks_2_attn_v_proj_output_scales_to_fp16)[name = string("v_9_cast_fp16")]; - tensor var_568 = const()[name = string("op_568"), val = tensor([1, 32, 128, 512])]; - tensor q_15_cast_fp16 = reshape(shape = var_568, x = q_13_cast_fp16)[name = string("q_15_cast_fp16")]; - tensor var_570 = const()[name = string("op_570"), val = tensor([1, 32, 128, 512])]; - tensor k_15_cast_fp16 = reshape(shape = var_570, x = k_13_cast_fp16)[name = string("k_15_cast_fp16")]; - tensor var_572 = const()[name = string("op_572"), val = tensor([1, 32, 128, 512])]; - tensor v_cast_fp16 = reshape(shape = var_572, x = v_9_cast_fp16)[name = string("v_cast_fp16")]; - tensor var_584_begin_0 = const()[name = string("op_584_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_584_end_0 = const()[name = string("op_584_end_0"), val = tensor([1, 32, 64, 512])]; - tensor var_584_end_mask_0 = const()[name = string("op_584_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_584_cast_fp16 = slice_by_index(begin = var_584_begin_0, end = var_584_end_0, end_mask = var_584_end_mask_0, x = q_15_cast_fp16)[name = string("op_584_cast_fp16")]; - tensor var_590_begin_0 = const()[name = string("op_590_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_590_end_0 = const()[name = string("op_590_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_590_end_mask_0 = const()[name = string("op_590_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_590_cast_fp16 = slice_by_index(begin = var_590_begin_0, end = var_590_end_0, end_mask = var_590_end_mask_0, x = q_15_cast_fp16)[name = string("op_590_cast_fp16")]; + tensor v_13_cast_fp16 = mul(x = var_567_cast_fp16, y = blocks_2_attn_v_proj_output_scales_to_fp16)[name = string("v_13_cast_fp16")]; + tensor var_569 = const()[name = string("op_569"), val = tensor([1, 32, 128, 512])]; + tensor q_15_cast_fp16 = reshape(shape = var_569, x = q_13_cast_fp16)[name = string("q_15_cast_fp16")]; + tensor var_571 = const()[name = string("op_571"), val = tensor([1, 32, 128, 512])]; + tensor k_15_cast_fp16 = reshape(shape = var_571, x = k_13_cast_fp16)[name = string("k_15_cast_fp16")]; + tensor var_573 = const()[name = string("op_573"), val = tensor([1, 32, 128, 512])]; + tensor v_15_cast_fp16 = reshape(shape = var_573, x = v_13_cast_fp16)[name = string("v_15_cast_fp16")]; + tensor var_585_begin_0 = const()[name = string("op_585_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_585_end_0 = const()[name = string("op_585_end_0"), val = tensor([1, 32, 64, 512])]; + tensor var_585_end_mask_0 = const()[name = string("op_585_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_585_cast_fp16 = slice_by_index(begin = var_585_begin_0, end = var_585_end_0, end_mask = var_585_end_mask_0, x = q_15_cast_fp16)[name = string("op_585_cast_fp16")]; + tensor var_591_begin_0 = const()[name = string("op_591_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_591_end_0 = const()[name = string("op_591_end_0"), val = tensor([1, 32, 128, 512])]; + tensor var_591_end_mask_0 = const()[name = string("op_591_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_591_cast_fp16 = slice_by_index(begin = var_591_begin_0, end = var_591_end_0, end_mask = var_591_end_mask_0, x = q_15_cast_fp16)[name = string("op_591_cast_fp16")]; fp16 const_32_promoted_to_fp16 = const()[name = string("const_32_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_592_cast_fp16 = mul(x = var_590_cast_fp16, y = const_32_promoted_to_fp16)[name = string("op_592_cast_fp16")]; + tensor var_593_cast_fp16 = mul(x = var_591_cast_fp16, y = const_32_promoted_to_fp16)[name = string("op_593_cast_fp16")]; bool rotated_9_interleave_0 = const()[name = string("rotated_9_interleave_0"), val = bool(false)]; - tensor rotated_9_cast_fp16 = concat(axis = var_506, interleave = rotated_9_interleave_0, values = (var_592_cast_fp16, var_584_cast_fp16))[name = string("rotated_9_cast_fp16")]; - tensor var_595_cast_fp16 = mul(x = q_15_cast_fp16, y = cos)[name = string("op_595_cast_fp16")]; - tensor var_596_cast_fp16 = mul(x = rotated_9_cast_fp16, y = sin)[name = string("op_596_cast_fp16")]; - tensor roped_9_cast_fp16 = add(x = var_595_cast_fp16, y = var_596_cast_fp16)[name = string("roped_9_cast_fp16")]; - tensor var_609_begin_0 = const()[name = string("op_609_begin_0"), val = tensor([0, 0, 0, 0])]; - tensor var_609_end_0 = const()[name = string("op_609_end_0"), val = tensor([1, 32, 64, 512])]; - tensor var_609_end_mask_0 = const()[name = string("op_609_end_mask_0"), val = tensor([true, true, false, true])]; - tensor var_609_cast_fp16 = slice_by_index(begin = var_609_begin_0, end = var_609_end_0, end_mask = var_609_end_mask_0, x = k_15_cast_fp16)[name = string("op_609_cast_fp16")]; - tensor var_615_begin_0 = const()[name = string("op_615_begin_0"), val = tensor([0, 0, 64, 0])]; - tensor var_615_end_0 = const()[name = string("op_615_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_615_end_mask_0 = const()[name = string("op_615_end_mask_0"), val = tensor([true, true, true, true])]; - tensor var_615_cast_fp16 = slice_by_index(begin = var_615_begin_0, end = var_615_end_0, end_mask = var_615_end_mask_0, x = k_15_cast_fp16)[name = string("op_615_cast_fp16")]; + tensor rotated_9_cast_fp16 = concat(axis = var_506, interleave = rotated_9_interleave_0, values = (var_593_cast_fp16, var_585_cast_fp16))[name = string("rotated_9_cast_fp16")]; + tensor var_596_cast_fp16 = mul(x = q_15_cast_fp16, y = cos)[name = string("op_596_cast_fp16")]; + tensor var_597_cast_fp16 = mul(x = rotated_9_cast_fp16, y = sin)[name = string("op_597_cast_fp16")]; + tensor roped_9_cast_fp16 = add(x = var_596_cast_fp16, y = var_597_cast_fp16)[name = string("roped_9_cast_fp16")]; + tensor var_610_begin_0 = const()[name = string("op_610_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor var_610_end_0 = const()[name = string("op_610_end_0"), val = tensor([1, 32, 64, 512])]; + tensor var_610_end_mask_0 = const()[name = string("op_610_end_mask_0"), val = tensor([true, true, false, true])]; + tensor var_610_cast_fp16 = slice_by_index(begin = var_610_begin_0, end = var_610_end_0, end_mask = var_610_end_mask_0, x = k_15_cast_fp16)[name = string("op_610_cast_fp16")]; + tensor var_616_begin_0 = const()[name = string("op_616_begin_0"), val = tensor([0, 0, 64, 0])]; + tensor var_616_end_0 = const()[name = string("op_616_end_0"), val = tensor([1, 32, 128, 512])]; + tensor var_616_end_mask_0 = const()[name = string("op_616_end_mask_0"), val = tensor([true, true, true, true])]; + tensor var_616_cast_fp16 = slice_by_index(begin = var_616_begin_0, end = var_616_end_0, end_mask = var_616_end_mask_0, x = k_15_cast_fp16)[name = string("op_616_cast_fp16")]; fp16 const_34_promoted_to_fp16 = const()[name = string("const_34_promoted_to_fp16"), val = fp16(-0x1p+0)]; - tensor var_617_cast_fp16 = mul(x = var_615_cast_fp16, y = const_34_promoted_to_fp16)[name = string("op_617_cast_fp16")]; + tensor var_618_cast_fp16 = mul(x = var_616_cast_fp16, y = const_34_promoted_to_fp16)[name = string("op_618_cast_fp16")]; bool rotated_interleave_0 = const()[name = string("rotated_interleave_0"), val = bool(false)]; - tensor rotated_cast_fp16 = concat(axis = var_506, interleave = rotated_interleave_0, values = (var_617_cast_fp16, var_609_cast_fp16))[name = string("rotated_cast_fp16")]; - tensor var_620_cast_fp16 = mul(x = k_15_cast_fp16, y = cos)[name = string("op_620_cast_fp16")]; - tensor var_621_cast_fp16 = mul(x = rotated_cast_fp16, y = sin)[name = string("op_621_cast_fp16")]; - tensor roped_cast_fp16 = add(x = var_620_cast_fp16, y = var_621_cast_fp16)[name = string("roped_cast_fp16")]; - bool q_interleave_0 = const()[name = string("q_interleave_0"), val = bool(false)]; - tensor q_cast_fp16 = concat(axis = var_506, interleave = q_interleave_0, values = roped_9_cast_fp16)[name = string("q_cast_fp16")]; - bool k_interleave_0 = const()[name = string("k_interleave_0"), val = bool(false)]; - tensor k_cast_fp16 = concat(axis = var_506, interleave = k_interleave_0, values = roped_cast_fp16)[name = string("k_cast_fp16")]; - tensor var_636_begin_0 = const()[name = string("op_636_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_636_end_0 = const()[name = string("op_636_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_636_end_mask_0 = const()[name = string("op_636_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_k_cache_2 = slice_by_index(begin = var_636_begin_0, end = var_636_end_0, end_mask = var_636_end_mask_0, x = k_cast_fp16)[name = string("op_636_cast_fp16")]; - tensor var_637_begin_0 = const()[name = string("op_637_begin_0"), val = tensor([0, 0, 0, 1])]; - tensor var_637_end_0 = const()[name = string("op_637_end_0"), val = tensor([1, 32, 128, 512])]; - tensor var_637_end_mask_0 = const()[name = string("op_637_end_mask_0"), val = tensor([true, true, true, true])]; - tensor new_v_cache_2 = slice_by_index(begin = var_637_begin_0, end = var_637_end_0, end_mask = var_637_end_mask_0, x = v_cast_fp16)[name = string("op_637_cast_fp16")]; + tensor rotated_cast_fp16 = concat(axis = var_506, interleave = rotated_interleave_0, values = (var_618_cast_fp16, var_610_cast_fp16))[name = string("rotated_cast_fp16")]; + tensor var_621_cast_fp16 = mul(x = k_15_cast_fp16, y = cos)[name = string("op_621_cast_fp16")]; + tensor var_622_cast_fp16 = mul(x = rotated_cast_fp16, y = sin)[name = string("op_622_cast_fp16")]; + tensor roped_cast_fp16 = add(x = var_621_cast_fp16, y = var_622_cast_fp16)[name = string("roped_cast_fp16")]; + tensor v_perm_0 = const()[name = string("v_perm_0"), val = tensor([0, 1, -1, -2])]; + tensor var_626_begin_0 = const()[name = string("op_626_begin_0"), val = tensor([0, 0, 0, 1])]; + tensor var_626_end_0 = const()[name = string("op_626_end_0"), val = tensor([1, 32, 128, 509])]; + tensor var_626_end_mask_0 = const()[name = string("op_626_end_mask_0"), val = tensor([true, true, true, false])]; + tensor new_k_cache_2 = slice_by_index(begin = var_626_begin_0, end = var_626_end_0, end_mask = var_626_end_mask_0, x = roped_cast_fp16)[name = string("op_626_cast_fp16")]; + tensor var_627_begin_0 = const()[name = string("op_627_begin_0"), val = tensor([0, 0, 1, 0])]; + tensor var_627_end_0 = const()[name = string("op_627_end_0"), val = tensor([1, 32, 509, 128])]; + tensor var_627_end_mask_0 = const()[name = string("op_627_end_mask_0"), val = tensor([true, true, false, true])]; + tensor v_cast_fp16 = transpose(perm = v_perm_0, x = v_15_cast_fp16)[name = string("transpose_1")]; + tensor new_v_cache_2 = slice_by_index(begin = var_627_begin_0, end = var_627_end_0, end_mask = var_627_end_mask_0, x = v_cast_fp16)[name = string("op_627_cast_fp16")]; fp16 var_641_to_fp16 = const()[name = string("op_641_to_fp16"), val = fp16(0x1.6ap-4)]; - tensor var_642_cast_fp16 = mul(x = q_cast_fp16, y = var_641_to_fp16)[name = string("op_642_cast_fp16")]; - bool attn_weights_9_transpose_x_0 = const()[name = string("attn_weights_9_transpose_x_0"), val = bool(true)]; - bool attn_weights_9_transpose_y_0 = const()[name = string("attn_weights_9_transpose_y_0"), val = bool(false)]; - tensor attn_weights_9_cast_fp16 = matmul(transpose_x = attn_weights_9_transpose_x_0, transpose_y = attn_weights_9_transpose_y_0, x = var_642_cast_fp16, y = k_cast_fp16)[name = string("attn_weights_9_cast_fp16")]; - tensor attn_weights_cast_fp16 = add(x = attn_weights_9_cast_fp16, y = mask)[name = string("attn_weights_cast_fp16")]; - tensor var_650_cast_fp16 = softmax(axis = var_502, x = attn_weights_cast_fp16)[name = string("op_650_cast_fp16")]; - bool attn_5_transpose_x_0 = const()[name = string("attn_5_transpose_x_0"), val = bool(false)]; - bool attn_5_transpose_y_0 = const()[name = string("attn_5_transpose_y_0"), val = bool(true)]; - tensor attn_5_cast_fp16 = matmul(transpose_x = attn_5_transpose_x_0, transpose_y = attn_5_transpose_y_0, x = v_cast_fp16, y = var_650_cast_fp16)[name = string("attn_5_cast_fp16")]; + tensor var_642_cast_fp16 = mul(x = roped_9_cast_fp16, y = var_641_to_fp16)[name = string("op_642_cast_fp16")]; + bool attn_weights_13_transpose_x_0 = const()[name = string("attn_weights_13_transpose_x_0"), val = bool(true)]; + bool attn_weights_13_transpose_y_0 = const()[name = string("attn_weights_13_transpose_y_0"), val = bool(false)]; + tensor attn_weights_13_cast_fp16 = matmul(transpose_x = attn_weights_13_transpose_x_0, transpose_y = attn_weights_13_transpose_y_0, x = var_642_cast_fp16, y = roped_cast_fp16)[name = string("attn_weights_13_cast_fp16")]; + tensor attn_weights_15_cast_fp16 = add(x = attn_weights_13_cast_fp16, y = mask)[name = string("attn_weights_15_cast_fp16")]; + tensor attn_weights_cast_fp16 = softmax(axis = var_502, x = attn_weights_15_cast_fp16)[name = string("attn_weights_cast_fp16")]; + bool var_651_transpose_x_1 = const()[name = string("op_651_transpose_x_1"), val = bool(false)]; + bool var_651_transpose_y_1 = const()[name = string("op_651_transpose_y_1"), val = bool(true)]; + tensor var_651_cast_fp16 = matmul(transpose_x = var_651_transpose_x_1, transpose_y = var_651_transpose_y_1, x = attn_weights_cast_fp16, y = v_15_cast_fp16)[name = string("op_651_cast_fp16")]; + tensor attn_5_perm_0 = const()[name = string("attn_5_perm_0"), val = tensor([0, 1, -1, -2])]; tensor var_654 = const()[name = string("op_654"), val = tensor([1, 4096, 1, -1])]; + tensor attn_5_cast_fp16 = transpose(perm = attn_5_perm_0, x = var_651_cast_fp16)[name = string("transpose_0")]; tensor input_17_cast_fp16 = reshape(shape = var_654, x = attn_5_cast_fp16)[name = string("input_17_cast_fp16")]; tensor var_658 = const()[name = string("op_658"), val = tensor([1, 1])]; tensor var_660 = const()[name = string("op_660"), val = tensor([1, 1])];