# Count tasks matching the shuffle_split tuning rule. Input table: internal_platform_db (HiveSQL)

## Prompt
Task Objective: Count tasks that match the `shuffle_split` tuning rule, producing task-level tuning rule hit results.

Inputs:
- `internal_platform_db.nextgen_platform_dsl_spark_props_fht0_query_engine_021`
- `internal_platform_db.ods_spark_props_extra_query_engine_021`
- `internal_platform_db.deep_tuning_rule_hit_app_info_query_engine_021`
- `internal_platform_db.nextgen_platform_dsl_us_task_detail_fdt0_query_engine_021`
- `internal_platform_db.nextgen_platform_dsl_gputj_task_detail_fdt0_query_engine_021`
- `internal_platform_db.app_group_product_info_history_query_engine_021`

Processing Rules: Join the Spark configuration table, task detail tables (from both the `us` and `gputj` sources), and the application group product mapping table. Based on `deep_tuning_rule_hit_app_info`, filter records that match the `shuffle_split` tuning rule, and aggregate at the task granularity to produce deduplicated results. The join conditions and aggregation logic must be consistent with the historical logic, without introducing additional filtering or business rules.

Output Requirements: The output field order must match the target table schema, retaining task-level fields related to tuning rule hits, deduplicated at the task granularity.

Write Requirements: Write to table `internal_platform_db.shuffle_split_tuning_rule_hit_task_info_cand_query_engine_021`.

Please write the final HiveSQL to `result.sql` and execute it.

## Reference Solution
insert overwrite table internal_platform_db.shuffle_split_tuning_rule_hit_task_info_query_engine_021 partition (databus_imp_date = '20260507')
with spark_props as
(
    select
    t1.app_id
    ,max(spark_partition_min) spark_partition_min
    ,max(spark_partition_max) spark_partition_max
    ,max(round(ifnull(spark_task_shuffle_size,'')/(1024*1024))) spark_task_shuffle_size
    ,max(spark_shuffle_sort_bypassMergeThreshold) spark_shuffle_sort_bypassMergeThreshold
    ,max(spark_default_parallelism) spark_default_parallelism
    ,max(spark_sql_shuffle_partitions) spark_sql_shuffle_partitions
    ,max(mapreduce_input_fileinputformat_split_minsize) mapreduce_input_fileinputformat_split_minsize
    ,max(mapreduce_input_fileinputformat_split_maxsize) mapreduce_input_fileinputformat_split_maxsize
    ,max(mapreduce_input_fileinputformat_split_minsize_per_node) mapreduce_input_fileinputformat_split_minsize_per_node
    ,max(mapreduce_input_fileinputformat_split_minsize_per_rack) mapreduce_input_fileinputformat_split_minsize_per_rack
    from
    (
        select
        app_id
        ,max(spark_partition_max) spark_partition_max
        ,max(spark_task_shuffle_size) spark_task_shuffle_size
        ,max(spark_default_parallelism) spark_default_parallelism
        ,max(spark_sql_shuffle_partitions) spark_sql_shuffle_partitions
        from internal_platform_db.nextgen_platform_dsl_spark_props_fht0_query_engine_021
        where databus_imp_date >= '2026050700' and databus_imp_date <= '2026050723'
        group by app_id
    ) t1
    left join
    (
        select
        app_id
        ,max(spark_partition_min) spark_partition_min
        ,max(spark_shuffle_sort_bypassMergeThreshold) spark_shuffle_sort_bypassMergeThreshold
        ,max(mapreduce_input_fileinputformat_split_minsize) mapreduce_input_fileinputformat_split_minsize
        ,max(mapreduce_input_fileinputformat_split_maxsize) mapreduce_input_fileinputformat_split_maxsize
        ,max(mapreduce_input_fileinputformat_split_minsize_per_node) mapreduce_input_fileinputformat_split_minsize_per_node
        ,max(mapreduce_input_fileinputformat_split_minsize_per_rack) mapreduce_input_fileinputformat_split_minsize_per_rack
        from internal_platform_db.ods_spark_props_extra_query_engine_021
        where dt = '20260507'
        group by app_id
    ) t2
    on t1.app_id=t2.app_id
    group by t1.app_id
),
app_info as
(
    select
    t1.*
    ,dw_appgroup
    ,case when (project_id is null or project_id = '') then 'us' else 'datawd' end tag
    from
    (
        select *
        from internal_platform_db.deep_tuning_rule_hit_app_info_query_engine_021
        where databus_imp_date = '20260507' and stage_type = 'shuffleInputStage'
    ) t1
    join
    (
        select
        task_id
        ,max(project_id) project_id
        ,max(dw_appgroup) dw_appgroup
        from internal_platform_db.nextgen_platform_dsl_us_task_detail_fdt0_query_engine_021
        where databus_imp_date = '20260507'
        group by task_id
    ) t2
    on t1.usp_task_id=t2.task_id

    union all

    select
    t1.*
    ,dw_appgroup
    ,'gputj' tag
    from
    (
        select *
        from internal_platform_db.deep_tuning_rule_hit_app_info_query_engine_021
        where databus_imp_date = '20260507' and stage_type = 'shuffleInputStage'
    ) t1
    join
    (
        select
        task_id
        ,max(app_group_id) dw_appgroup
        from internal_platform_db.nextgen_platform_dsl_gputj_task_detail_fdt0_query_engine_021
        where databus_imp_date >= '2026050700' and databus_imp_date <= '2026050723'
        group by task_id
    ) t2
    on t1.usp_task_id=t2.task_id
),
task_info as
(
    select
    usp_task_id
    ,t1.app_id app_id
    ,app_time_usage
    ,dw_appgroup
    ,tag
    ,product_name
    ,shuffle_split
    ,max_shuffle_stage_task_num
    ,shuffle_stage_time
    ,shuffle_stage_task_time_50th
    ,shuffle_stage_task_time_90th
    ,spark_partition_min
    ,spark_partition_max
    ,spark_task_shuffle_size
    ,spark_default_parallelism
    ,spark_sql_shuffle_partitions
    ,spark_shuffle_sort_bypassMergeThreshold
    ,spark_version
    ,start_time
    ,end_time
    ,row_number() over (partition by usp_task_id order by app_time_usage desc) as rk
    from
    (
        select
        usp_task_id
        ,a.app_id app_id
        ,dw_appgroup
        ,tag
        ,app_time_usage
        ,spark_version
        ,case when
            (   (spark_version <>'2.1.0' and (spark_default_parallelism is not null and spark_default_parallelism <>'') and cast(task_num as int)>=cast(spark_default_parallelism as int))
                or
                (spark_version<>'2.1.0'and (spark_sql_shuffle_partitions is not null and spark_sql_shuffle_partitions <>'') and cast(task_num as int)>=cast(spark_sql_shuffle_partitions as int))
                or
                (spark_version='2.1.0' and (spark_partition_max is not null and spark_partition_max <>'') and cast(task_num as int)>=cast(spark_partition_max as int))
                or
                (spark_version='2.1.0' and (spark_partition_max ='' or spark_partition_max is null))
                or
                (spark_version<>'2.1.0' and (spark_default_parallelism ='' or spark_default_parallelism is null) and (spark_sql_shuffle_partitions ='' or spark_sql_shuffle_partitions is null))
            ) then '1' else '0' end as shuffle_split
        ,task_num max_shuffle_stage_task_num
        ,stage_time shuffle_stage_time
        ,task_time_50th shuffle_stage_task_time_50th
        ,task_time_90th shuffle_stage_task_time_90th
        ,spark_partition_min
        ,spark_partition_max
        ,spark_task_shuffle_size
        ,spark_default_parallelism
        ,spark_sql_shuffle_partitions
        ,spark_shuffle_sort_bypassMergeThreshold
        ,start_time
        ,end_time
        from app_info a
        left join spark_props b
        on a.app_id = b.app_id
    ) t1
    left join
    (
        select
        application_group
        ,max(product_name) product_name
        from internal_platform_db.app_group_product_info_history_query_engine_021
        where dt = '20260507'
        group by
        application_group
    ) t2
    on t1.dw_appgroup = t2.application_group
    where shuffle_split = '1'
)

select
    usp_task_id
    ,app_id
    ,app_time_usage
    ,tag
    ,dw_appgroup
    ,product_name
    ,shuffle_split
    ,max_shuffle_stage_task_num
    ,shuffle_stage_time
    ,shuffle_stage_task_time_50th
    ,shuffle_stage_task_time_90th
    ,spark_partition_min
    ,spark_partition_max
    ,spark_task_shuffle_size
    ,spark_default_parallelism
    ,spark_sql_shuffle_partitions
    ,spark_shuffle_sort_bypassMergeThreshold
    ,spark_version
    ,concat('https://internal-tools.example.com/optimization/appgroupfigure/config/recommended?taskId=',usp_task_id,'&applicationId=',app_id,'&startTime=',start_time,'&endTime=',end_time,'&applicationGroup=',dw_appgroup) brain_url
from task_info
where rk = 1