# Campaign User Label Computation (PySpark)

## Prompt
I need you to generate a PySpark script that implements campaign user targeting label computation based on brand campaign activities (Campaign User Label Computation).

**Business Background and Objective**: In the Payment Platform W brand management platform, the targeting labels for campaign configurations are stored in JSON format (containing `label_groups` and `exclude_label_groups`). These need to be parsed into a structured label detail table, and transaction frequency labels (`label_type=7`) need to be automatically backfilled, for downstream user filtering and campaign delivery.

**Input Tables**:
- `internal_platform_db.t_dwd_brand_mgmt_activity_all_hour_pyspark_206`
- `internal_platform_db.t_dwd_gift_mgmt_targeted_act_hour_pyspark_206`

**Data Range and Filter Conditions**:
- Run date `ds` is fixed as `'p_2026051300'` (hourly partition)
- From `t_dwd_brand_mgmt_activity_all_hour`, filter campaigns where `factid>10 AND factstatus IN (2,6,7)`
- Union with the `factid` and `factivitylabel` from `t_dwd_gift_mgmt_targeted_act_hour`

**Processing Logic**:
1. Obtain the campaign `factivitylabel` JSON data by unioning the brand management campaign table and the pay-gift table
2. Use `flatMap` to parse the `label_groups.target_labels` and `exclude_label_groups.target_labels` from each row's JSON into independent rows
3. Extract `label_type`, `label_value`, `effective_range_type`, `effective_range_value`, and `is_exclude` fields from `raw_json`
4. Filter out invalid labels (records where `label_type`, `label_value`, or `effective_range_type` is empty or 0), and deduplicate
5. For all brands (`fbrandid`) with valid campaigns, backfill the four values (1, 2, 3, 4) for `label_type=7` (transaction frequency), with `effective_range_type=2` and `is_exclude=0`
6. Use a left_anti join to exclude already existing records, avoiding duplication

**Output Requirements**:
- Target table: `internal_platform_db.brand_mgmt_act_label_pyspark_206`
- Write strategy: `INSERT OVERWRITE TABLE`
- Output columns: `ds` (STRING, date), `label_type` (STRING, label type), `label_value` (STRING, label value), `effective_range_type` (STRING, effective range type), `effective_range_value` (STRING, effective range value), `raw_json` (STRING, original JSON), `is_exclude` (INT, 0=include, 1=exclude)

## Reference Solution
#!/usr/bin/env python3
"""pyspark_019 ground truth: campaign user label computation"""
import json
from pyspark.sql import SparkSession
from pyspark.sql.functions import lit, get_json_object, when, to_json, struct

spark = SparkSession.builder \
    .appName('dataclaw_eval_gt_pyspark_019') \
    .enableHiveSupport() \
    .config('spark.sql.warehouse.dir', '/tmp/hive_warehouse') \
    .getOrCreate()

hour = '2026051300'
HOUR_PARTITION = 'p_' + str(hour)

# Output table DDL
spark.sql('''
DROP TABLE IF EXISTS internal_platform_db.brand_mgmt_act_label_pyspark_206
''')
spark.sql('''
CREATE TABLE IF NOT EXISTS internal_platform_db.brand_mgmt_act_label_pyspark_206 (
    ds STRING,
    label_type STRING,
    label_value STRING,
    effective_range_type STRING,
    effective_range_value STRING,
    raw_json STRING,
    is_exclude INT
)
STORED AS ORC
''')

# Step 1: Read activity data from brand manager + paygift, filter and union
activity_label = (
    spark.sql(f'''
        SELECT factid, factivitylabel
        FROM internal_platform_db.t_dwd_brand_mgmt_activity_all_hour_pyspark_206
        WHERE ds = '{HOUR_PARTITION}'
            AND factid > 10
            AND factstatus IN (2, 6, 7)
    ''')
).union(
    spark.sql(f'''
        SELECT factid, factivitylabel
        FROM internal_platform_db.t_dwd_gift_mgmt_targeted_act_hour_pyspark_206
        WHERE ds = '{HOUR_PARTITION}'
    ''')
)

activity_label.show(truncate=False)


def parse_labels(row):
    result = []
    if row["factivitylabel"] is None:
        return result
    labels = json.loads(row["factivitylabel"])
    if labels is None:
        return result
    if "label_groups" in labels.keys():
        for group in labels["label_groups"]:
            if "target_labels" in group.keys():
                for l in group["target_labels"]:
                    json_str = json.dumps(l)
                    result.append((json_str,))
    if "exclude_label_groups" in labels.keys():
        for group in labels["exclude_label_groups"]:
            if "target_labels" in group.keys():
                for l in group["target_labels"]:
                    json_str = json.dumps(l)
                    result.append((json_str,))
    return result


# Step 2: flatMap to parse JSON labels into individual rows
single_labels = activity_label.rdd.flatMap(lambda row: parse_labels(row))
print(1)
if single_labels.isEmpty():
    single_labels_df = spark.createDataFrame([], "raw_json: string")
else:
    single_labels_df = spark.createDataFrame(single_labels, ["raw_json"])

single_labels_df = (
    single_labels_df.withColumn("ds", lit(hour))
    .withColumn("label_type", get_json_object(single_labels_df.raw_json, "$.label_type"))
    .withColumn("label_value", get_json_object(single_labels_df.raw_json, "$.label_value"))
    .withColumn("effective_range_type", get_json_object(single_labels_df.raw_json, "$.effective_range_type"))
    .withColumn("effective_range_value", get_json_object(single_labels_df.raw_json, "$.effective_range_value"))
    .withColumn("is_exclude", when(get_json_object(single_labels_df.raw_json, "$.is_exclude") == "true", 1).otherwise(0))
)

single_labels_df.show(truncate=False)

# Step 3: Dedup and filter valid labels
filtered_labels = (
    single_labels_df.select(
        "ds", "label_type", "label_value",
        "effective_range_type", "effective_range_value", "raw_json", "is_exclude",
    )
    .filter("nvl(label_type, 0) != 0 AND nvl(label_value, '0') != '0' AND nvl(effective_range_type, 0) != 0")
    .distinct()
)

# Step 4: Supplement label_type=7 (transaction frequency labels)
all_act_brands = (
    spark.sql(f'''
        SELECT DISTINCT fbrandid
        FROM internal_platform_db.t_dwd_brand_mgmt_activity_all_hour_pyspark_206
        WHERE ds = '{HOUR_PARTITION}'
            AND factid > 10
            AND factstatus IN (2, 6, 7)
    ''')
    .withColumnRenamed("fbrandid", "effective_range_value")
)

freq_label_values = spark.createDataFrame(
    [("1",), ("2",), ("3",), ("4",)], ["label_value"]
)

extra_freq_labels = (
    all_act_brands.crossJoin(freq_label_values)
    .withColumn("ds", lit(hour))
    .withColumn("label_type", lit("7"))
    .withColumn("effective_range_type", lit("2"))
    .withColumn("is_exclude", lit(0))
)

extra_freq_labels = extra_freq_labels.withColumn(
    "raw_json",
    to_json(
        struct(
            extra_freq_labels.label_type,
            extra_freq_labels.label_value,
            extra_freq_labels.effective_range_type,
            extra_freq_labels.effective_range_value,
        )
    ),
).select(
    "ds", "label_type", "label_value",
    "effective_range_type", "effective_range_value", "raw_json", "is_exclude",
)

# Use left_anti join to keep only records not already in filtered_labels
new_freq_labels = extra_freq_labels.join(
    filtered_labels,
    on=["label_type", "label_value", "effective_range_type", "effective_range_value", "is_exclude"],
    how="left_anti",
)

result_labels = filtered_labels.unionByName(new_freq_labels)

# Write output
result_labels.write.mode('overwrite').format('orc').saveAsTable(
    'internal_platform_db.brand_mgmt_act_label_pyspark_206'
)

print('Ground truth computation complete')
spark.stop()