# 小程序H5监控 (PySpark)

## Prompt
我需要你生成一段 PySpark 代码，实现小程序打开第三方 H5 页面的大盘监控指标计算（Mini-Program H5 Monitoring）。

**业务背景与目标**：小程序开放平台需要监控小程序内打开第三方 H5 页面的整体情况，包括 PV、UV、域名数、链接数等核心指标，按整体大盘和 Top100 小程序两个维度分别统计，同时按域名归属关系（同主体/关联主体/第三方）进行细分，产出结构化监控指标表。

**输入表**：
- `internal_platform_db.ods_daily_miniapp_h5_expand_raw_pyspark_212`
- `internal_platform_db.ods_daily_miniapp_h5_top100_pv_raw_pyspark_212`

**数据范围与过滤条件**：
- 运行日期 ds 固定为 '20260513'（日分区）
- ods_daily_miniapp_h5_expand_raw 提供全量 H5 访问明细
- ods_daily_miniapp_h5_top100_pv_raw 提供 Top100 PV 小程序的 H5 访问明细
- url 列重命名为 url_wop（无参数 URL）

**处理逻辑**：
1. 从 expand_raw 表计算整体大盘指标：total PV=总行数, total UV=去重 useruin, total domain=去重 domain, total url_wop=去重 url_wop，stat_type='total'
2. 从 top100_raw 表按 appid 分组计算：PV=count, UV=去重 useruin, domain=去重 domain, url_wop=去重 url_wop，stat_type='total_top'
3. 从 expand_raw 表按 related（域名归属关系）分组计算：PV=count, UV=去重 useruin, domain=去重 domain, url_wop=去重 url_wop，stat_type='contactor_{related}'
4. 从 top100_raw 表按 appid+related 分组计算同样指标，stat_type='contactor_{related}_top'
5. 使用 pandas merge 将各维度指标合并为统一格式
6. 添加 ds 列（BIGINT 类型），生成最终结果

**输出要求**：
- 目标表：`internal_platform_db.ods_daily_miniapp_webview_total_pyspark_212`
- 写入策略：INSERT OVERWRITE TABLE
- 输出列：ds（BIGINT，日期）, appid（STRING，小程序appid，大盘级为空字符串）, pv（BIGINT，页面浏览量）, uv（BIGINT，独立访客数）, domain（BIGINT，域名数）, url_wop（BIGINT，去参链接数）, stat_type（STRING，统计类型：total/total_top/contactor_{related}/contactor_{related}_top）

## Reference Solution
#!/usr/bin/env python3
"""pyspark_020 ground truth: mini-program H5 monitoring metrics"""
import pandas as pd
from pyspark.sql import SparkSession
import pyspark.sql.functions as F

spark = SparkSession.builder \
    .appName('dataclaw_eval_gt_pyspark_020') \
    .enableHiveSupport() \
    .config('spark.sql.warehouse.dir', '/tmp/hive_warehouse') \
    .getOrCreate()

ds = '20260513'

# Output table DDL
spark.sql('''
DROP TABLE IF EXISTS internal_platform_db.ods_daily_miniapp_webview_total_pyspark_212
''')
spark.sql('''
CREATE TABLE IF NOT EXISTS internal_platform_db.ods_daily_miniapp_webview_total_pyspark_212 (
    ds BIGINT COMMENT 'date',
    appid STRING COMMENT 'appid',
    pv BIGINT COMMENT 'page views',
    uv BIGINT COMMENT 'unique visitors',
    domain BIGINT COMMENT 'domain count',
    url_wop BIGINT COMMENT 'url without param count',
    stat_type STRING COMMENT 'statistic type'
)
STORED AS ORC
''')

cols = ['appid', 'pv', 'uv', 'domain', 'url_wop', 'stat_type']

# Read input data
web_stats_raw = spark.sql(f'''
    SELECT useruin, appid, url_, domain, url, contactor_domain, related
    FROM internal_platform_db.ods_daily_miniapp_h5_expand_raw_pyspark_212
    WHERE ds = '{ds}'
''')
web_stats = web_stats_raw.withColumnRenamed('url', 'url_wop')
web_stats.persist()
web_stats_cnt = web_stats.count()
print(f"Fetch total web_stats info: {web_stats_cnt}")

web_stats_topuser_id_raw = spark.sql(f'''
    SELECT useruin, appid, url_, domain, url, contactor_domain, related
    FROM internal_platform_db.ods_daily_miniapp_h5_top100_pv_raw_pyspark_212
    WHERE ds = '{ds}'
''')
web_stats_topuin = web_stats_topuser_id_raw.withColumnRenamed('url', 'url_wop')
web_stats_topuin.persist()
web_stats_topuser_id_cnt = web_stats_topuin.count()
print(f"Fetch total web_stats_topuin info: {web_stats_topuser_id_cnt}")

# 1. Overall metrics: total PV/UV/domain/URL counts
cate = 'total'
total_pv = web_stats_cnt
total_uv = web_stats[['useruin']].drop_duplicates().count()
total_domain_cnt = web_stats[['domain']].drop_duplicates().count()
total_url_cnt = web_stats[['url_wop']].drop_duplicates().count()
df_total = pd.DataFrame(data=[('', total_pv, total_uv, total_domain_cnt, total_url_cnt, cate)], columns=cols)

# Per-app metrics for top100
cate = 'total_top'
total_top_pv = web_stats_topuin.groupby('appid').count().withColumnRenamed('count', 'pv').toPandas()
total_top_uv = web_stats_topuin[['appid', 'useruin']].drop_duplicates().groupby('appid').count().withColumnRenamed('count', 'uv').toPandas()
total_top_domain_cnt = web_stats_topuin[['appid', 'domain']].drop_duplicates().groupby('appid').count().withColumnRenamed('count', 'domain').toPandas()
total_top_url_cnt = web_stats_topuin[['appid', 'url_wop']].drop_duplicates().groupby('appid').count().withColumnRenamed('count', 'url_wop').toPandas()
total_top_metrics = total_top_pv.merge(total_top_uv, how='left', on='appid') \
    .merge(total_top_domain_cnt, how='left', on='appid') \
    .merge(total_top_url_cnt, how='left', on='appid')
total_top_metrics['stat_type'] = cate

pdf_total = pd.concat([df_total[cols], total_top_metrics[cols]], ignore_index=True, axis=0)

# 2. Contactor distribution metrics
cate = 'contactor'
contactor_pv_dist = web_stats.groupby('related').count().withColumnRenamed('count', 'pv').toPandas()
contactor_pv_dist['stat_type'] = cate + '_' + contactor_pv_dist.related
contactor_uv_dist = web_stats[['related', 'useruin']].drop_duplicates().groupby('related').count().withColumnRenamed('count', 'uv').toPandas()
contactor_uv_dist['stat_type'] = cate + '_' + contactor_uv_dist.related
contactor_domain_dist = web_stats[['related', 'domain']].drop_duplicates().groupby('related').count().withColumnRenamed('count', 'domain').toPandas()
contactor_domain_dist['stat_type'] = cate + '_' + contactor_domain_dist.related
contactor_url_dist = web_stats[['related', 'url_wop']].drop_duplicates().groupby('related').count().withColumnRenamed('count', 'url_wop').toPandas()
contactor_url_dist['stat_type'] = cate + '_' + contactor_url_dist.related
df_contactor = contactor_pv_dist[['stat_type', 'pv']].merge(contactor_uv_dist[['stat_type', 'uv']]) \
    .merge(contactor_domain_dist[['stat_type', 'domain']]) \
    .merge(contactor_url_dist[['stat_type', 'url_wop']])
df_contactor['appid'] = ''

# Per-app contactor metrics for top100
contactor_top_pv = web_stats_topuin.groupby(['appid', 'related']).count().withColumnRenamed('count', 'pv').toPandas()
contactor_top_uv = web_stats_topuin[['appid', 'related', 'useruin']].drop_duplicates().groupby(['appid', 'related']).count().withColumnRenamed('count', 'uv').toPandas()
contactor_top_domain = web_stats_topuin[['appid', 'related', 'domain']].drop_duplicates().groupby(['appid', 'related']).count().withColumnRenamed('count', 'domain').toPandas()
contactor_top_url = web_stats_topuin[['appid', 'related', 'url_wop']].drop_duplicates().groupby(['appid', 'related']).count().withColumnRenamed('count', 'url_wop').toPandas()
contactor_top_metrics = contactor_top_pv.merge(contactor_top_uv, how='left', on=['appid', 'related']) \
    .merge(contactor_top_domain, how='left', on=['appid', 'related']) \
    .merge(contactor_top_url, how='left', on=['appid', 'related'])
contactor_top_metrics['stat_type'] = cate + '_' + contactor_top_metrics.related + '_top'

pdf_contactor = pd.concat([df_contactor[cols], contactor_top_metrics[cols]], ignore_index=True, axis=0)

# Combine all metrics
pdf_metrics = pd.concat([pdf_total, pdf_contactor], ignore_index=True, axis=0)
sdf_metrics = spark.createDataFrame(pdf_metrics)
res = sdf_metrics.withColumn('ds', F.lit(int(ds))).select('ds', 'appid', 'pv', 'uv', 'domain', 'url_wop', 'stat_type').repartition(1)
res.persist()
res_cnt = res.count()
print(f"Fetch statistic metrics: {res_cnt}")

# Write output
res.write.mode('overwrite').format('orc').saveAsTable(
    'internal_platform_db.ods_daily_miniapp_webview_total_pyspark_212'
)

print('Ground truth computation complete')
spark.stop()