# 网页解析结果Top500详情统计 (PySpark)

## Prompt
我需要你生成一段 PySpark 代码，对网页解析结果进行质量统计，产出 Top500 站点的详情日报表。

**业务背景与目标**：每天都会对大量网页进行解析，解析结果记录在 `ai_engine_classify_parse_result_daily` 表中。安全部门需要按站点维度统计解析质量，包括成功/失败解析数、索引页占比、短内容占比、各错误步骤分布等，并与站点优先级信息关联，生成一张按日期和站点粒度的解析质量详情表。

**输入表**：
- `internal_platform_db.sec_app_hy_top_500_sites_tag_v1_pyspark_100`
- `internal_platform_db.ai_engine_classify_parse_result_daily_pyspark_100`

**数据范围与过滤条件**：
- 运行日期 today_str 固定为 '20260513'
- 数据范围：date_key 从 '20260508' 到 '20260513'（最近 5 天）

**计算规则**：
1. 从 parse_html JSON 字段中提取 content 字段（使用 from_json + schema_of_json）
2. 计算 content_length = length(content)
3. 按 (date_key, host, fld) 分组聚合：
   - 成功解析数 successful_parses、失败解析数 failed_parses、总解析数 total_parses
   - 索引页数 index_page_count（web_type='索引页'）、索引页比例 index_page_count_rate = index_page_count/total_parses
   - 短内容数 short_content_count（content_length < 50）、短内容比例 short_content_count_rate = short_content_count/successful_parses
   - 各错误步骤计数：error_step_1_count、error_step_2_count、error_step_3_count
4. LEFT JOIN 关联 priority_info：匹配条件为 source_fld = priority_info.fld OR source_host = priority_info.host
5. 去重：dropDuplicates(["date_key", "host", "fld", "priority", "successful_parses", "total_parses", "index_page_count"])

**输出要求**：
- 目标表：`internal_platform_db.ai_engine_classify_parse_result_daily_detail_pyspark_100`
- 写入策略：先建表（ORC 存储，16 列），再使用 df.write.mode("overwrite").insertInto() 写入
- 输出列：date_key, host, fld, is_host, priority, successful_parses, failed_parses, total_parses, index_page_count, index_page_count_rate, short_content_count, short_content_count_rate, error_step_1_count, error_step_2_count, error_step_3_count, error_info_list

## Reference Solution
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
pyspark_012: AI search parse results top500 detail
- 3 input tables: priority_info, daily parse results
- Date range: hardcode today_str='20260513', five_days_ago='20260508'
- Parse content JSON from parse_html field using from_json
- Compute summary stats, index_page_ratio, content_length_ratio, error_reasons
- Join with priority_info on fld or host match
- Output via df.write.mode("overwrite").insertInto()
"""

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, count, when, length, sum as _sum, from_json, schema_of_json, lit
from pyspark.sql import functions as F

spark = (
    SparkSession.builder
    .appName('pyspark_012_ai_engine_parse_detail')
    .enableHiveSupport()
    .getOrCreate()
)

today_str = '20260513'
five_days_ago = '20260508'
print(today_str, five_days_ago)

# Read priority info
priority_info = spark.table("internal_platform_db.sec_app_hy_top_500_sites_tag_v1_pyspark_100").select(
    "host", "fld", "priority"
)

# Read data from past 5 days
df = spark.table("internal_platform_db.ai_engine_classify_parse_result_daily_pyspark_100").filter(
    (col("date_key") >= five_days_ago) & (col("date_key") <= today_str)
)

# Parse data
parsed_data = df.select(
    "date_key",
    col("host").alias("source_host"),
    when(col("is_error") == 0, 1).otherwise(0).alias("success"),
    when(col("is_error") == 1, 1).otherwise(0).alias("failure"),
    "web_type",
    from_json(col("parse_html"), schema_of_json('{"content": "string"}')).getField("content").alias("content"),
    "error_step",
    col("fld").alias("source_fld")
)

# Compute content length
parsed_data = parsed_data.withColumn("content_length", length(col("content")))

# Summary stats
summary = parsed_data.groupBy("date_key", "source_host", "source_fld").agg(
    _sum("success").alias("successful_parses"),
    _sum("failure").alias("failed_parses"),
    count("*").alias("total_parses")
)

# Index page ratio
index_page_ratio = parsed_data.groupBy("date_key", "source_host", "source_fld").agg(
    _sum(when(col("web_type") == "索引页", 1).otherwise(0)).alias("index_page_count")
)

# Content length ratio
content_length_ratio = parsed_data.filter(col("success") == 1).groupBy("date_key", "source_host", "source_fld").agg(
    _sum(when(col("content_length") < 50, 1).otherwise(0)).alias("short_content_count")
)

# Error reasons
error_reasons = parsed_data.filter(col("failure") == 1).groupBy("date_key", "source_host", "source_fld").agg(
    _sum(when(col("error_step") == 1, 1).otherwise(0)).alias("error_step_1_count"),
    _sum(when(col("error_step") == 2, 1).otherwise(0)).alias("error_step_2_count"),
    _sum(when(col("error_step") == 3, 1).otherwise(0)).alias("error_step_3_count"),
    _sum(when(col("error_step").isNotNull(), 1).otherwise(0)).alias("total_error_steps")
)

# Merge all aggregations
final_result = summary.join(index_page_ratio, ["date_key", "source_host", "source_fld"], "left") \
    .join(content_length_ratio, ["date_key", "source_host", "source_fld"], "left") \
    .join(error_reasons, ["date_key", "source_host", "source_fld"], "left")

# Calculate ratios
final_result = final_result.withColumn(
    "index_page_count_rate",
    col("index_page_count") / col("total_parses")
).withColumn(
    "short_content_count_rate",
    col("short_content_count") / col("successful_parses")
)

# Add placeholder columns
final_result = final_result.withColumn("error_info_list", lit("default_value"))
final_result = final_result.withColumn("is_host", lit("default_value"))

# Join with priority info
final_result = final_result.join(
    priority_info,
    (final_result.source_fld == priority_info.fld) |
    (final_result.source_host == priority_info.host),
    "left"
)

# Select final columns
final_result = final_result.select(
    "date_key",
    F.col("source_host").alias("host"),
    F.col("source_fld").alias("fld"),
    "is_host",
    "priority",
    "successful_parses",
    "failed_parses",
    "total_parses",
    "index_page_count",
    "index_page_count_rate",
    "short_content_count",
    "short_content_count_rate",
    "error_step_1_count",
    "error_step_2_count",
    "error_step_3_count",
    "error_info_list"
)

final_result = final_result.dropDuplicates(["date_key", "host", "fld", "priority", "successful_parses", "total_parses", "index_page_count"])

# Output table DDL
spark.sql('DROP TABLE IF EXISTS internal_platform_db.ai_engine_classify_parse_result_daily_detail_pyspark_100')
spark.sql('''
CREATE TABLE internal_platform_db.ai_engine_classify_parse_result_daily_detail_pyspark_100 (
    `date_key` STRING,
    `host` STRING,
    `fld` STRING,
    `is_host` STRING,
    `priority` STRING,
    `successful_parses` STRING,
    `failed_parses` STRING,
    `total_parses` STRING,
    `index_page_count` STRING,
    `index_page_count_rate` STRING,
    `short_content_count` STRING,
    `short_content_count_rate` STRING,
    `error_step_1_count` STRING,
    `error_step_2_count` STRING,
    `error_step_3_count` STRING,
    `error_info_list` STRING
)
STORED AS ORC
''')

# Write output
final_result.write.mode("overwrite").insertInto("internal_platform_db.ai_engine_classify_parse_result_daily_detail_pyspark_100")

print("pyspark_012 done")

spark.stop()