# 银行XURL安全关系关联 (PySpark)

## Prompt
我需要你生成一段 PySpark 代码，从平台 URL 安全关系中提取银行X相关的贷中和意愿用户样本。

**业务背景与目标**：安全部门维护了一份平台 guid 与 URL 的关系表，记录了用户访问过的 URL 及其 cgi 信息。风控业务需要从这份关系表中筛选出访问过银行X（bankx.com.cn）特定页面的用户，按贷款阶段分为"贷中用户"和"意愿用户"两类，输出到贷款用户样本表中，供下游风控模型使用。

**输入表**：
- `internal_platform_db.t_dwd_urlsafe_rela_guid_url_di_pyspark_101`

**数据范围与过滤条件**：
- 运行日期 ds 固定为 20260513（BIGINT 类型）
- 域名过滤：domain = 'bankx.com.cn'
- 站点过滤：site = 'm1.bankx.com.cn'

**计算规则**：
1. **贷中用户识别**：cgi 包含 `http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/LOANS/WXMAIN/LOAN-REPAY-PLAN`，loan_stage 标记为 'in-loan'
2. **意愿用户识别**：cgi 包含以下任一 URL 模式（18 个 will_cgis），loan_stage 标记为 'will'：
   - LOANS/MAIN/XWMINI, LOANS/AUTHORIZATION, LOANS/DATA-SELF-ATTESTATION-CHECK, LOANS/DATA-SELF-ATTESTATION-EDIT, LOANS/MAIN/XWMINI-RESULT
   - FEIMA 系列路径
3. 两类用户分别选取字段：uid（来自 guid）, loan_type='unknown', city='unknown', industry='unknown', entity_name='unknown', legal_person='unknown', loan_stage, details（来自 url）
4. 各用户组内按 uid 去重
5. Union 合并后，整体按 uid 去重
6. 写入时同时指定分区值：ds=20260513, data_source='social', id_type='guid'

**输出要求**：
- 目标表：`internal_platform_db.t_rta_loan_url_samples_pyspark_101`
- 分区字段：ds BIGINT, data_source STRING, id_type STRING
- 写入策略：先建表（ORC 存储，8 个非分区列 + 3 个分区列），再 INSERT OVERWRITE TABLE PARTITION(ds=..., data_source=..., id_type=...) SELECT * FROM tmp_view

## Reference Solution
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
pyspark_013: URL safety relation join (银行X)
- 2 input tables: t_dwd_urlsafe_rela_guid_url_di, t_rta_loan_url_samples
- Hardcode: ds='20260513', target_domain='bankx.com.cn', data_source='social'
- Define in_loan_cgi and will_cgis URL patterns for BankX bank
- Build in-loan users: match on in_loan_cgi, select uid + unknown placeholders
- Build will users: match on will_cgis, select uid + unknown placeholders
- Union, dropDuplicates, insert overwrite into output table partitioned by (ds, data_source, id_type)
"""

from pyspark.sql import SparkSession
from pyspark.sql import functions as F

spark = SparkSession.builder.appName("pyspark_013_bankx").enableHiveSupport().getOrCreate()

ds = '20260513'
target_domain = 'bankx.com.cn'
data_source = 'social'
dst_tb = "internal_platform_db.t_rta_loan_url_samples_pyspark_101"
target_dfs = []

df = spark.sql(
    f"select guid as uid, url, site, cgi from internal_platform_db.t_dwd_urlsafe_rela_guid_url_di_pyspark_101 where ds={ds} and domain='{target_domain}'"
).cache()

will_cgis = [
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/LOANS/MAIN/XWMINI",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/4/786_3ZC6V00SCTS/KVP64MGHE6",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-NG/LOAN/OPEN-MINI-PROGRAM",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/4/786_3ZC6V00SCTS/EH29NY7VX6O",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/LOANS/AUTHORIZATION",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/404/5297_ID8OJSK0CA/GO6WAHCH1Q8",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/4/2030_X896J30LP0M/32Z34ISSCYM",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/404/4995_PQD15LFBWHK/J4AUCBG4T9Q",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/LOANS/DATA-SELF-ATTESTATION-CHECK",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/4/782_GCZRA2IPYHI/M5L9X3HI1YD",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/4/1488_AIX7S10A5D/YXES5G44DDI",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/LOANS/DATA-SELF-ATTESTATION-EDIT",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/4/672_C1HLFOASYSA/3RVSG4RH23C",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/LOANS/MAIN/XWMINI-RESULT",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/4/1488_AIX7S10A5D/2NS3G56ZR4P",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/4/2030_X896J30LP0M/2NS3G56ZR4P",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/404/4995_PQD15LFBWHK/2NS3G56ZR4P",
    "http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/FEIMA/4/1792_ZHAWK9C9K1/LXTCEQ1UIO"
]

in_loan_cgi = ["http://m1.bankx.com.cn/BankX_MBSERVER/NEW/APP/MOBILE-BANK-H5/LOANS/WXMAIN/LOAN-REPAY-PLAN"]

# Build in-loan users
inloan_condition = None
for url in in_loan_cgi:
    if inloan_condition is None:
        inloan_condition = F.col("cgi").contains(url)
    else:
        inloan_condition = inloan_condition | F.col("cgi").contains(url)

target_dfs.append(
    df.filter(F.col("site") == "m1.bankx.com.cn")
    .filter(inloan_condition)
    .withColumn("loan_stage", F.lit('in-loan'))
    .selectExpr(
        "uid",
        "'unknown' as loan_type",
        "'unknown' as city",
        "'unknown' as industry",
        "'unknown' as entity_name",
        "'unknown' as legal_person",
        "loan_stage",
        "url as details",
    ).dropDuplicates(["uid"])
)

# Build will users
will_condition = None
for url in will_cgis:
    if will_condition is None:
        will_condition = F.col("cgi").contains(url)
    else:
        will_condition = will_condition | F.col("cgi").contains(url)

target_dfs.append(
    df.filter(F.col("site") == "m1.bankx.com.cn")
    .filter(will_condition)
    .withColumn("loan_stage", F.lit('will'))
    .selectExpr(
        "uid",
        "'unknown' as loan_type",
        "'unknown' as city",
        "'unknown' as industry",
        "'unknown' as entity_name",
        "'unknown' as legal_person",
        "loan_stage",
        "url as details",
    ).dropDuplicates(["uid"])
)

# Union all
result_df = target_dfs[0]
for df_item in target_dfs[1:]:
    result_df = result_df.union(df_item)

result_df = result_df.cache()
result_df = result_df.dropDuplicates(["uid"])
print(f"len of result_df: {result_df.count()}")

result_df.createOrReplaceTempView("tmp_df")

# Output table DDL
spark.sql('DROP TABLE IF EXISTS internal_platform_db.t_rta_loan_url_samples_pyspark_101')
spark.sql('''
CREATE TABLE internal_platform_db.t_rta_loan_url_samples_pyspark_101 (
    `uid` STRING,
    `loan_type` STRING,
    `city` STRING,
    `industry` STRING,
    `entity_name` STRING,
    `legal_person` STRING,
    `loan_stage` STRING,
    `details` STRING
) PARTITIONED BY (`ds` BIGINT, `data_source` STRING, `id_type` STRING)
STORED AS ORC
''')

sql = f"insert overwrite table {dst_tb} partition(ds={ds}, data_source='{data_source}', id_type='guid') select * from tmp_df"
print(sql)
spark.sql(sql)
print("pyspark_013 done")

spark.stop()