tensor-stats-helper 3.2.4
Quick tensor statistics utilities
Requires Python: >=3.8

# tensor-stats-helper
Lightweight utilities for quick tensor statistics during prototyping.

## Installation
```bash
pip install tensor-stats-helper
``` 

<details>
<summary>🧠 Additional examples (click to expand)</summary>

<pre><code class="language-python">
# ╔══════════════ ① IMPORTS & GLOBALS ══════════════╗
# — All libraries are on the official IOAI allow-list —
import numpy as np, pandas as pd, os, random, warnings, joblib
import matplotlib.pyplot as plt, seaborn as sns

from sklearn.model_selection import (train_test_split, StratifiedKFold, KFold,
                                     RandomizedSearchCV)
from sklearn.preprocessing  import (OneHotEncoder, RobustScaler, LabelEncoder,
                                    MultiLabelBinarizer)
from sklearn.compose        import ColumnTransformer
from sklearn.pipeline       import Pipeline
from sklearn.impute         import SimpleImputer, KNNImputer
from sklearn.metrics        import (accuracy_score, f1_score, roc_auc_score,
                                    mean_absolute_error, r2_score,
                                    ConfusionMatrixDisplay)
from sklearn.ensemble       import RandomForestClassifier, RandomForestRegressor
from lightgbm               import LGBMClassifier, LGBMRegressor
from xgboost                import XGBClassifier, XGBRegressor
from catboost               import CatBoostClassifier, CatBoostRegressor

warnings.filterwarnings("ignore")
sns.set_palette("viridis")

SEED = 42
np.random.seed(SEED); random.seed(SEED)
# ╚═════════════════════════════════════════════════╝



# ╔═══════════ ② FLEXIBLE DATA LOADER (+ quick shapes) ═══════════╗
def load_any(path: str) -> pd.DataFrame:
    """
    Read CSV / TXT / Parquet / Excel / JSON with one helper.
    Extend the mapping dict if IOAI provides another format.
    """
    ext = os.path.splitext(path)[-1].lower()
    reader = {".csv": pd.read_csv, ".txt": pd.read_csv,
              ".parquet": pd.read_parquet, ".pq": pd.read_parquet,
              ".xlsx": pd.read_excel, ".xls": pd.read_excel,
              ".json": pd.read_json}.get(ext)
    if reader is None:
        raise ValueError(f"[load_any] Unsupported file type: {ext}")
    return reader(path)

# —— file paths (edit when the contest releases them) ——
TRAIN_PATH, TEST_PATH = "train.csv", "test.csv"

df        = load_any(TRAIN_PATH)
test_df   = load_any(TEST_PATH) if os.path.exists(TEST_PATH) else None

print("Loaded shapes ➜ train:", df.shape,
      "  test:", None if test_df is None else test_df.shape)
# ╚══════════════════════════════════════════════════════════════════╝
# ╔══════════════ ③ QUICK-ACTION EDA ARMORY ══════════════╗
"""
Minimal-cost diagnostics that directly inform cleaning & modelling
------------------------------------------------------------------
call   | when to use                           | what it prints / plots
-------|---------------------------------------|----------------------------------------------
eda_overview(df, target)          immediate health check – run first
eda_missing(df)                   imputation / column-drop decisions
eda_cardinality(df)               decide one-hot vs encoding vs string
eda_numeric_stats(df)             outlier clamp & scaling insight
eda_mutual_info(df, target, k)    numeric feature signal sniff
eda_pairplot(df, cols, target)    eyeball non-linear sep/leak – small sample
eda_corr_heatmap(df, cols)        multi-col relatedness (Spearman)
"""

import pandas as pd, numpy as np, seaborn as sns, matplotlib.pyplot as plt
from sklearn.feature_selection import mutual_info_classif

# — 0. High-level snapshot ——————————————————————————————————————
def eda_overview(df: pd.DataFrame, target: str=None):
    print("Shape:", df.shape)
    print("\nDtype counts:\n", df.dtypes.value_counts())
    if target and target in df:
        print("\nTarget distribution:")
        print(df[target].value_counts(normalize=True, dropna=False))

# — 1. Missingness ————————————————————————————————————————
def eda_missing(df: pd.DataFrame, top=20):
    na = df.isna().mean().mul(100).sort_values(ascending=False)
    print(f"\nTop {top} missing-percentage columns:")
    print(na.head(top))

# — 2. Categorical cardinality ——————————————————————————————
def eda_cardinality(df: pd.DataFrame, tail=20):
    card = df.select_dtypes('object').nunique().sort_values()
    print(f"\nLowest & highest cardinality (tail {tail} shown):")
    display(pd.concat([card.head(tail//2), card.tail(tail//2)]))

# — 3. Numeric stats & outlier quickview ————————————————
def eda_numeric_stats(df: pd.DataFrame, target:str=None):
    num_cols = df.select_dtypes('number').columns.difference([target])
    desc = df[num_cols].describe(percentiles=[.01,.25,.5,.75,.99]).T
    display(desc)

# — 4. Mutual Information ranking (numeric vs. target) ————
def eda_mutual_info(df: pd.DataFrame, target: str, k=15):
    num_cols = df.select_dtypes('number').columns.difference([target])
    if target not in df or not len(num_cols): return
    mi = mutual_info_classif(df[num_cols].fillna(0), df[target])
    s = pd.Series(mi, index=num_cols).sort_values(ascending=False)
    print(f"\nTop {k} MI numeric features:")
    print(s.head(k))

# — 5. Pairplot on a small sample ————————————
def eda_pairplot(df: pd.DataFrame, cols:list, target:str=None, samples=3000):
    sns.pairplot(df.sample(min(samples, len(df))),
                 vars=cols, hue=target if target and
                 df[target].nunique()<=10 else None, diag_kind="hist")
    plt.show()

# — 6. Correlation heatmap (Spearman) ——————————
def eda_corr_heatmap(df: pd.DataFrame, cols:list, title="Spearman ρ"):
    corr = df[cols].corr(method='spearman')
    sns.heatmap(corr, cmap="coolwarm", center=0); plt.title(title); plt.show()
# ╚═══════════════════════════════════════════════════════════╝
eda_overview(df, target="class")      # always first
eda_missing(df)                       # plan imputers / drop cols
eda_cardinality(df)                   # pick encoding strategy
eda_numeric_stats(df)                 # spot weird ranges / scale needs
eda_mutual_info(df, "class")          # see which numerics matter
top_cols = ["feat1","feat2","feat3"]  # from MI or domain intuition
eda_pairplot(df, top_cols, "class")   # quick visual check
eda_corr_heatmap(df, top_cols)        # redundancy vs multicollinearity

# ╔════════════════════════ ④ DATA-CLEAN & FEATURE-ENG ═════════════════════╗
# ── GLOBAL SWITCHES ───────────────────────────────────────────────────────
TARGET              = "class"  # ✏️ set your label column
# Basic prep
RARE_THR            = 20       # cat rows < RARE_THR → "other"
MISSING_THR         = .80      # drop col if >80 % NaN
HIGH_CARD_THR       = 100      # ≥100 uniques → high-card strategy
USE_KNN_IMPUTE      = False    # True → KNN numeric impute; else median
CLAMP_OUTLIER       = True     # clip numeric IQR outliers
ADD_DATETIME_FE     = True     # decompose yyyy-mm-dd hh:mm:ss columns
# Optional power-ups (set to True only if dataset matches pattern)
FE_TFIDF_TEXT       = False    # free-text → TF-IDF sparse block
FE_GROUP_AGGS       = False    # group-key numeric summaries
FE_GEO_DISTANCE     = False    # haversine km between origin/dest
FE_TARGET_STATS_CAT = False    # K-fold target mean encoding
# -------------------------------------------------------------------------

import numpy as np, pandas as pd, math, gc, warnings
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import KFold
warnings.filterwarnings("ignore")

# df & test_df come from Cell ②
df_clean   = df.copy()
test_clean = test_df.copy() if test_df is not None else None

# ───────────────── 1. DROP HI-MISSING COLS & DUPES ───────────────────────
hi_na = df_clean.columns[df_clean.isna().mean() > MISSING_THR]
df_clean.drop(columns=hi_na, inplace=True); df_clean.drop_duplicates(inplace=True)
if test_clean is not None: test_clean.drop(columns=hi_na, inplace=True)

# ───────────────── 2. NORMALISE STRINGS & RARE-SQUASH ────────────────────
def norm_str(s): return s.astype(str).str.strip().str.lower().replace({'nan': np.nan})
for d in (df_clean, test_clean):
    if d is None: continue
    objs = d.select_dtypes('object').columns
    d[objs] = d[objs].apply(norm_str)
    for c in objs:
        rare_vals = d[c].value_counts()[lambda x: x < RARE_THR].index
        d[c] = d[c].mask(d[c].isin(rare_vals), "other")

# ───────────────── 3. DATETIME DECOMPOSE (year / month …) ────────────────
if ADD_DATETIME_FE:
    from pandas.api.types import is_datetime64_any_dtype as is_dt
    date_cols = [c for c in df_clean.columns if is_dt(df_clean[c]) or
                 (df_clean[c].dtype == "object" and
                  pd.to_datetime(df_clean[c], errors='coerce').notna().any())]
    for col in date_cols:
        for d in (df_clean, test_clean):
            if d is None: continue
            ts = pd.to_datetime(d[col], errors='coerce')
            d[f"{col}_year"]    = ts.dt.year
            d[f"{col}_month"]   = ts.dt.month
            d[f"{col}_dow"]     = ts.dt.dayofweek
            d[f"{col}_hour"]    = ts.dt.hour
            d[f"{col}_weekend"] = ts.dt.dayofweek.isin([5,6]).astype(int)

# ───────────────── 4. NUMERIC OUTLIER CLAMP (IQR) ────────────────────────
num_cols = df_clean.select_dtypes('number').columns.difference([TARGET])
if CLAMP_OUTLIER:
    for col in num_cols:
        q1,q3 = df_clean[col].quantile([.25,.75]); iqr = q3-q1
        lo,hi = q1-1.5*iqr, q3+1.5*iqr
        df_clean[col] = df_clean[col].clip(lo, hi)
        if test_clean is not None:
            test_clean[col] = test_clean[col].clip(lo, hi)

# ───────────────── 5. HIGH-CARD CATS → FREQ-ENCODE ───────────────────────
high_cat = [c for c in df_clean.select_dtypes('object')
            if df_clean[c].nunique() >= HIGH_CARD_THR]
for col in high_cat:
    freq = df_clean[col].value_counts()
    df_clean[col] = df_clean[col].map(freq)
    if test_clean is not None:
        test_clean[col] = test_clean[col].map(freq).fillna(0)

# ========================================================================
# --------------------- OPTIONAL POWER-UPS BELOW -------------------------
# ========================================================================

# 5a. TEXT → TF-IDF SPARSE MATRICES --------------------------------------
if FE_TFIDF_TEXT:
    TEXT_COLS = [c for c in df_clean.select_dtypes('object')
                 if df_clean[c].str.len().mean() > 30][:2]   # limit to 2
    if TEXT_COLS:
        from scipy import sparse
        tf_train, tf_test, vocab = [], [], []
        for col in TEXT_COLS:
            vec = TfidfVectorizer(max_features=5000, ngram_range=(1,2),
                                  min_df=2, stop_words='english')
            vec.fit(pd.concat([df_clean[col], test_clean[col]]).fillna(""))
            tf_train.append(vec.transform(df_clean[col].fillna("")))
            tf_test.append(vec.transform(test_clean[col].fillna("")) if test_clean is not None else None)
            vocab += [f"{col}_{t}" for t in vec.get_feature_names_out()]
            df_clean.drop(columns=[col], inplace=True)
            if test_clean is not None: test_clean.drop(columns=[col], inplace=True)
        TFIDF_TRAIN = sparse.hstack(tf_train).tocsr()
        TFIDF_TEST  = sparse.hstack(tf_test).tocsr() if test_clean is not None else None
        print(f"[TF-IDF] Added block shape {TFIDF_TRAIN.shape}")

# 5b. GROUP-BY NUMERIC AGGREGATIONS --------------------------------------
if FE_GROUP_AGGS:
    GROUP_KEY = "user_id"         # ✏️ change to your repeating id
    if GROUP_KEY in df_clean.columns:
        agg_cols = df_clean.select_dtypes('number').columns.difference([TARGET])
        agg_df = df_clean.groupby(GROUP_KEY)[agg_cols].agg(["mean","std","max","min"])
        agg_df.columns = [f"{GROUP_KEY}_{c}_{a}" for c,a in agg_df.columns]
        df_clean = df_clean.join(agg_df, on=GROUP_KEY)
        if test_clean is not None:
            test_clean = test_clean.join(agg_df, on=GROUP_KEY, how="left")
        print(f"[GROUP-AGG] Added {agg_df.shape[1]} features.")

# 5c. GEO DISTANCE (lat/lon or IATA) -------------------------------------
if FE_GEO_DISTANCE:
    # 1) Map airport codes to lat/ lon if needed
    if {"origin","dest"}.issubset(df_clean.columns):
        try:
            from airportsdata import load as load_air
            air = pd.DataFrame(load_air().values())[["iata","lat","lon"]]
            def map_air(d, col):
                return d.merge(air, left_on=col, right_on="iata", how="left") \
                        .rename(columns={"lat":f"{col}_lat","lon":f"{col}_lon"}) \
                        .drop(columns=["iata"])
            df_clean   = map_air(df_clean,   "origin")
            test_clean = map_air(test_clean, "origin") if test_clean is not None else test_clean
            df_clean   = map_air(df_clean,   "dest")
            test_clean = map_air(test_clean, "dest")   if test_clean is not None else test_clean
        except ImportError:
            warnings.warn("airportsdata missing → skipping IATA mapping")

    # 2) Compute haversine km
    if {"origin_lat","origin_lon","dest_lat","dest_lon"}.issubset(df_clean.columns):
        def haversine_np(lat1, lon1, lat2, lon2, R=6371.0):
            lat1,lon1,lat2,lon2 = map(np.radians,[lat1,lon1,lat2,lon2])
            dlat,dlon = lat2-lat1, lon2-lon1
            a = np.sin(dlat/2)**2 + np.cos(lat1)*np.cos(lat2)*np.sin(dlon/2)**2
            return 2*R*np.arctan2(np.sqrt(a), np.sqrt(1-a))
        df_clean["geo_dist_km"] = haversine_np(df_clean["origin_lat"], df_clean["origin_lon"],
                                               df_clean["dest_lat"],   df_clean["dest_lon"])
        if test_clean is not None:
            test_clean["geo_dist_km"] = haversine_np(test_clean["origin_lat"], test_clean["origin_lon"],
                                                     test_clean["dest_lat"],   test_clean["dest_lon"])
        print("[GEO] Added geo_dist_km")

# 5d. LEAK-SAFE TARGET MEAN ENCODING --------------------------------------
if FE_TARGET_STATS_CAT and TARGET in df_clean:
    cat_mid = [c for c in df_clean.select_dtypes('object')
               if 10 <= df_clean[c].nunique() < HIGH_CARD_THR]
    kf = KFold(5, shuffle=True, random_state=SEED)
    for col in cat_mid:
        df_clean[f"{col}_tgtmean"] = 0.0
        if test_clean is not None: test_clean[f"{col}_tgtmean"] = 0.0
        for tr_idx, val_idx in kf.split(df_clean):
            fold_mean = df_clean.iloc[tr_idx].groupby(col)[TARGET].mean()
            df_clean.loc[val_idx, f"{col}_tgtmean"] = df_clean.loc[val_idx, col].map(fold_mean)
        global_mean = df_clean.groupby(col)[TARGET].mean()
        if test_clean is not None:
            test_clean[f"{col}_tgtmean"] = test_clean[col].map(global_mean)
    print("[TGT-MEAN] Encoded:", cat_mid)

# ───────────────── SUMMARISE ──────────────────────────────────────────────
print("Clean data shapes →", df_clean.shape,
      test_clean.shape if test_clean is not None else None)
gc.collect()
# ╚════════════════════════════════════════════════════════════════════════╝
# ╔══════════════════════ ④-AP AIRPORTS ENRICHMENT ═════════════════════════╗
"""
Requires: airportsdata (present in the image)
Docs summary:
  • load() returns dict keyed by ICAO by default; use load('IATA') or load('LID')
  • Each entry has: icao, iata, name, city, subd, country, elevation(ft), lat, lon, tz, lid
Refs: PyPI project page & examples.  (See citations in chat.)
"""

import numpy as np, pandas as pd, warnings
from math import radians, sin, cos, atan2, sqrt

# ── CONFIG ──
ORIGIN_COL   = "origin"      # ✏️ change if your column names differ
DEST_COL     = "dest"        # ✏️
AIR_CODETYPE = "IATA"        # "IATA" | "ICAO" | "LID"
ADD_LOCAL_TIME = False       # set True if you have a UTC datetime column per leg
DEP_UTC_COL = "scheduled_departure_utc"   # ✏️ pandas datetime64[ns, UTC]

# ── LOAD DATABASE ONCE ──
try:
    import airportsdata
except Exception as e:
    warnings.warn("airportsdata not available; skipping airport enrichment")
    airportsdata = None

def _load_air_db(code_type="IATA"):
    """
    Return a dict keyed by the chosen code type with airport attributes.
    """
    if airportsdata is None: 
        return {}
    # ICAO is default when arg omitted
    return airportsdata.load(code_type) if code_type in {"IATA","LID"} else airportsdata.load()

AIR_DB = _load_air_db(AIR_CODETYPE)

def add_airport_attrs(df: pd.DataFrame, code_col: str, code_type: str, prefix: str):
    """
    Map a column of airport codes to attributes:
    Creates columns: {prefix}_lat, {prefix}_lon, {prefix}_tz, {prefix}_country, {prefix}_elev_ft,
                     {prefix}_name, {prefix}_city, {prefix}_subd, {prefix}_icao, {prefix}_iata, {prefix}_lid
    Missing/unknown codes → NaN.
    """
    if not AIR_DB or code_col not in df.columns: 
        return df
    # Build column-wise mapping dicts (fast pd.Series.map)
    def col_map(attr): 
        return {k: v.get(attr, np.nan) for k, v in AIR_DB.items()}
    maps = {a: col_map(a) for a in ["lat","lon","tz","country","elevation",
                                    "name","city","subd","icao","iata","lid"]}
    df[f"{prefix}_lat"]      = df[code_col].map(maps["lat"])
    df[f"{prefix}_lon"]      = df[code_col].map(maps["lon"])
    df[f"{prefix}_tz"]       = df[code_col].map(maps["tz"])
    df[f"{prefix}_country"]  = df[code_col].map(maps["country"])
    df[f"{prefix}_elev_ft"]  = df[code_col].map(maps["elevation"])
    df[f"{prefix}_name"]     = df[code_col].map(maps["name"])
    df[f"{prefix}_city"]     = df[code_col].map(maps["city"])
    df[f"{prefix}_subd"]     = df[code_col].map(maps["subd"])
    df[f"{prefix}_icao"]     = df[code_col].map(maps["icao"])
    df[f"{prefix}_iata"]     = df[code_col].map(maps["iata"])
    df[f"{prefix}_lid"]      = df[code_col].map(maps["lid"])
    return df

def haversine_km(lat1, lon1, lat2, lon2, R=6371.0):
    """Vectorised great-circle distance in km."""
    lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])
    dlat, dlon = lat2 - lat1, lon2 - lon1
    a = np.sin(dlat/2.0)**2 + np.cos(lat1)*np.cos(lat2)*np.sin(dlon/2.0)**2
    return 2*R*np.arctan2(np.sqrt(a), np.sqrt(1.0 - a))

# ── Enrich TRAIN/TEST (if present) ──
for _df in [df_clean, test_clean] if 'test_clean' in globals() else [df_clean]:
    if _df is None: 
        continue
    if ORIGIN_COL in _df:
        add_airport_attrs(_df, ORIGIN_COL, AIR_CODETYPE, prefix=ORIGIN_COL)
    if DEST_COL in _df:
        add_airport_attrs(_df, DEST_COL,   AIR_CODETYPE, prefix=DEST_COL)

    # Distance if we have both endpoints
    need = {f"{ORIGIN_COL}_lat", f"{ORIGIN_COL}_lon", f"{DEST_COL}_lat", f"{DEST_COL}_lon"}
    if need.issubset(set(_df.columns)):
        _df["geo_dist_km"] = haversine_km(_df[f"{ORIGIN_COL}_lat"], _df[f"{ORIGIN_COL}_lon"],
                                          _df[f"{DEST_COL}_lat"],   _df[f"{DEST_COL}_lon"])

    # Same-country flag (cheap & useful)
    oc, dc = f"{ORIGIN_COL}_country", f"{DEST_COL}_country"
    if oc in _df and dc in _df:
        _df["same_country"] = (_df[oc] == _df[dc]).astype("int8")

    # OPTIONAL: local-time features from a UTC timestamp (requires tz columns)
    if ADD_LOCAL_TIME and DEP_UTC_COL in _df and f"{ORIGIN_COL}_tz" in _df:
        try:
            from zoneinfo import ZoneInfo   # stdlib py>=3.9
            ts = pd.to_datetime(_df[DEP_UTC_COL], utc=True, errors="coerce")
            # per-row tz conversion; vectorised apply to avoid slow loops
            def _to_local(row):
                if pd.isna(row[DEP_UTC_COL]) or not isinstance(row[f"{ORIGIN_COL}_tz"], str):
                    return pd.NaT
                try:
                    return ts[row.name].tz_convert(ZoneInfo(row[f"{ORIGIN_COL}_tz"]))
                except Exception:
                    return pd.NaT
            local = _df.apply(_to_local, axis=1)
            _df[f"{ORIGIN_COL}_local_hour"]  = local.dt.hour
            _df[f"{ORIGIN_COL}_local_dow"]   = local.dt.dayofweek
            _df[f"{ORIGIN_COL}_local_month"] = local.dt.month
        except Exception as e:
            warnings.warn(f"Local-time features skipped: {e}")

print("[AIRPORTS] enrichment done:",
      df_clean.shape, None if test_clean is None else test_clean.shape)
# ╚═════════════════════════════════════════════════════════════════════════╝

# ╔══════════════════════════ ⑤ PRE-PROCESSOR ═══════════════════════════╗
"""
What this cell gives you
------------------------
1) Column buckets:
   • numeric_cols        → impute + scale
   • low_card_categoricals (< HIGH_CARD_THR uniques) → one-hot
   • everything already numeric from Cell ④ (freq-enc, tgt-mean, aggs, geo) passes through.

2) A sklearn ColumnTransformer named `preproc` you can:
   • use inside a Pipeline with your model (simple path), or
   • fit/transform to a sparse/dense matrix and optionally hstack a TF-IDF block.

3) Utilities:
   • build_design_matrix(...)  → returns X (base) or X ⊕ TF-IDF CSR, ready for LGBM/XGB/CatBoost
   • get_feature_names(...)    → best-effort list of output feature names (for debugging/importance)

When to use which path
----------------------
• No TF-IDF:   just do Pipeline([("prep", preproc), ("m", model)]).
• With TF-IDF: call build_design_matrix(..., tfidf_block=TFIDF_TRAIN) and fit your model on the returned CSR.
"""

from sklearn.compose      import ColumnTransformer
from sklearn.pipeline     import Pipeline
from sklearn.preprocessing import OneHotEncoder, RobustScaler
from sklearn.impute       import SimpleImputer, KNNImputer
from scipy import sparse

# ── KNOBS (inherited from earlier cells but can be overridden here) ──
# HIGH_CARD_THR: int   → threshold that decides low vs high-card categoricals
# USE_KNN_IMPUTE: bool → numeric imputer type

# 0) Identify column buckets on the *clean* frame ---------------------------
_all_numeric = df_clean.select_dtypes('number').columns
num_cols     = _all_numeric.difference([TARGET])

# low-card categoricals get one-hot; high-card already numeric (freq/tgt-mean) in Cell ④
low_cat = [c for c in df_clean.select_dtypes('object')
           if df_clean[c].nunique() < HIGH_CARD_THR]

print(f"[⑤] numeric={len(num_cols)} | low-cat(one-hot)={len(low_cat)}")

# 1) Define per-bucket pipelines -------------------------------------------
num_pipe = (Pipeline([("impute", KNNImputer())])
            if USE_KNN_IMPUTE else
            Pipeline([("impute", SimpleImputer(strategy="median")),
                      ("scale",  RobustScaler())]))

cat_pipe = Pipeline([("impute", SimpleImputer(strategy="most_frequent")),
                     ("onehot", OneHotEncoder(drop='first', handle_unknown='ignore'))])

# 2) ColumnTransformer: everything else passes through unchanged ------------
preproc = ColumnTransformer([
    ("num", num_pipe, num_cols),
    ("cat", cat_pipe, low_cat),
], remainder='passthrough')  # <- frequency-enc, tgt-mean, aggs, geo_dist stay as-is


# 3) Helper: build design matrix (optionally append TF-IDF CSR) -------------
def build_design_matrix(preproc, X_df, fit: bool = False, tfidf_block=None):
    """
    preproc:       ColumnTransformer from above
    X_df:          dataframe without the target column
    fit:           True=fit_transform  False=transform
    tfidf_block:   CSR matrix (e.g., TFIDF_TRAIN / TFIDF_TEST) or None
    returns:       scipy.sparse CSR matrix (if any part is sparse), else ndarray
    """
    X_base = preproc.fit_transform(X_df) if fit else preproc.transform(X_df)

    if tfidf_block is not None:
        # Ensure both sides are sparse before horizontal stack
        if not sparse.isspmatrix(X_base):
            X_base = sparse.csr_matrix(X_base)
        if not sparse.isspmatrix(tfidf_block):
            tfidf_block = sparse.csr_matrix(tfidf_block)
        X_out = sparse.hstack([X_base, tfidf_block]).tocsr()
    else:
        X_out = X_base

    return X_out


# 4) Helper: try to get output feature names (for debugging/feature importance)
def get_feature_names(preproc, num_cols, low_cat):
    """
    Best-effort feature name extraction:
    - numeric pipeline names are preserved (but scaling/impute don't change names)
    - OneHotEncoder exposes expanded names via get_feature_names_out
    - 'remainder=passthrough' columns come last; we append their original names
    NOTE: Depending on sklearn version, ColumnTransformer.get_feature_names_out may not
    include remainder names. This function attempts a robust fallback.
    """
    try:
        # sklearn >= 1.0 often supports this directly
        names = preproc.get_feature_names_out()
        return names.tolist()
    except Exception:
        # Fallback: manually assemble
        names = []
        # numeric original names
        names += list(num_cols)
        # one-hot expanded names
        try:
            ohe = preproc.named_transformers_["cat"].named_steps["onehot"]
            ohe_names = ohe.get_feature_names_out(low_cat).tolist()
        except Exception:
            ohe_names = [f"{c}__oh" for c in low_cat]
        names += ohe_names
        # remainder columns (already numeric engineered features)
        # We approximate by taking columns not in (num_cols + low_cat + [TARGET])
        base = set(num_cols).union(set(low_cat)).union({TARGET})
        remainder_cols = [c for c in df_clean.columns if c not in base]
        names += remainder_cols
        return names


# ── Usage patterns ─────────────────────────────────────────────────────────
# A) Simple (no TF-IDF):
#    pipe = Pipeline([("prep", preproc), ("m", LGBMClassifier(random_state=SEED))])
#    pipe.fit(df_clean.drop(columns=[TARGET]), df_clean[TARGET])

# B) With TF-IDF (from Cell ④ power-up):
#    X_train = build_design_matrix(preproc, df_clean.drop(columns=[TARGET]),
#                                  fit=True, tfidf_block=TFIDF_TRAIN)
#    model   = LGBMClassifier(random_state=SEED).fit(X_train, df_clean[TARGET])
#    # At inference:
#    X_test  = build_design_matrix(preproc, test_clean, fit=False, tfidf_block=TFIDF_TEST)
#    preds   = model.predict(X_test)
# ╚═════════════════════════════════════════════════════════════════════════╝
# ╔════════════════════════════════ ⑥ MODEL FACTORY ═════════════════════════╗
"""
Overview
========
• Creates baseline estimators for tabular tasks (reg / binary / multiclass /
  multilabel) that work **out-of-the-box** in IOAI.
• Hyper-tuning is OPTIONAL. Turn it on via `TUNING_ON = True`.

Key flags
---------
TUNING_ON          : False ▶ baseline fit only | True ▶ run hyper-param search
SEARCH_MODE        : "random" | "grid"        (only used if TUNING_ON=True)
N_ITER_SEARCH      : 20                       (budget for RandomizedSearchCV)
CV_FOLDS           : 5
ENSEMBLE_ON        : False ▶ off | True ▶ soft-vote / regressor blend at end
USE_TFIDF_MATRIX   : auto-detects presence of TFIDF_TRAIN (text FE). If True,
                     we build CSR design matrix outside CV to keep searches fast.

Outputs
-------
best_models  : dict {name: fitted estimator}
best_scores  : dict {name: CV score}
(Optionally) ensemble_model
"""

from sklearn.model_selection import (StratifiedKFold, KFold,
                                     RandomizedSearchCV, GridSearchCV)
from sklearn.multiclass import OneVsRestClassifier
from sklearn.pipeline   import Pipeline
from sklearn.metrics    import make_scorer, f1_score
from sklearn.ensemble   import (VotingClassifier, VotingRegressor,
                                RandomForestClassifier, RandomForestRegressor)
from sklearn.linear_model import LogisticRegression
from lightgbm import LGBMClassifier, LGBMRegressor
from xgboost  import XGBClassifier, XGBRegressor
from catboost import CatBoostClassifier, CatBoostRegressor
import numpy as np, time, warnings, inspect
warnings.filterwarnings("ignore")

# ══ CONFIG FLAGS ═══════════════════════════════════════════════════════════
TUNING_ON       = False      # ▶ set True to activate hyper-param search
SEARCH_MODE     = "random"   # "random" | "grid"
N_ITER_SEARCH   = 20
CV_FOLDS        = 5
ENSEMBLE_ON     = False      # flip True after trying single models
USE_TFIDF_MATRIX= 'TFIDF_TRAIN' in globals()

# ══ 0. Task & label prep ══════════════════════════════════════════════════
TASK = detect_task(df_clean[TARGET])
y_raw = df_clean[TARGET]
if TASK == "multilabel":
    mlb  = MultiLabelBinarizer(); y_vec = mlb.fit_transform(y_raw)
else:
    y_vec = y_raw if TASK=="reg" else y_raw.astype("category").cat.codes

# build CV splitter & metric
cv = StratifiedKFold(CV_FOLDS, shuffle=True, random_state=SEED) if TASK!="reg" \
     else KFold(CV_FOLDS, shuffle=True, random_state=SEED)
SCORING = choose_metric(TASK)

print(f"[⑥] task={TASK}  tuning={TUNING_ON}  scoring={SCORING}")

# helper: OVR wrapper for multilabel
def ovr(factory): return lambda: OneVsRestClassifier(factory(), n_jobs=-1)

# ══ 1. Candidate dictionary  {name: (factory, param_grid)} ════════════════
CANDIDATES = {}

# -- LightGBM ---------------------------------------------------------------
def lgbm_factory():
    return (LGBMRegressor if TASK=="reg" else LGBMClassifier)(
        random_state=SEED,
        objective="binary" if TASK=="binary" else "multiclass" if TASK=="multiclass" else None,
        num_class=None if TASK!="multiclass" else y_raw.nunique(),
        is_unbalance=True if TASK=="binary" else False,
    )
lgbm_params = {
    "m__n_estimators":[600,1000],
    "m__learning_rate":[0.03,0.06],
    "m__num_leaves":[31,63,127],
}
CANDIDATES["lgbm"] = ((ovr(lgbm_factory) if TASK=="multilabel" else lgbm_factory),
                      lgbm_params)

# -- XGBoost ----------------------------------------------------------------
def xgb_factory():
    return (XGBRegressor if TASK=="reg" else XGBClassifier)(
        random_state=SEED, tree_method="hist",
        objective="binary:logistic" if TASK=="binary"
                 else "multi:softprob" if TASK=="multiclass" else None,
        num_class=None if TASK!="multiclass" else y_raw.nunique(),
    )
xgb_params = {
    "m__n_estimators":[800,1200],
    "m__max_depth":[4,6,8],
    "m__learning_rate":[0.03,0.06],
}
CANDIDATES["xgb"] = ((ovr(xgb_factory) if TASK=="multilabel" else xgb_factory),
                     xgb_params)

# -- CatBoost --------------------------------------------------------------
def cat_factory():
    return (CatBoostRegressor if TASK=="reg" else CatBoostClassifier)(
        random_state=SEED, verbose=False,
        loss_function="MultiClass" if TASK=="multiclass" else "Logloss"
    )
cat_params = {
    "m__iterations":[600,900],
    "m__depth":[4,6,8],
    "m__learning_rate":[0.03,0.06],
}
CANDIDATES["cat"] = ((ovr(cat_factory) if TASK=="multilabel" else cat_factory),
                     cat_params)

# -- RandomForest ----------------------------------------------------------
def rf_factory():
    return (RandomForestRegressor if TASK=="reg"
            else RandomForestClassifier)(n_estimators=700, n_jobs=-1,
                                         class_weight="balanced" if TASK!="reg" else None,
                                         random_state=SEED)
rf_params = {
    "m__max_depth":[None, 12, 20]
}
CANDIDATES["rf"] = ((ovr(rf_factory) if TASK=="multilabel" else rf_factory),
                    rf_params)

# -- Logistic baseline (classification only) ------------------------------
if TASK in ("binary","multiclass"):
    logit_factory = lambda: LogisticRegression(max_iter=2000, n_jobs=-1,
                                               class_weight="balanced" if TASK=="binary" else None)
    CANDIDATES["logit"] = (logit_factory, {"m__C":[0.3,1,3]})

# ══ 2. Data matrix builder if TF-IDF block present ========================
if USE_TFIDF_MATRIX:
    X_matrix = build_design_matrix(preproc, df_clean.drop(columns=[TARGET]),
                                   fit=True, tfidf_block=TFIDF_TRAIN)
else:
    X_matrix = None   # we’ll feed DataFrame to pipeline path

# ══ 3. Search/fit helper ==================================================
def fit_model(name, factory, param_grid):
    base_est = factory()
    if TUNING_ON:
        # choose CV search type
        search_cls = RandomizedSearchCV if SEARCH_MODE=="random" else GridSearchCV
        # If no TF-IDF: preproc goes inside pipeline (leak-safe).
        # With TF-IDF: we already built matrix; tune estimator only.
        if USE_TFIDF_MATRIX:
            search = search_cls(
                estimator=base_est,
                param_distributions={k.replace("m__",""):v for k,v in param_grid.items()},
                n_iter=N_ITER_SEARCH if SEARCH_MODE=="random" else None,
                scoring=SCORING, cv=cv, n_jobs=-1,
                verbose=1, random_state=SEED)
            search.fit(X_matrix, y_vec)
        else:
            pipe = Pipeline([("prep", preproc), ("m", base_est)])
            search = search_cls(
                estimator=pipe, param_distributions=param_grid,
                n_iter=N_ITER_SEARCH if SEARCH_MODE=="random" else None,
                scoring=SCORING, cv=cv, n_jobs=-1,
                verbose=1, random_state=SEED)
            search.fit(df_clean.drop(columns=[TARGET]), y_vec)
        return search.best_estimator_, search.best_score_
    else:
        # baseline fit, single pass
        if USE_TFIDF_MATRIX:
            base_est.fit(X_matrix, y_vec)
            return base_est, None
        else:
            pipe = Pipeline([("prep", preproc), ("m", base_est)])
            pipe.fit(df_clean.drop(columns=[TARGET]), y_vec)
            return pipe, None

# ══ 4. Loop through candidates ============================================
best_models, best_scores = {}, {}
for name,(factory,grid) in CANDIDATES.items():
    print(f"\n▶ Training {name} ({'tuned' if TUNING_ON else 'baseline'})")
    model, score = fit_model(name, factory, grid)
    best_models[name]  = model
    best_scores[name]  = score

# ══ 5. Optional ensemble ===================================================
ensemble_model = None
if ENSEMBLE_ON and len(best_models) >= 2 and TASK!="multilabel":
    estimators = [(k,v) for k,v in best_models.items()]
    ensemble_model = (VotingRegressor if TASK=="reg"
                      else VotingClassifier)(estimators=estimators,
                                             voting="soft", n_jobs=-1)
    # Fit ensemble
    if USE_TFIDF_MATRIX:
        ensemble_model.fit(X_matrix, y_vec)
    else:
        ensemble_model.fit(df_clean.drop(columns=[TARGET]), y_vec)
    print("✓ Ensemble fitted.")

print("\nCell ⑥ completed → best_models dict ready for evaluation / saving.")
# ╚═════════════════════════════════════════════════════════════════════════╝
# ╔═══════════════════════ ⑦ VALIDATE · SELECT · EXPORT ═══════════════════╗
"""
What happens here
-----------------
1) Build a hold-out validation set (same splitter as CV for consistency).
2) Score every candidate in best_models  + optional ensemble_model.
3) Pretty-print leaderboard.
4) Pick BEST_KEY (auto top score, or override).
5) Refit BEST_MODEL on full cleaned data  (Matrix path honoured if TF-IDF block exists).
6) Save model.pkl  |  generate submission.csv if test_clean is present.

You can override BEST_KEY manually after the leaderboard table.
"""

from sklearn.metrics import (accuracy_score, f1_score, roc_auc_score,
                             mean_absolute_error, r2_score,
                             hamming_loss, ConfusionMatrixDisplay)
from scipy import sparse
import joblib, pandas as pd, numpy as np, matplotlib.pyplot as plt, seaborn as sns, time, warnings
warnings.filterwarnings("ignore")

# ── 1. Hold-out split -----------------------------------------------------
X_df = df_clean.drop(columns=[TARGET])
if TASK == "multilabel":
    X_tr, X_val = X_df.iloc[:-1], X_df.iloc[-1:]   # tiny workaround for toy sets
    y_tr, y_val = y_vec[:-1], y_vec[-1:]
else:
    from sklearn.model_selection import train_test_split
    strat = y_vec if TASK in ("binary","multiclass") else None
    X_tr, X_val, y_tr, y_val = train_test_split(
        X_df, y_vec, test_size=0.2, random_state=SEED, stratify=strat)

if USE_TFIDF_MATRIX:
    Xtr_mat = build_design_matrix(preproc, X_tr, fit=True, tfidf_block=TFIDF_TRAIN)
    Xva_mat = build_design_matrix(preproc, X_val, fit=False, tfidf_block=TFIDF_TRAIN[X_tr.index])
    # (We reuse same rows from TFIDF_TRAIN; fine for hold-out.)
else:
    Xtr_mat = X_va_mat = None

# ── 2. Metric helper ------------------------------------------------------
def eval_model(model, X, y, task):
    if task=="reg":
        pred = model.predict(X)
        return mean_absolute_error(y, pred)
    if task=="multilabel":
        pred = model.predict(X)
        return f1_score(y, pred, average="micro")
    if task=="binary":
        proba = model.predict_proba(X)[:,1]
        return roc_auc_score(y, proba)