Kaggle competition: House Prices - Advanced Regression Techniques
train["compiled_leak"] = 0
test["compiled_leak"] = 0
for i in range(NLAGS):
train.loc[train["compiled_leak"] == 0, "compiled_leak"] = train.loc[train["compiled_leak"] == 0, "leaked_target_"+str(i)]
test.loc[test["compiled_leak"] == 0, "compiled_leak"] = test.loc[test["compiled_leak"] == 0, "leaked_target_"+str(i)]
print("Leak values found in train and test ", sum(train["compiled_leak"] > 0), sum(test["compiled_leak"] > 0))
print("% of correct leaks values in train ", sum(train["compiled_leak"] == train["target"])/sum(train["compiled_leak"] > 0))
train.loc[train["compiled_leak"] == 0, "compiled_leak"] = train.loc[train["compiled_leak"] == 0, "nonzero_mean"]
test.loc[test["compiled_leak"] == 0, "compiled_leak"] = test.loc[test["compiled_leak"] == 0, "nonzero_mean"]<compute_test_metric>
def label_encoding(train: pd.DataFrame, test: pd.DataFrame, col_definition: dict):

 n_train = len(train)
 train = pd.concat([train, test], sort=False ).reset_index(drop=True)
 for f in col_definition['encode_col']:
try:
lbl = preprocessing.LabelEncoder()
train[f] = lbl.fit_transform(list(train[f].values))
except:
print(f)
 test = train[n_train:].reset_index(drop=True)
 train = train[:n_train]
return train, test
np.sqrt(mean_squared_error(y, np.log1p(train["compiled_leak"] ).fillna(14.49)) )<save_to_csv>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 sub = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv' )
sub = test[["ID"]]
sub["target"] = test["compiled_leak"]
sub.to_csv("baseline_submission_with_leaks.csv", index=False )<load_from_csv>
unique_cols = list(train.columns[train.nunique() == 1])
 print(unique_cols )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 transact_cols = [f for f in train.columns if f not in ["ID", "target"]]
y = np.log1p(train["target"] ).values<set_options>
duplicated_cols = list(train.columns[train.T.duplicated() ])
 print(duplicated_cols )
gc.collect() ;
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1', '15ace8c9f',
'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9', 'd6bb78916', 'b43a7cfd5',
'58232a6fb', '1702b5bf0', '324921c7b', '62e59a501', '2ec5b290f', '241f0f867',
'fb49e4212', '66ace2992', 'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7',
'1931ccfdd', '703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535', 'fc99f9426', '91f701ba2', '0572565c2', '190db8488',
'adb64ff71', 'c47340d97', 'c5a231d81'
]<statistical_test>
train, test = label_encoding(train, test, col_definition={'encode_col':categorical_cols} )
def get_beautiful_test(test):
test_rnd = np.round(test.iloc[:, 1:], 2)
 ugly_indexes = []
non_ugly_indexes = []
for idx in tqdm(range(len(test))):
if not np.all(
 test_rnd.iloc[idx, :].values==test.iloc[idx, 1:].values 
):
ugly_indexes.append(idx)
 else:
non_ugly_indexes.append(idx)
 print(len(ugly_indexes), len(non_ugly_indexes))
np.save('test_ugly_indexes', np.array(ugly_indexes))
np.save('test_non_ugly_indexes', np.array(non_ugly_indexes))
test = test.iloc[non_ugly_indexes].reset_index(drop=True)
 return test, non_ugly_indexes, ugly_indexes<feature_engineering>
X_train = train.drop(['Id', 'SalePrice'], axis=1)
 y_train = np.log1p(train['SalePrice'])
 X_test = test.drop(['Id', 'SalePrice'], axis=1 )
test, non_ugly_indexes, ugly_indexes = get_beautiful_test(test)
 test["target"] = train["target"].mean()<categorify>
def objective(trial):
y_preds = []
models = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=0)
 params = {
'num_leaves': trial.suggest_int('num_leaves', 20, 30),
'max_depth': trial.suggest_int('max_depth', 3, 6),
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.05
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.loc[train_index, :]
X_val = X_train.loc[valid_index, :]
y_tr = y_train[train_index]
y_val = y_train[valid_index]
lgb_train = lgb.Dataset(X_tr,
y_tr,
categorical_feature=categorical_cols)
 lgb_eval = lgb.Dataset(X_val,
y_val,
reference=lgb_train,
categorical_feature=categorical_cols)
 model = lgb.train(params,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 oof_train[valid_index] = model.predict(X_val,
num_iteration=model.best_iteration)
 y_pred = model.predict(X_test,
num_iteration=model.best_iteration)
 y_preds.append(y_pred)
 models.append(model)
 score = np.sqrt(mean_squared_error(y_train, oof_train))
return score
CPU_CORES = 1
def _get_leak(df, cols, lag=0, verbose=False):

 series_str = df[cols[lag+2:]].apply(lambda x: "_".join(x.round(2 ).astype(str)) , axis=1)
 series_shifted_str = df[cols].shift(lag+2, axis=1)[cols[lag+2:]].apply(lambda x: "_".join(x.round(2 ).astype(str)) , axis=1)
 if verbose:
target_rows = series_shifted_str.progress_apply(lambda x: np.where(x == series_str)[0])
 else:
target_rows = series_shifted_str.apply(lambda x: np.where(x == series_str)[0])
 target_vals = target_rows.apply(lambda x: df.loc[x[0], cols[lag]] if len(x)==1 else 0)
 return target_vals
def get_all_leak(df, cols=None, nlags=15):

 df = df.copy()
for i in range(nlags):
if "leaked_target_"+str(i)not in df.columns:
print("Processing lag {}".format(i))
df["leaked_target_"+str(i)] = _get_leak(df, cols, i)
 return df
def compiled_leak_result() :
max_nlags = len(cols)- 2
train_leak = train[["ID", "target"] + cols]
train_leak["compiled_leak"] = 0
train_leak["nonzero_mean"] = train[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_value_corrects = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 train_leak[c] = _get_leak(train_leak, cols, i)
 leaky_cols.append(c)
 train_leak = train.join(
 train_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)
 zeroleak = train_leak["compiled_leak"]==0
train_leak.loc[zeroleak, "compiled_leak"] = train_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(train_leak["compiled_leak"] > 0))
_correct_counts = sum(train_leak["compiled_leak"]==train_leak["target"])
 leaky_value_corrects.append(_correct_counts/leaky_value_counts[-1])
 print("Leak values found in train", leaky_value_counts[-1])
 print(
 "% of correct leaks values in train ",
leaky_value_corrects[-1] 
)
 tmp = train_leak.copy()
tmp.loc[zeroleak, "compiled_leak"] = tmp.loc[zeroleak, "nonzero_mean"]
scores.append(np.sqrt(mean_squared_error(y, np.log1p(tmp["compiled_leak"] ).fillna(14.49))))
print(
 'Score(filled with nonzero mean)',
scores[-1] 
)
 result = dict(
 score=scores,
leaky_count=leaky_value_counts,
leaky_correct=leaky_value_corrects, 
)
 return train_leak, result<save_to_csv>
study = optuna.create_study(sampler=optuna.samplers.RandomSampler(seed=0))
study.optimize(objective, n_trials=40 )
result = pd.DataFrame.from_dict(result, orient='columns')
 result
result.to_csv('train_leaky_stat.csv', index=False)
 train_leak.head()<compute_test_metric>
study.best_params
best_score = np.min(result['score'])
 best_lag = np.argmin(result['score'])
 print('best_score', best_score, '
best_lag', best_lag )<feature_engineering>
X_train = X_train.drop(['GrLivArea', 'TotalBsmtSF', 'BsmtFinSF1'], axis=1)
 X_test = X_test.drop(['GrLivArea', 'TotalBsmtSF', 'BsmtFinSF1'], axis=1 )
def rewrite_compiled_leak(leak_df, lag):
leak_df["compiled_leak"] = 0
for i in range(lag):
c = "leaked_target_"+str(i)
 zeroleak = leak_df["compiled_leak"]==0
leak_df.loc[zeroleak, "compiled_leak"] = leak_df.loc[zeroleak, c]
return leak_df<categorify>
y_preds = []
models = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=0)
 params = {
'num_leaves': study.best_params['num_leaves'],
'max_depth': study.best_params['max_depth'],
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.05
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.loc[train_index, :]
X_val = X_train.loc[valid_index, :]
y_tr = y_train[train_index]
y_val = y_train[valid_index]
lgb_train = lgb.Dataset(X_tr,
y_tr,
categorical_feature=categorical_cols)
 lgb_eval = lgb.Dataset(X_val,
y_val,
reference=lgb_train,
categorical_feature=categorical_cols)
 model = lgb.train(params,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 oof_train[valid_index] = model.predict(X_val,
num_iteration=model.best_iteration)
 y_pred = model.predict(X_test,
num_iteration=model.best_iteration)
 y_preds.append(y_pred)
 models.append(model )
leaky_cols = [c for c in train_leak.columns if 'leaked_target_' in c]
train_leak = rewrite_compiled_leak(train_leak, best_lag)
 train_leak[['ID']+leaky_cols+['compiled_leak']].head()<save_to_csv>
print(f'CV: {np.sqrt(mean_squared_error(y_train, oof_train)) }' )
train_res = train_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 train_res.to_csv('train_leak.csv', index=False )<feature_engineering>
importance = pd.DataFrame(model.feature_importance() , index=X_train.columns, columns=['importance'])
 result = importance.sort_values('importance', ascending=False )
def compiled_leak_result_test(max_nlags):
test_leak = test[["ID", "target"] + cols]
test_leak["compiled_leak"] = 0
test_leak["nonzero_mean"] = test[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 test_leak[c] = _get_leak(test_leak, cols, i, verbose=True)
 leaky_cols.append(c)
 test_leak = test.join(
 test_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)[["ID", "target"] + cols + leaky_cols+["compiled_leak", "nonzero_mean"]]
zeroleak = test_leak["compiled_leak"]==0
test_leak.loc[zeroleak, "compiled_leak"] = test_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(test_leak["compiled_leak"] > 0))
print("Leak values found in test", leaky_value_counts[-1])
 result = dict(
 leaky_count=leaky_value_counts, 
)
 return test_leak, result<categorify>
y_sub = sum(y_preds)/ len(y_preds)
 y_sub = np.expm1(y_sub)
 sub['SalePrice'] = y_sub
sub.to_csv('submission.csv', index=False)
 sub.head()