Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<train_model>
from scipy.stats import spearmanr, rankdata
import pandas as pd
import numpy as np
import pickle
print("Notebook Runtime: %0.2f Hours"%(( time.time() - notebookstart)/60/60))<load_from_csv>
data_folder = ".. /input/ieee-fraud-detection/"
sub = pd.read_csv(f'{data_folder}sample_submission.csv' )
Xy_train = pd.read_csv(".. /input/cat-in-the-dat/train.csv", index_col="id")
 X_train = Xy_train.drop(columns=["target"])
 y_train = Xy_train["target"]
X_test = pd.read_csv(".. /input/cat-in-the-dat/test.csv", index_col="id")
 X_comb_onehot = pd.get_dummies(pd.concat([X_train, X_test]), sparse=True, columns=X_train.columns)
 X_train_sparse = X_comb_onehot.loc[y_train.index].sparse.to_coo().tocsr()
X_test_sparse = X_comb_onehot.drop(index=y_train.index ).sparse.to_coo().tocsr()
lr_params = dict(solver="lbfgs", C=0.2, max_iter=5000, random_state=0)
 models = [LogisticRegression(**lr_params ).fit(X_train_sparse[t], y_train[t])
 for t, _ in KFold(5, random_state=0 ).split(X_train_sparse)]
predictions = np.average([model.predict_proba(X_test_sparse)[:, 1] for model in models], axis=0)
 output = pd.DataFrame({"id": X_test.index, "target": predictions})
 output.to_csv("submission.csv", index=False)
<load_from_csv>
with open(".. /input/ieee-sol1/test_uid_nocoverage.p", "rb")as f:
test_uid_nocoverage = pickle.load(f )
Xy_labelled = pd.read_csv(".. /input/cat-in-the-dat/train.csv", index_col="id")
 X_labelled = Xy_labelled.drop(columns=["target"])
 y_labelled = Xy_labelled["target"]
X_test = pd.read_csv(".. /input/cat-in-the-dat/test.csv", index_col="id")
 target_mean = y_labelled.mean()
for col in X_labelled.columns:
stats = Xy_labelled.groupby(col)["target"].agg(["sum", "count"])
 likelihoods =(( stats["sum"] +(800 * target_mean)) /(stats["count"] + 800)).to_dict()
X_labelled[col] = X_labelled[col].map(likelihoods)
 X_test[col] = X_test[col].map(lambda v: likelihoods.get(v, target_mean))
model = LogisticRegression(solver="saga", n_jobs=-1, random_state=0 ).fit(X_labelled, y_labelled)
 proba_predictions = model.predict_proba(X_test)[:, 1]
output = pd.DataFrame({"id": X_test.index, "target": proba_predictions})
 output.to_csv("submission.csv", index=False)
<define_variables>
sub1 = rankdata(pd.read_csv(".. /input/ieee-submissions-and-uids/final_model_blend.csv" ).isFraud)
 sub1 /= len(sub1)
 sub2 = rankdata(pd.read_csv(".. /input/ieee-sol1/blend_of_blends_1.csv" ).isFraud)
 sub2 /= len(sub2 )
data_path = "/kaggle/input/simple-preprocessing/"<load_pretrained>
spearmanr(sub1, sub2 )
with open(data_path+"/df_train.pkl", "rb")as myfile:
df_train = pickle.load(myfile )<load_pretrained>
sub3 = sub1
sub3[test_uid_nocoverage] = 0.5 * sub1[test_uid_nocoverage] + 0.5 * sub2[test_uid_nocoverage]
<define_variables><EOS> 
sub['isFraud'] = sub3
sub.to_csv('submission.csv', index=False)
 sub.head()