Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<split>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
env = janestreet.make_env()
env_iter = env.iter_test()<categorify>
%matplotlib inline
plt.rcParams['figure.figsize'] = [20.0, 7.0]
plt.rcParams.update({'font.size': 22,})
 sns.set_palette('viridis')
 sns.set_style('white')
 sns.set_context('talk', font_scale=0.8 )
opt_th = 0.5
f = np.median
f_mean = np.load('./f_mean.npy')
 for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:,:].sum()):
x_tt[:, :] = np.nan_to_num(x_tt[:, :])+ np.isnan(x_tt[:, :])* f_mean
pred = []
for xgb_ in xgb_models:
batch = treelite_runtime.Batch.from_npy2d(x_tt, rbegin=0, rend=1)
 pred.append(np.array(xgb_.predict(batch)).reshape(1,1))
for clf in models:
predict = np.mean(clf(x_tt, training = False ).numpy() , axis = 0)
 pred.append(f(predict))
pred_df.action = np.where(np.mean(pred)>= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<load_pretrained>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('Train Shape: ', train.shape)
 print('Test Shape: ', test.shape)
 train.head()
def save_pickle(dict_param,nom_fich):
with open(nom_fich,"wb")as f :
pickle.dump(dict_param,f )<load_from_csv>
X_train_df = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train_df)
 X_test = scaler.transform(X_test )
path = "/kaggle/input/jane-street-market-prediction/"
train = pd.read_csv(path + "train.csv")
 features = pd.read_csv(path + "features.csv" )<create_dataframe>
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=100)
 lr_scores = cross_val_score(lr,
X_train,
y_train,
cv=5,
scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, X_train, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )
missing_table = pd.DataFrame({c:(train[c].isna().sum() /len(train)) *100 for c in train.columns},index=["% missing values"])
 missing_table<feature_engineering>
train.isnull().any().any()
def detect_correlated_features(df,threshold=0.5):

 correlated= defaultdict(list)
 for col in df.columns:
dex = list(df.columns ).index(col)
 for ind in df.index[dex+1:] :
if df.loc[col,ind] > threshold:
correlated[col].append(ind)
 return correlated<feature_engineering>
sel = feature_selection.VarianceThreshold()
train_variance = sel.fit_transform(train)
 train_variance.shape
correlated_features = detect_correlated_features(correlation_table,threshold=0.9 )<define_variables>
to_drop = [column for column in matrix.columns if any(matrix[column] > 0.50)]
print('Columns to drop: ' ,(len(to_drop)) )
features_to_drop = [f for f in ay_features if f not in ax_features]<drop_column>
k_best = feature_selection.SelectKBest(score_func=feature_selection.f_classif, k=100)
 fit = k_best.fit(X_train, y_train)
 univariate_features = fit.transform(X_train )
train_df = train[[ f for f in list(train.columns)if(( f not in features_to_drop)or(f =="resp")) ]]<create_dataframe>
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=100)
 lr_scores = cross_val_score(lr, univariate_features, y_train, cv=5, scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, univariate_features, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )
label_correlation = pd.DataFrame({c:train_df["resp"].corr(train_df[c])for\
c in train_df.columns if c!="resp" and c!="feature_0"},index=["action"])
<feature_engineering>
rfe = feature_selection.RFE(lr, n_features_to_select=100)
 fit = rfe.fit(X_train, y_train)
 recursive_features = fit.transform(X_train )
train_df.loc[:,"target"] = list(((train_df["resp"] > 0)&(train_df["resp_1"] >0)&(train_df["resp_2"]>0)\
&(train_df["resp_3"]>0)&(train_df["resp_4"]>0)).astype("int"))
train_df.loc[:,"vl"] = list(train_df["target"].values)
 pvt_table=train_df[["feature_0","target"]].pivot_table(index=["feature_0"],columns=["target"],aggfunc=len)
 tx= train_df["feature_0"].value_counts()
ty = train_df["target"].value_counts()
tx = pd.DataFrame(tx)
 ty = pd.DataFrame(ty)
 n = len(train_df)
 tx.columns = ["values"]
ty.columns = ["values"]
cnt = tx.dot(ty.T)/n
ind = cnt.index
pvt_table = pvt_table.loc[ind,:]
mesure =(cnt - pvt_table)**2/cnt
xin = mesure.sum().sum()
del(train_df["target"])
 del(train_df["vl"])
 fig = plt.figure(figsize=(12,8))
sns.heatmap(mesure,annot=True,linewidths=0.5,cmap="BrBG",vmin=0,vmax=1)
 plt.title("Correlation table between feature_0 and the target value ",size=15,color="red")
 print("The total correlation between feature_0 and the target equal to {}".format(xin)) 
<create_dataframe>
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=10)
 lr_scores = cross_val_score(lr, recursive_features, y_train, cv=5, scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, recursive_features, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )
missing_b_imputation = pd.DataFrame({c:(train_df[c].isna().sum())for c in train_df.columns},index=["% missing values"])
 print("The number of missing values in the train_df dataframe before imputation processing :{}".format(\
missing_b_imputation.sum().sum()))
missing_b_imputation<filter>
select_model = feature_selection.SelectFromModel(lr)
 fit = select_model.fit(X_train, y_train)
 model_features = fit.transform(X_train )
features_with_missing_values = []
for f in list(train_df.columns):
if missing_table.loc["% missing values",f] > 0 :
features_with_missing_values.append(f )<train_on_grid>
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=100)
 lr_scores = cross_val_score(lr, model_features, y_train, cv=5, scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, model_features, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )
for f in features_with_missing_values :
model = LinearRegression()
if len(correlated_features[f])> 0 :
correlated = correlated_features[f][0]
if correlated in train.columns :
model.fit(train.loc[(train[correlated].notna())&(train[f].notna()),correlated].values.reshape(-1,1),\
train.loc[(train[correlated].notna())&(train[f].notna()),f])
 values_to_impute = train_df.loc[(train[f].isna())&(train[correlated].notna()),f]
imputer = train.loc[(train[f].isna())&(train[correlated].notna()),correlated].values
if(len(values_to_impute)> 0)&(len(imputer)> 0):
train_df.loc[(train[f].isna())&(train[correlated].notna()),f] = model.predict(train.loc[(train[f].isna())&(train[correlated].notna()),correlated].values.\
reshape(-1,1)) 
<create_dataframe>
pca = PCA(0.90)
 principal_components = pca.fit_transform(X_train)
 principal_df = pd.DataFrame(data = principal_components)
 principal_df.shape
missing_a_imputation = pd.DataFrame({c:(train_df[c].isna().sum())for c in train_df.columns},index=["Number missing values"])
 print("The number of missing values in the train_df dataframe after imputation processing :{}".format(\
missing_a_imputation.sum().sum()))
missing_a_imputation<data_type_conversions>
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=100)
 lr_scores = cross_val_score(lr, principal_df, y_train, cv=5, scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, principal_df, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )
for f in features_with_missing_values:
train_df.fillna(train_df[f].mean() ,inplace=True )<prepare_x_and_y>
pca = PCA(0.75)
 principal_components = pca.fit_transform(X_train)
 principal_df = pd.DataFrame(data = principal_components)
 principal_df.shape
train_df["action"] =(( train_df["resp"] > 0)&(train_df["resp_1"] >0)&(train_df["resp_2"]>0)\
&(train_df["resp_3"]>0)&(train_df["resp_4"]>0)).astype("int")
<load_from_csv>
lr = LogisticRegression(solver='liblinear')
 rfc = RandomForestClassifier(n_estimators=100)
 lr_scores = cross_val_score(lr, principal_df, y_train, cv=5, scoring='roc_auc')
 rfc_scores = cross_val_score(rfc, principal_df, y_train, cv=5, scoring='roc_auc')
 print('LR Scores: ', lr_scores)
 print('RFC Scores: ', rfc_scores )
features = pd.read_csv(path + "features.csv" )<data_type_conversions>
rfe = feature_selection.RFE(lr, n_features_to_select=100)
 fit = rfe.fit(X_train, y_train)
 recursive_X_train = fit.transform(X_train)
 recursive_X_test = fit.transform(X_test)
 lr = LogisticRegression(C=1, class_weight={1:0.6, 0:0.4}, penalty='l1', solver='liblinear')
 lr_scores = cross_val_score(lr, recursive_X_train, y_train, cv=5, scoring='roc_auc')
 lr_scores.mean()
features.set_index('feature',inplace=True)
 features = features.astype("int8" )<choose_model_class>
predictions = lr.fit(recursive_X_train, y_train ).predict_proba(recursive_X_test )
<find_best_params><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions
submission.to_csv('submission.csv', index=False)
 submission.head()