Kaggle competition: Titanic - Machine Learning from Disaster
_, best_score1 = cross_val_and_print(pipe)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3 )<categorify>
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
[f for f in train.columns if not f in ["target"]])
 pipe = make_pipeline(make_column_transformer(ohe1), LogisticRegression(**lr_params))<compute_train_metric>
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2 )<data_type_conversions>
for df in [train, test]:
df["bin_0_bin_3"] = df["bin_3"].astype(str)+ df["bin_0"].astype(str )<categorify>
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import make_scorer, accuracy_score
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import LabelEncoder
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
[f for f in train.columns if not f in ["target", "bin_0", "bin_3"]])
 pipe = make_pipeline(make_column_transformer(ohe1), LogisticRegression(**lr_params))<compute_train_metric>
df = pd.read_csv('.. /input/train.csv', header = 0, dtype={'Age': np.float64})
 test = pd.read_csv('.. /input/test.csv' , header = 0, dtype={'Age': np.float64} )
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )<drop_column>
df.groupby(by=['Pclass'])['Survived'].agg(['mean','count'] )
def transf_ordinal_features(serie):
dtransf = {"ord_1":{'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4}
, "ord_2":{'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot':4, 'Lava Hot':5}
, "nom_0":{"Blue":1, "Green":2, "Red":3}
, "nom_1":{"Circle":1, "Trapezoid":2, "Star":3, "Polygon":4, "Square":5, "Triangle":6}
, "nom_2":{"Dog":1, "Lion":2, "Snake":3, "Axolotl":4, "Cat":5, "Hamster":6}
, "nom_3":{"Finland":1, "Russia":2, "China":3, "Costa Rica":4, "Canada":5, "India":6}
, "nom_4":{"Bassoon":1, "Piano":2, "Oboe":3, "Theremin":4}
, "bin_0_bin_3_bis":{"T0":0, "F1":2, "F0":1}
}
if serie.name == "ord_0":
new_serie = serie - 1
elif serie.name == "ord_5":
lm = serie.unique()
new_serie = serie.map({l:i for i, l in enumerate(list(np.sort(lm)))})
 elif serie.name in ["ord_3", "ord_4"]:
new_serie = serie.str.lower().map({l:i for i, l in enumerate(list(ascii_lowercase)) })
 else:
new_serie = serie.map(dtransf[serie.name])
 return new_serie<categorify>
Image(url= "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/2bc37b51-c9e4-402e-938e-70d3145815f2/d787jna-1b3767d2-f297-4b73-a874-7cfa6d1e8a69.png/v1/fill/w_1600,h_460,q_80,strp/r_m_s__titanic_class_system_by_monroegerman_d787jna-fullview.jpg?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1cm46YXBwOjdlMGQxODg5ODIyNjQzNzNhNWYwZDQxNWVhMGQyNmUwIiwiaXNzIjoidXJuOmFwcDo3ZTBkMTg4OTgyMjY0MzczYTVmMGQ0MTVlYTBkMjZlMCIsIm9iaiI6W1t7ImhlaWdodCI6Ijw9NDYwIiwicGF0aCI6IlwvZlwvMmJjMzdiNTEtYzllNC00MDJlLTkzOGUtNzBkMzE0NTgxNWYyXC9kNzg3am5hLTFiMzc2N2QyLWYyOTctNGI3My1hODc0LTdjZmE2ZDFlOGE2OS5wbmciLCJ3aWR0aCI6Ijw9MTYwMCJ9XV0sImF1ZCI6WyJ1cm46c2VydmljZTppbWFnZS5vcGVyYXRpb25zIl19.6krQcPQvsfcQ_ZJ_CGvufi9MT-PJkkg1I8-grLy7Hiw" )
class MyFeaturesEngineering(BaseEstimator, TransformerMixin):
def __init__(self, list_ordinal_features =[]):
self.list_ordinal_features = list_ordinal_features
def fit(self, x, y=None):
return self
def transform(self, x, y=None):
df = x.copy()
for v in self.list_ordinal_features:
df[v] = transf_ordinal_features(df[v])
 return df<categorify>
sex_survived= df.groupby(by=['Sex','Survived'])['Survived'].agg(['count'] ).reset_index()
sex_survived
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day", "month"]
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
for feat in ordinal_features:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
list(set(ohe1_feats+ordinal_features)-{feat}))
MyFeE = MyFeaturesEngineering(list_ordinal_features = [feat])
 StdScalE =(StandardScaler(copy=False), [feat])
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, best_score=best_score1, comment1=feat)
<categorify>
def configure_plotly_browser_state() :
display(IPython.core.display.HTML() )
ordinal_features = ["ord_0"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4", "nom_5"
, "nom_6", "nom_7", "nom_8", "nom_9", "day", "month"] + ["ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features)
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )<categorify>
male_survived=pd.DataFrame(df['Age'][(df['Sex']=='male')&(df['Survived']==1)].value_counts().sort_index(ascending=False)).reset_index().rename(columns={'index':'Age','Age':'Number'})
 female_survived=pd.DataFrame(df['Age'][(df['Sex']=='female')&(df['Survived']==1)].value_counts().sort_index(ascending=False)).reset_index().rename(columns={'index':'Age','Age':'Number'})
 male_not_survived=pd.DataFrame(df['Age'][(df['Sex']=='male')&(df['Survived']==0)].value_counts().sort_index(ascending=False)).reset_index().rename(columns={'index':'Age','Age':'Number'})
 female_not_survived=pd.DataFrame(df['Age'][(df['Sex']=='female')&(df['Survived']==0)].value_counts().sort_index(ascending=False)).reset_index().rename(columns={'index':'Age','Age':'Number'})
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day", "month"]
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features)
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))<compute_train_metric>
df_all = [df,test]
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )<categorify>
for data in df_all:
print(f"
-------- {data.index } -------
")
 print(data.isnull().sum() )
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore"),
ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features)
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )<categorify>
for data in df_all:
data['isAlone']=1
data['Family_No'] = data['Parch'] + data['SibSp'] + 1
data['isAlone'].loc[data['Family_No']>1]=0
data['Age'].fillna(round(data['Age'].mean()), inplace=True)
 data['Embarked'].fillna(data['Embarked'].mode().iloc[0], inplace=True)
 data['Fare'] = df.groupby('Pclass')['Fare'].apply(lambda x: x.fillna(x.mean()))
class MyBinsEncoder(BaseEstimator, TransformerMixin):
def __init__(self, nbins=100, nmin=20):
self.nbins = nbins
self.nmin = nmin
def fit(self, x, y=None):
temp = pd.concat([x, y], axis=1)
 averages = temp.groupby(by=x.name)[y.name].mean()
means = dict(zip(averages.index.values, averages.values))
bins = np.linspace(averages.min() , averages.max() , self.nbins)
 self.map_ = dict(zip(averages.index.values, np.digitize(averages.values, bins=bins)))
 count = temp.groupby(by=x.name)[y.name].count()
nobs = dict(zip(averages.index.values, count))
for key, value in nobs.items() :
if value > self.nmin:
self.map_[key] = key
return self
def transform(self, x, y=None):
temp = x.map(self.map_)
 temp.fillna(random.choice(list(self.map_.values())) , inplace=True)
 temp = temp.astype(str)
 return temp
essai = MyBinsEncoder(nbins=4, nmin=100000)
 essai.fit(train["day"], train["target"])
 print(essai.map_)
 av = essai.transform(train["day"])
 print(av.value_counts(dropna=False))
print(av.nunique() )<categorify>
test.isAlone.value_counts()
class MyFeaturesEngineering(BaseEstimator, TransformerMixin):
def __init__(self, list_ordinal_features =[], feat_to_bins_encode = {}):
self.list_ordinal_features = list_ordinal_features
self.feat_to_bins_encode = feat_to_bins_encode
self.BinsEncoder={}
def fit(self, x, y=None):
for feat, value in self.feat_to_bins_encode.items() :
self.BinsEncoder[feat] = MyBinsEncoder(nbins=value[0], nmin=value[1])
 self.BinsEncoder[feat].fit(x[feat], y)
 return self
def transform(self, x, y=None):
df = x.copy()
for v in self.feat_to_bins_encode.keys() :
df[v] = self.BinsEncoder[v].transform(df[v])
 for v in self.list_ordinal_features:
df[v] = transf_ordinal_features(df[v])
 return df<categorify>
for data in df_all:
data.drop(columns=['PassengerId','Name','Cabin','Ticket','SibSp','Parch'],inplace=True,axis=1 )
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
feat = "nom_5"
for nbins in [[20, 35]]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={feat:nbins})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1, comment1=feat, comment2=nbins )<compute_train_metric>
for data in df_all:
print(f"
-------- {data.index } -------
")
 print(data.isnull().sum() )
_, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat, comment2=nbins)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3, comment1=feat, comment2=nbins )<categorify>
test = pd.get_dummies(test,columns=['Sex','Embarked'])
 df = pd.get_dummies(df,columns=['Sex','Embarked'] )
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
feat = "nom_6"
for nbins in [[20, 5]]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={feat:nbins, "nom_5":[20, 35]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1, comment1=feat, comment2=nbins )<compute_train_metric>
y=df['Survived']
X=df.drop(columns=['Survived'],axis=1 )
_, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat, comment2=nbins)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3, comment1=feat, comment2=nbins )<categorify>
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3 )
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
feat = "nom_8"
for nbins in [[15, 3]]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={feat:nbins, "nom_5":[20, 35]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, StdScalE), LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1, comment1=feat, comment2=nbins )<compute_train_metric>
DT= DecisionTreeClassifier()
DT.fit(X_train, y_train)
 DT.score(X_test,y_test )
_, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat, comment2=nbins)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3, comment1=feat, comment2=nbins )<categorify>
IFrame("https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_8", "day_bis", "month"]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop="first")
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, best_score=best_score1, comment1=feat )<categorify>
parameters1 = [{'max_depth':np.linspace(1, 15, 15),'min_samples_split': np.linspace(0.1, 1.0, 5, endpoint=True)}]
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_8", "day_bis", "month"]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
lval_to_drop = [train[feat].value_counts().index[0]]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, best_score=best_score1, comment1=feat )<normalization>
Grid1 = GridSearchCV(DT, parameters1, cv=4,return_train_score=True,iid=True)
 Grid1.fit(X_train,y_train )
ohe1_feats = ["bin_0_bin_3", "bin_1", "nom_5", "nom_7", "nom_9", "month"]
ohe2_feats = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_6', "day_bis"]
ohe3_feats = ['bin_2', 'bin_4', 'nom_8']
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 lval_to_drop = [train[v].value_counts().index[0] for v in ohe2_feats]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , ohe2_feats)
 ohe3 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop='first'), ohe3_feats)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, ohe3, StdScalE)
 , LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )<categorify>
scores = Grid1.cv_results_
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_6', "day_bis"]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
lval_to_drop = [train[feat].value_counts().index[0]]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat)
<categorify>
for param, mean_train in zip(scores['params'],scores['mean_train_score']):
print(f"{param} accuracy on training data is {mean_train}" )
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ['nom_1', 'nom_3', "day_bis"]:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
lval_to_drop = [train[feat].value_counts().index[0]]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, cv=folds3, best_score=best_score3, comment1=feat )<categorify>
Grid1.best_estimator_
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
ohe1_feats = ["bin_0_bin_3", "bin_1", "bin_2", "bin_4", "nom_0", "nom_1", "nom_2", "nom_3", "nom_4"
, "nom_5", "nom_6", "nom_7", "nom_8", "nom_9", "day_bis", "month"]
for feat in ['bin_2', 'bin_4', 'nom_8']:
ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , list(set(ohe1_feats)-{feat}))
lval_to_drop = [train[feat].value_counts().index[0]]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop='first')
 , [feat])
 MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, StdScalE)
 , LogisticRegression(**lr_params))
_, _ = cross_val_and_print(pipe, cv=folds2, best_score=best_score2, comment1=feat)
<normalization>
XGB = XGBClassifier()
ohe1_feats = ["bin_0_bin_3", "bin_1", 'bin_2', 'bin_4', 'nom_0', 'nom_2', 'nom_4', "nom_5", 'nom_6'
, "nom_7", "nom_9", "month"]
ohe2_feats = ['nom_1', 'nom_3', "day_bis"]
ohe3_feats = ['nom_8']
ordinal_features = ["ord_0", "ord_1", "ord_2", "ord_3", "ord_4", "ord_5"]
MyFeE = MyFeaturesEngineering(list_ordinal_features = ordinal_features
, feat_to_bins_encode={"nom_5":[20, 35], "nom_8":[15, 3]})
 StdScalE =(StandardScaler(copy=False), ordinal_features)
 ohe1 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', handle_unknown="ignore")
 , ohe1_feats)
 lval_to_drop = [train[v].value_counts().index[0] for v in ohe2_feats]
ohe2 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop=lval_to_drop)
 , ohe2_feats)
 ohe3 =(OneHotEncoder(categories = 'auto', sparse = True, dtype = 'uint8', drop='first'), ohe3_feats)
 pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, ohe3, StdScalE)
 , LogisticRegression(**lr_params))
_, best_score1 = cross_val_and_print(pipe, best_score=best_score1)
 _, best_score2 = cross_val_and_print(pipe, cv=folds2, best_score=best_score2)
 _, best_score3 = cross_val_and_print(pipe, cv=folds3, best_score=best_score3 )<save_to_csv>
parameters3 =[{"learning_rate": [0.05, 0.10, 0.15, 0.20] ,"max_depth": [ 3, 4, 5, 6], "min_child_weight": [3,5,7],"gamma": [ 0.0, 0.1, 0.2 ,0.3],"colsample_bytree" : [ 0.4, 0.5]}]
lr_params = {'penalty': 'l2', 'solver': 'lbfgs', 'C': 0.123456789, 'max_iter':500}
pipe = make_pipeline(MyFeE, make_column_transformer(ohe1, ohe2, ohe3, StdScalE)
 , LogisticRegression(**lr_params))
pred = pipe.fit(train.drop(["target"], axis = 1), train["target"] ).predict_proba(test)[:, 1]
pd.DataFrame({"id": test.index, "target": pred} ).to_csv("submission.csv", index=False )<import_modules>
Grid1 = GridSearchCV(XGB, parameters3, cv=2,return_train_score=True)
 Grid1.fit(X_train,y_train )
import numpy as np
import pandas as pd
import scipy
import sklearn<import_modules>
scores = Grid1.cv_results_
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder<import_modules>
Grid1.best_estimator_
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_validate<load_from_csv>
XGB = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bytree=0.5, gamma=0.0, learning_rate=0.1,
max_delta_step=0, max_depth=3, min_child_weight=5, missing=None,
n_estimators=100, n_jobs=1, nthread=None,
objective='binary:logistic', random_state=0, reg_alpha=0,
reg_lambda=1, scale_pos_weight=1, seed=None, silent=True,
subsample=1 )
df_train = pd.read_csv('/kaggle/input/cat-in-the-dat/train.csv', index_col='id')
 df_test = pd.read_csv('/kaggle/input/cat-in-the-dat/test.csv', index_col='id' )<load_from_csv>
XGB.fit(X_train, y_train )
sample_submission = pd.read_csv('/kaggle/input/cat-in-the-dat/sample_submission.csv', index_col='id' )<define_variables>
XGB.score(X_test,y_test )
binvar = ['bin_' + str(i)for i in range(1,5)]
ordvar = ['ord_' + str(i)for i in range(6)]
nomvar = ['nom_' + str(i)for i in range(10)]
dmvar = ['day', 'month']<prepare_x_and_y>
df_work = df_train.copy()
y_train = df_work['target'].copy()
df_work = pd.concat([df_work.drop('target', axis = 1), df_test] )<drop_column>
df = pd.read_csv('.. /input/train.csv' , header = 0,dtype={'Age': np.float64})
 test = pd.read_csv('.. /input/test.csv' , header = 0,dtype={'Age': np.float64} )
df_work.drop('bin_0', inplace=True, axis=1 )<categorify>
for data in [df,test]:
data['isAlone']=1
data['Family_No'] = data['Parch'] + data['SibSp'] + 1
data['isAlone'].loc[data['Family_No']>1]=0
data['Age'].fillna(data['Age'].mean() , inplace=True)
 data['Embarked'].fillna(data['Embarked'].mode().iloc[0], inplace=True)
 data['Fare'] = df.groupby('Pclass')['Fare'].apply(lambda x: x.fillna(x.mean()))
df_work['bin_3'] = df_work['bin_3'].map({'F':0, 'T':1})
 df_work['bin_4'] = df_work['bin_4'].map({'N':0, 'Y':1} )<feature_engineering>
df_work['ord_0'] = df_work['ord_0'] - 1<categorify>
ord1dict = {'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4}
df_work['ord_1'] = df_work['ord_1'].map(ord1dict )<categorify>
LE = LabelEncoder()
for data in [test,df]:
bins = [-1,0,5,10, 15, 25, 50,100]
labels = ['Unknown','Baby','Child','Young','Teen','Adult','Old']
data['Age'] = pd.cut(data['Age'], bins=bins,labels=labels)
 data['Age'] = data['Age'].astype(str)
 test['Age'] = LE.fit_transform(test['Age'])
 df['Age'] = LE.fit_transform(df['Age'])
ord2dict = {'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot':4, 'Lava Hot':5}
df_work['ord_2'] = df_work['ord_2'].map(ord2dict )<categorify>
for data in [test,df]:
for i,k in enumerate(data['Name']):
x=k.split(",")[1]
data['Name'].replace(data['Name'][i],x.split(" ")[1],inplace=True)
oe = OrdinalEncoder(categories='auto')
 df_work[ordvar[3:]] = oe.fit_transform(df_work[ordvar[3:]])
 for var, cl in zip(ordvar[3:], oe.categories_):
print(var)
 print(cl )<normalization>
df['Name'].value_counts()
df_work[ordvar] = StandardScaler().fit_transform(df_work[ordvar] )<count_unique_values>
all_data = [df,test]
Known = ['Mr.','Miss.','Mrs.','Master.','Ms.','Mlle.','Mme.']
for k in(all_data):
for i,data in enumerate(k['Name']):
if(data)in Known:
if(data=='Mlle.'):
k['Name'] = k['Name'].replace('Mlle.','Miss.')
 elif(data=='Ms.'):
k['Name'] = k['Name'].replace('Ms.','Miss.')
 elif(data=='Mme.'):
k['Name'] = k['Name'].replace('Mme.','Mrs.')
 else:
continue
else:
k['Name'] = k['Name'].replace(data,'not_known')
df_work[nomvar[5:]].nunique()<feature_engineering>
df['Name'][df['Survived']==1].value_counts() /df['Name'].value_counts()
df_work['nom_5'] = df_work['nom_5'].str[4:]
df_work['nom_6'] = df_work['nom_6'].str[3:]
df_work['nom_7'] = df_work['nom_7'].str[3:]
df_work['nom_8'] = df_work['nom_8'].str[3:]
df_work['nom_9'] = df_work['nom_9'].str[3:]<count_unique_values>
test=pd.get_dummies(test,columns=['Embarked','Name'])
 df=pd.get_dummies(df,columns=['Embarked','Name'])
 test['Sex'] = LE.fit_transform(test['Sex'])
 df['Sex'] = LE.fit_transform(df['Sex'] )
df_work[nomvar[5:]].nunique()<categorify>
for data in [df,test]:
data.drop(columns=['Ticket','Cabin','SibSp','Parch','PassengerId'], inplace=True, axis=1 )
enc = OneHotEncoder(categories = 'auto', dtype = 'float64', drop = 'first')
 nom_matrix = enc.fit_transform(df_work[nomvar])
 df_work.drop(nomvar, inplace=True, axis=1 )<categorify>
df.drop(columns=['Embarked_Q'],axis=1,inplace=True)
 test.drop(columns=['Embarked_Q'],axis=1,inplace=True )
enc = OneHotEncoder(categories='auto', dtype = 'float64', drop = 'first')
 dm_matrix = enc.fit_transform(df_work[dmvar])
 df_work.drop(dmvar, inplace=True, axis=1 )<data_type_conversions>
for data in [df,test]:
scale = StandardScaler().fit(data[['Fare']])
 data[['Fare']] = scale.transform(data[['Fare']])
df_work_sprs =scipy.sparse.hstack([nom_matrix,
scipy.sparse.coo_matrix(df_work ).astype('float64'),
dm_matrix] ).tocsr()
display(df_work_sprs )<prepare_x_and_y>
y=df['Survived']
X=df.drop(columns=['Survived'],axis=1)
X_train = df_work_sprs[:y_train.shape[0]]
X_test = df_work_sprs[y_train.shape[0]:]<compute_train_metric>
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3 )
%%time
C = 0.12
clf = LogisticRegression(C=C, solver='lbfgs', max_iter=1000, verbose=0, n_jobs=-1)
 score = cross_validate(clf, X_train, y_train, cv=3, scoring="roc_auc")
 mean = score['test_score'].mean()
print(score['test_score'])
 print('C =', C, f'{mean:.8f}' )<train_model>
from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, VotingClassifier
%%time
clf = LogisticRegression(C=C, solver='lbfgs', max_iter=1000, verbose=0, n_jobs=-1)
 clf.fit(X_train, y_train )<predict_on_test>
parameters_DC = [{'max_depth':[50,100],'min_samples_split': [0.1,0.2,0.5,0.8,0.9]}]
paramaters_RF = [{'max_depth':[2,5,10,15,20,50],'min_samples_split': [0.1,0.2,0.5,0.8],'n_estimators':[100]}]
parameters_XGB =[{"learning_rate": [0.2,0.5,0.8,0.9] ,"max_depth": [1, 3,5, 10], "min_child_weight": [3,5,7,10,20],"gamma": [0.1, 0.2 ,0.4,0.7],'reg_alpha':[0, 0.001, 0.005, 0.01, 0.05],'n_estimator':[100,1000,2000,4000]}]
parameters_GBC =[{"learning_rate": [0.5, 0.25, 0.1, 0.05, 0.01] ,"max_depth": [ 3, 4, 5, 6], "min_samples_leaf" :[50,100,150],"n_estimators" : [16, 32, 64, 128]}]
parameters_ADA =[{'algorithm':['SAMME'],"base_estimator__criterion" : ["gini"],"base_estimator__splitter" : ["best", "random"],"n_estimators": [500,1000],"learning_rate": [ 0.01, 0.1, 1.0]}]
y_preds = clf.predict_proba(X_test)[:,1]<save_to_csv>
DC = DecisionTreeClassifier()
Grid_DC = GridSearchCV(DC, parameters_DC, cv=4,scoring="accuracy", n_jobs= 4,return_train_score=True, verbose = 1)
 Grid_DC.fit(X_train,y_train)
 DC_best = Grid_DC.best_estimator_
Grid_DC.best_score_
sample_submission['target'] = y_preds
sample_submission.to_csv('submission.csv' )<load_from_csv>
RF = RandomForestClassifier()
Grid_RF = GridSearchCV(RF, paramaters_RF, cv=4,scoring="accuracy", n_jobs= 4,return_train_score=True, verbose = 1)
 Grid_RF.fit(X_train,y_train)
 RF_best = Grid_RF.best_estimator_
Grid_RF.best_score_
%%time
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape )<import_modules>
XGB = XGBClassifier()
Grid_XGB = GridSearchCV(XGB, parameters_XGB, cv=4,scoring="accuracy", n_jobs= 4,return_train_score=True, verbose = 1)
 Grid_XGB.fit(X_train,y_train)
 XGB_best = Grid_XGB.best_estimator_
Grid_XGB.best_score_
from sklearn.model_selection import StratifiedKFold<categorify>
GBC = GradientBoostingClassifier()
Grid_GBC = GridSearchCV(GBC,parameters_GBC, cv=4, scoring="accuracy", n_jobs= 4, return_train_score=True,verbose = 1)
 Grid_GBC.fit(X_train,y_train)
 GBC_best = Grid_GBC.best_estimator_
Grid_GBC.best_score_
<drop_column>
ADA = AdaBoostClassifier(DC_best)
 Grid_ADA = GridSearchCV(ADA,parameters_ADA, cv=4, scoring="accuracy", n_jobs= 4, return_train_score=True,verbose = 1)
 Grid_ADA.fit(X_train,y_train)
 ADA_best = Grid_ADA.best_estimator_
Grid_ADA.best_score_
%%time
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 print(train.shape)
 print(test.shape )<categorify>
parameters_SVM = {'C': [0.1, 1, 10,50,100], 'gamma' : [0.001, 0.01, 0.1, 1,10]}
%%time
traintest = pd.concat([train, test])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe = dummies.iloc[:train.shape[0], :]
test_ohe = dummies.iloc[train.shape[0]:, :]
print(train_ohe.shape)
 print(test_ohe.shape )<categorify>
SVMC =SVC(probability=True)
 Grid_SVC = GridSearchCV(SVMC, parameters_SVM, scoring="accuracy", return_train_score=True,verbose = 1,cv=2)
 Grid_SVC.fit(X_train, y_train)
 SVM_best = Grid_SVC.best_estimator_
Grid_SVC.best_score_
cat_feat_to_encode = train.columns.tolist()
smoothing=50.0
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(
 n_splits=5, random_state=1, shuffle=True ).split(
 train, target):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], target.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, target)
 train_enc = oof.sort_index()
test_enc = ce_target_encoder.transform(test )<import_modules>
voting = VotingClassifier(estimators=[('ADA', ADA_best),('DC', DC_best),('RF', RF_best),('GBC',GBC_best),('XGB',XGB_best),('SVC',SVM_best)],weights=[3,0,0,1,3,3], voting='hard', n_jobs=4)
 voting_result = voting.fit(X_train, y_train )
scipy.sparse.hstack<concatenate>
voting.score(X_test,y_test )
train_merge = scipy.sparse.hstack([train_ohe, train_enc.values] ).tocsr()
test_merge = scipy.sparse.hstack([test_ohe, test_enc.values] ).tocsr()<train_on_grid>
pred = voting.predict(test )
%%time
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = KFold(n_splits=10)
 fold_splits = kf.split(train, target)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0]))
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/10')
 dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 print(label + ' cv score {}: {}'.format(i, cv_score))
i += 1
print('{} cv scores : {}'.format(label, cv_scores))
print('{} cv mean score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std score : {}'.format(label, np.std(cv_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores}
return results
def runLR(train_X, train_y, test_X, test_y, test_X2, params):
print('Train LR')
 model = LogisticRegression(**params)
 model.fit(train_X, train_y)
 print('Predict 1/2')
 pred_test_y = model.predict_proba(test_X)[:, 1]
print('Predict 2/2')
 pred_test_y2 = model.predict_proba(test_X2)[:, 1]
return pred_test_y, pred_test_y2
def runCat(train_X, train_y, test_X, test_y, test_X2, params):
print('Train LR')
 model = CatBoostClassifier(**params)
 model.fit(train_X, train_y)
 print('Predict 1/2')
 pred_test_y = model.predict_proba(test_X)[:, 1]
print('Predict 2/2')
 pred_test_y2 = model.predict_proba(test_X2)[:, 1]
return pred_test_y, pred_test_y2
def runLgb(train_X, train_y, test_X, test_y, test_X2, params):
model = lgb.train(
 params={
'max_depth': 3,
'num_leaves': 150,
'reg_alpha': 0.6,
'reg_lambda': 0.6,
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
"verbosity": -1,
'random_state': 1,
'lr': 0.01
},
train_set=lgb.Dataset(train_X, label=train_y),
num_boost_round=700)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2)
 return pred_test_y, pred_test_y2
model = CatBoostClassifier(learning_rate=0.006, iterations=1000, thread_count=32,
eval_metric='Accuracy')
 lr_params = {'solver': 'lbfgs', 'C': 0.1, "max_iter":500, 'thread_count':32, "eval_metric":"AUC"}
cat_params = {"iterations":500, 'learning_rate': 0.006, 'thread_count':32, 'eval_metric':'AUC'}
results = run_cv_model(train_merge, test_merge, target, runLgb, {}, auc, 'lgb' )<save_to_csv>
test_2 = pd.read_csv('.. /input/test.csv' , header = 0, dtype={'Age': np.float64} )
submission = pd.DataFrame({'id': test_id, 'target': results['test']})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
result = pd.DataFrame(pred,columns=['Survived'])
 submission13 = result.join(test_2['PassengerId'] ).iloc[:,::-1]
<rename_columns><EOS> 
submission13.to_csv('submission13.csv', index=False )