Kaggle competition: Titanic - Machine Learning from Disaster
train['Pure_breed'] = 0
train.loc[train['Breed2'] == 0, 'Pure_breed'] = 1
test['Pure_breed'] = 0
test.loc[test['Breed2'] == 0, 'Pure_breed'] = 1
all_data['Pure_breed'] = 0
all_data.loc[all_data['Breed2'] == 0, 'Pure_breed'] = 1
print(f"Rate of pure breed pets in train data: {train['Pure_breed'].sum() * 100 / train['Pure_breed'].shape[0]:.4f}%.")
 print(f"Rate of pure breed pets in test data: {test['Pure_breed'].sum() * 100 / test['Pure_breed'].shape[0]:.4f}%." )<define_variables>
%pylab inline
plt.style.use('seaborn-darkgrid')
 sns.set(font_scale=2)
 warnings.filterwarnings(action="ignore")
 n_arbres = 200
max_depth = 6
noms = [
"Random_Forest",
"Ada_Boost",
"Gradient_Boosting",
"LightGBM",
"XGBoost",
"CatBoost"
]
classifieurs = [
RandomForestClassifier(n_estimators=n_arbres,max_depth=max_depth),
AdaBoostClassifier(n_estimators=n_arbres),
GradientBoostingClassifier(n_estimators=n_arbres,max_depth=max_depth, max_leaf_nodes=4,min_samples_split=5),
XGBClassifier(n_estimators=n_arbres,max_depth=max_depth),
LGBMClassifier(n_estimators=n_arbres,max_depth=max_depth),
CatBoostClassifier(n_estimators=n_arbres,silent=True)
 ]
classifieursArbresDict = {nom:classifieur for nom, classifieur in zip(noms, classifieurs)}
def comparaisonsClassifieurs(classifieursDict, X_train, X_test, y_train, y_test):
t0 = time.time()
np.random.seed(123456)
 aucROC,accuracy,logloss,hammingloss,precision,sensibilite,f1,jaccard_similarity,avgPrecRec = \
dict() ,dict() ,dict() ,dict() ,dict() ,dict() ,dict() ,dict() ,dict()
yClassifications = pd.DataFrame()
yClassifications['Observations']=y_test
plt.figure(figsize=(18,36))
for nom in classifieursDict:
t1 = time.time()
classifieursDict[nom].fit(X_train, y_train)
 score = classifieursDict[nom].score(X_test, y_test)*100
print(f'Prédiction : {nom:32s}'+(' %.8f' % score ).lstrip('0'),end=' -- ')
 yClassifications[nom+'_prob'] = classifieursDict[nom].predict_proba(X_test)[:, 1]
yClassifications[nom+'_pred'] = classifieursDict[nom].predict(X_test)
 fpr, tpr, thresholds = roc_curve(y_test.ravel() , yClassifications[nom+'_prob'])
 accuracy[nom] = accuracy_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 logloss[nom] = log_loss(yClassifications['Observations'],yClassifications[nom+'_pred'])
 hammingloss[nom] = hamming_loss(yClassifications['Observations'],yClassifications[nom+'_pred'])
 precision[nom] = precision_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 sensibilite[nom] = recall_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 f1[nom] = f1_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 jaccard_similarity[nom] = jaccard_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 aucROC[nom] = auc(fpr, tpr)
 plt.subplot(2, 1, 1)
 print("Area under the ROC curve : %0.8f" % aucROC[nom],end=' -- ')
 plt.plot(fpr, tpr, label=f"{nom}(AUC = {aucROC[nom]:0.8f})")
 plt.plot([0, 1], [0, 1], 'k--')
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('(1 - Specificity)--La Spécificité(Specificity)= VN /(FP + VN)',size=18)
 plt.ylabel('La Sensibilité(Sensitivity)= VP /(VP + FN)',size=18)
 plt.title('La courbe ROC(Receiver Operating Caracteristic)',size=20)
 plt.legend(loc="lower right")
 prec, rec, thresholds = precision_recall_curve(y_test.ravel() , yClassifications[nom+'_prob'])
 avgPrecRec[nom] = average_precision_score(y_test.ravel() , yClassifications[nom+'_prob'])
 plt.subplot(2, 1, 2)
 plt.step(rec, prec, where='post', label=f"{nom}(AP = {avgPrecRec[nom]:0.8f})")
 plt.fill_between(rec, prec, step='post', alpha=0.05)
 plt.plot([0, 1], [0.5, 0.5], 'k--')
 plt.xlabel('Le Rappel(Recall)= VP /(VP + FN)',size=18)
 plt.ylabel('La Précision = VP /(VP + FP)',size=18)
 plt.title('La courbe précision-rappel',size=20)
 plt.legend(loc="lower right")
 print(( '%0.2fs' %(time.time() - t1)).lstrip('0'))
plt.show()
print('Exécution :'+('%.2fs' %(time.time() - t0)).lstrip('0'))
resultats = pd.DataFrame(pd.Series(aucROC),columns=["aucROC"])
 resultats["avgPrecRec"] = pd.Series(avgPrecRec)
 resultats["accuracy"] = pd.Series(accuracy)
 resultats["log_loss"] = pd.Series(logloss)
 resultats["hamming_loss"] = pd.Series(hammingloss)
 resultats["precision"] = pd.Series(precision)
 resultats["sensibilite"] = pd.Series(sensibilite)
 resultats["f1"] = pd.Series(f1)
 resultats["jaccard_similarity"] = pd.Series(jaccard_similarity)
 resultats.sort_values(by='aucROC',ascending=False, inplace=True)
 return resultats
precision_score, recall_score, f1_score, jaccard_score, \
precision_recall_curve, average_precision_score
def controleClassifieur(classifieur, X_train, X_test, y_train, y_test, nom = 'CatBoost'):
t0 = time.time()
np.random.seed(123456)
 aucROC,accuracy,logloss,hammingloss,precision,sensibilite,f1,jaccard_similarity,avgPrecRec = \
dict() ,dict() ,dict() ,dict() ,dict() ,dict() ,dict() ,dict() ,dict()
yClassifications = pd.DataFrame()
yClassifications['Observations']=y_test
plt.figure(figsize=(18,36))
t1 = time.time()
classifieur.fit(X_train, y_train)
 score = classifieur.score(X_test, y_test)*100
print(f'Prédiction : {nom:32s}'+(' %.8f' % score ).lstrip('0'),end=' -- ')
 yClassifications[nom+'_prob'] = classifieur.predict_proba(X_test)[:, 1]
yClassifications[nom+'_pred'] = classifieur.predict(X_test)
 fpr, tpr, thresholds = roc_curve(y_test.ravel() , yClassifications[nom+'_prob'])
 accuracy[nom] = accuracy_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 logloss[nom] = log_loss(yClassifications['Observations'],yClassifications[nom+'_pred'])
 hammingloss[nom] = hamming_loss(yClassifications['Observations'],yClassifications[nom+'_pred'])
 precision[nom] = precision_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 sensibilite[nom] = recall_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 f1[nom] = f1_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 jaccard_similarity[nom] = jaccard_score(yClassifications['Observations'],yClassifications[nom+'_pred'])
 aucROC[nom] = auc(fpr, tpr)
 plt.subplot(2, 1, 1)
 print("Area under the ROC curve : %0.8f" % aucROC[nom],end=' -- ')
 plt.plot(fpr, tpr, label=f"{nom}(AUC = {aucROC[nom]:0.8f})")
 nb_decimales = 2
angleText = 36
transparence=0.6
for i,t in enumerate(thresholds*100):
if i > 0 :
plt.text(fpr[i], tpr[i],str(t.round(nb_decimales)) , rotation= angleText, alpha=transparence)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('(1 - Specificity)--La Spécificité(Specificity)= VN /(FP + VN)',size=18)
 plt.ylabel('La Sensibilité(Sensitivity)= VP /(VP + FN)',size=18)
 plt.title('La courbe ROC(Receiver Operating Caracteristic)',size=20)
 plt.legend(loc="lower right")
 prec, rec, thresholds = precision_recall_curve(y_test.ravel() , yClassifications[nom+'_prob'])
 avgPrecRec[nom] = average_precision_score(y_test.ravel() , yClassifications[nom+'_prob'])
 plt.subplot(2, 1, 2)
 plt.step(rec, prec, where='post', label=f"{nom}(AP = {avgPrecRec[nom]:0.8f})")
 plt.fill_between(rec, prec, step='post', alpha=0.05)
 plt.plot([0, 1], [0.5, 0.5], 'k--')
 plt.xlabel('Le Rappel(Recall)= VP /(VP + FN)',size=18)
 plt.ylabel('La Précision = VP /(VP + FP)',size=18)
 plt.title('La courbe précision-rappel',size=20)
 plt.legend(loc="lower right")
 print(( '%0.2fs' %(time.time() - t1)).lstrip('0'))
plt.show()
print('Exécution :'+('%.2fs' %(time.time() - t0)).lstrip('0'))
resultats = pd.DataFrame(pd.Series(aucROC),columns=["aucROC"])
 resultats["avgPrecRec"] = pd.Series(avgPrecRec)
 resultats["accuracy"] = pd.Series(accuracy)
 resultats["log_loss"] = pd.Series(logloss)
 resultats["hamming_loss"] = pd.Series(hammingloss)
 resultats["precision"] = pd.Series(precision)
 resultats["sensibilite"] = pd.Series(sensibilite)
 resultats["f1"] = pd.Series(f1)
 resultats["jaccard_similarity"] = pd.Series(jaccard_similarity)
 resultats.sort_values(by='aucROC',ascending=False, inplace=True)
 return resultats
def afficheImportanceVariables(classifieurs, X, nom = 'CatBoost'):
importances = pd.DataFrame(X.columns, columns=['nom'])
 importances['importance'] = classifieurs.feature_importances_
importances.sort_values(by='importance',ascending=False, inplace=True)
 sns_plot = sns.catplot(x='nom', y='importance',data=importances, kind='bar', size=18, aspect=4);
loc, labels = plt.xticks()
sns_plot.set_xticklabels(labels, rotation=90);
plt.show()
return importances
def effectueValidationCroisee(classifieur,
X,
y,
n_splits = 5,
nom = 'CatBoost'):
moyAucROC,stdAucRoc,moyAucPR,stdAucPR,moyAccuracy,stdAccuracy = \
dict() ,dict() ,dict() ,dict() ,dict() ,dict()
validationCroisee = StratifiedShuffleSplit(n_splits=n_splits)
 resultatsFinaux = pd.DataFrame()
classifieursCV={}
tprs = []
recs = []
aucROCs = pd.Series()
aucPRs = pd.Series()
scores = pd.Series()
mean_fpr = np.linspace(0, 1, 100)
 mean_prec = np.linspace(0, 1, 100)
 aucROC,avgPrecRec,accuracy,logloss,hammingloss,precision,sensibilite,f1,jaccard_similarity= \
dict() ,dict() ,dict() ,dict() ,dict() ,dict() ,dict() ,dict() ,dict()
plt.figure(figsize=(24,48))
for i,(train, test)in enumerate(validationCroisee.split(X, y)) :
X_train, X_test, y_train, y_test = X.iloc[train], X.iloc[test], y.iloc[train], y.iloc[test]
classifieur.fit(X_train, y_train)
 classifieursCV[i] = deepcopy(classifieur)
 scores = pd.concat([scores,pd.Series(classifieur.score(X_test, y_test)*100)])
 yClassifications_prob = classifieur.predict_proba(X_test)[:, 1]
yClassifications_pred = classifieur.predict(X_test)
 fpr, tpr, thresholds = roc_curve(y_test, yClassifications_prob)
 prec, rec, thresholds = precision_recall_curve(y_test, yClassifications_prob)
 avgPrecRec[(nom,i)] = average_precision_score(y_test, yClassifications_prob)
 accuracy[(nom,i)] = accuracy_score(y_test,yClassifications_pred)
 logloss[(nom,i)] = log_loss(y_test,yClassifications_pred)
 hammingloss[(nom,i)] = hamming_loss(y_test,yClassifications_pred)
 precision[(nom,i)] = precision_score(y_test,yClassifications_pred)
 sensibilite[(nom,i)] = recall_score(y_test,yClassifications_pred)
 f1[(nom,i)] = f1_score(y_test,yClassifications_pred)
 jaccard_similarity[(nom,i)] = jaccard_score(y_test,yClassifications_pred)
 aucROC[(nom,i)] = auc(fpr, tpr)
 tprs.append(interp(mean_fpr, fpr, tpr))
tprs[-1][0] = 0.0
recs.append(interp(mean_prec, prec, rec))
recs[-1][0] = 0.0
aucROCs = pd.concat([aucROCs,pd.Series(aucROC[(nom,i)])])
 aucPRs = pd.concat([aucPRs,pd.Series(avgPrecRec[(nom,i)])])
 plt.subplot(2, 1, 1)
 plt.plot(fpr, tpr, lw=1, alpha=0.3,
label='Segment %2d(AUC = %0.6f Accuracy = %0.6f)' % 
(i, aucROC[(nom,i)],accuracy[(nom,i)]))
plt.subplot(2, 1, 2)
 plt.step(rec, prec, where='post',
label=f"Segment {i:2d}(AP = {avgPrecRec[(nom,i)]:0.8f})")
 plt.fill_between(rec, prec, step='post', alpha=0.05)
 plt.subplot(2, 1, 1)
 plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',
label='AUC <= 0.5', alpha=.8)
 mean_tpr = np.mean(tprs, axis=0)
 mean_tpr[-1] = 1.0
mean_aucROC = auc(mean_fpr, mean_tpr)
 std_aucROC = np.std(aucROCs)
 moyAucROC[nom], stdAucRoc[nom],moyAucPR[nom], stdAucPR[nom], moyAccuracy[nom], stdAccuracy[nom] = \
aucROCs.mean() ,aucROCs.std() ,aucPRs.mean() ,aucPRs.std() ,scores.mean() ,scores.std()
plt.subplot(2, 1, 1)
 plt.plot(mean_fpr, mean_tpr, color='b',
label=r'Moyenne(AUC = %0.5f $\pm$ %0.5f Accuracy = %0.4f $\pm$ %0.4f)' % 
(mean_aucROC, std_aucROC, scores.mean() , scores.std()),
lw=2, alpha=.8)
 std_tpr = np.std(tprs, axis=0)
 tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
 tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
 plt.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.4,
label=r'$\pm$ %0.8f std.dev.'%(np.mean(std_tpr)))
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('Le taux de faux Positifs(1 - Specificity)',size=18)
 plt.ylabel('Le taux de vrais positifs(Sensitivity)',size=18)
 plt.title('La courbe ROC '+nom,size=20)
 plt.legend(loc="lower right")
 plt.subplot(2, 1, 2)
 plt.plot([0, 1], [0.5, 0.5], linestyle='--', lw=2, color='r',
label='AC <= 0.5', alpha=.8)
 plt.xlabel('Le Rappel(Recall)= VP /(VP + FN)',size=18)
 plt.ylabel('La Précision = VP /(VP + FP)',size=18)
 plt.title('La courbe précision-rappel '+nom,size=20)
 plt.legend(loc="lower right")
 plt.show()
resultats = pd.DataFrame(pd.Series(aucROC),columns=["aucROC"])
 resultats["avgPrecRec"] = pd.Series(avgPrecRec)
 resultats["accuracy"] = pd.Series(accuracy)
 resultats["log_loss"] = pd.Series(logloss)
 resultats["hamming_loss"] = pd.Series(hammingloss)
 resultats["precision"] = pd.Series(precision)
 resultats["sensibilite"] = pd.Series(sensibilite)
 resultats["f1"] = pd.Series(f1)
 resultats["jaccard_similarity"] = pd.Series(jaccard_similarity)
 resultatsFinaux = pd.concat([resultatsFinaux,resultats])
 return resultatsFinaux,classifieursCV
breeds_dict = {k: v for k, v in zip(breeds['BreedID'], breeds['BreedName'])}<feature_engineering>
train=pd.read_csv(".. /input/titanic/train.csv")
 print(train.shape)
 train.head()
train['Breed1_name'] = train['Breed1'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else 'Unknown')
 train['Breed2_name'] = train['Breed2'].apply(lambda x: '_'.join(breeds_dict[x])if x in breeds_dict else '-')
 test['Breed1_name'] = test['Breed1'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else 'Unknown')
 test['Breed2_name'] = test['Breed2'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else '-')
 all_data['Breed1_name'] = all_data['Breed1'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else 'Unknown')
 all_data['Breed2_name'] = all_data['Breed2'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else '-' )<count_values>
test=pd.read_csv(".. /input/titanic/test.csv")
 print(test.shape)
 test.head()
( all_data['Breed1_name'] + '__' + all_data['Breed2_name'] ).value_counts().head(15 )<feature_engineering>
donnees = pd.concat([train,test],sort=False)
 donnees.head()
colors_dict = {k: v for k, v in zip(colors['ColorID'], colors['ColorName'])}
train['Color1_name'] = train['Color1'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 train['Color2_name'] = train['Color2'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 train['Color3_name'] = train['Color3'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 test['Color1_name'] = test['Color1'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 test['Color2_name'] = test['Color2'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 test['Color3_name'] = test['Color3'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 all_data['Color1_name'] = all_data['Color1'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 all_data['Color2_name'] = all_data['Color2'].apply(lambda x: colors_dict[x] if x in colors_dict else '')
 all_data['Color3_name'] = all_data['Color3'].apply(lambda x: colors_dict[x] if x in colors_dict else '' )<count_unique_values>
donnees['Title'] = donnees.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(donnees['Title'], donnees['Sex'] )
gender_dict = {1: 'Male', 2: 'Female', 3: 'Mixed'}
for i in all_data['Type'].unique() :
for j in all_data['Gender'].unique() :
df = all_data.loc[(all_data['Type'] == i)&(all_data['Gender'] == j)]
top_colors = list(df['full_color'].value_counts().index)[:5]
j = gender_dict[j]
print(f"Most popular colors of {j} {i}s: {' '.join(top_colors)}" )<count_unique_values>
donnees['Title'] = donnees['Title'].replace(['Capt','Col','Major','Dr','Rev'], 'Autres')
 donnees['Title'] = donnees['Title'].replace(['Lady', 'Countess', 'Don', 'Sir', 'Jonkheer', 'Dona'], 'Noblesse')
 donnees['Title'] = donnees['Title'].replace('Mlle', 'Miss')
 donnees['Title'] = donnees['Title'].replace('Ms', 'Miss')
 donnees['Title'] = donnees['Title'].replace('Mme', 'Mrs')
 donnees['Sex'] = donnees['Sex'].replace("male","Homme")
 donnees['Sex'] = donnees['Sex'].replace("female","Femme")
 pd.crosstab(donnees['Title'], donnees['Sex'] )
images = [i.split('-')[0] for i in os.listdir('.. /input/train_images/')]
size_dict = {1: 'Small', 2: 'Medium', 3: 'Large', 4: 'Extra Large'}
for t in all_data['Type'].unique() :
for m in all_data['MaturitySize'].unique() :
df = all_data.loc[(all_data['Type'] == t)&(all_data['MaturitySize'] == m)]
top_breeds = list(df['Breed1_name'].value_counts().index)[:5]
m = size_dict[m]
print(f"Most common Breeds of {m} {t}s:")
 fig = plt.figure(figsize=(25, 4))
for i, breed in enumerate(top_breeds):
b_df = df.loc[(df['Breed1_name'] == breed)&(df['PetID'].isin(images)) , 'PetID']
if len(b_df)> 1:
pet_id = b_df.values[1]
else:
pet_id = b_df.values[0]
ax = fig.add_subplot(1, 5, i+1, xticks=[], yticks=[])
 im = Image.open(".. /input/train_images/" + pet_id + '-1.jpg')
 plt.imshow(im)
 ax.set_title(f'Breed: {breed}')
 plt.show() ;<define_variables>
donnees.Name = donnees.Name.str.extract('([A-Za-z]+)\,', expand=False)
 donnees.head()
c = 0
strange_pets = []
for i, row in all_data[all_data['Breed1_name'].str.contains('air')].iterrows() :
if 'Short' in row['Breed1_name'] and row['FurLength'] == 1:
pass
elif 'Medium' in row['Breed1_name'] and row['FurLength'] == 2:
pass
elif 'Long' in row['Breed1_name'] and row['FurLength'] == 3:
pass
else:
c += 1
strange_pets.append(( row['PetID'], row['Breed1_name'], row['FurLength']))
print(f"There are {c} pets whose breed and fur length don't match" )<data_type_conversions>
donnees['TailleFamille'] = donnees['Parch'] + donnees['SibSp'] + 1
donnees.TailleFamille = donnees.TailleFamille.astype('int8' )
train['health'] = train['Vaccinated'].astype(str)+ '_' + train['Dewormed'].astype(str)+ '_' + train['Sterilized'].astype(str)+ '_' + train['Health'].astype(str)
 test['health'] = test['Vaccinated'].astype(str)+ '_' + test['Dewormed'].astype(str)+ '_' + test['Sterilized'].astype(str)+ '_' + test['Health'].astype(str)
 make_factor_plot(df=train.loc[train['health'].isin(list(train.health.value_counts().index[:5])) ], x='health', col='AdoptionSpeed', title='Counts of pets by main health conditions and Adoption Speed' )<count_values>
donnees['Pont'] = donnees.Cabin.str.extract('([A-Za-z])', expand=False)
 donnees.Pont = donnees.Pont.fillna('Na')
 pd.crosstab(donnees.Pont, np.ones(donnees.shape[0]))
train['Quantity'].value_counts().head(10 )<sort_values>
donnees['TicketNum'] = donnees.Ticket.replace(regex=r'([^0-9]+)',value='')
 donnees.Ticket = donnees.Ticket.replace(regex=r'([^a-zA-Z]+)',value='')
 donnees.Ticket = donnees.Ticket.replace({r'^(CASOTON|SOTONO|STONO|STONOQ)$':'SOTONOQ',
r'^(SC|SCParis)$':'SCPARIS',
r'^FCC$':'FC',
r'^$':'Vide'}, regex=True )
all_data.sort_values('Fee', ascending=False)[['Name', 'Description', 'Fee', 'AdoptionSpeed', 'dataset_type']].head(10 )<feature_engineering>
donnees.rename(columns={"Pclass": "Classe",
"Embarked": "Port",
"Cabin": "Cabine",
"SibSp": "ConjointsOuFratrie",
"Parch": "EnfantsOuParents",
},
inplace=True )
states_dict = {k: v for k, v in zip(states['StateID'], states['StateName'])}
train['State_name'] = train['State'].apply(lambda x: '_'.join(states_dict[x].split())if x in states_dict else 'Unknown')
 test['State_name'] = test['State'].apply(lambda x: '_'.join(states_dict[x].split())if x in states_dict else 'Unknown')
 all_data['State_name'] = all_data['State'].apply(lambda x: '_'.join(states_dict[x].split())if x in states_dict else 'Unknown' )<count_values>
donnees.Port = donnees.Port.fillna('Pas')
 donnees.Cabine = donnees.Cabine.apply(lambda x: 0 if type(x)== float else 1)
 donnees.head()
all_data['State_name'].value_counts(normalize=True ).head()<count_values>
donnees.isnull().sum()
all_data['RescuerID'].value_counts().head()<count_values>
donnees[donnees['Fare'].isnull() ]
train['VideoAmt'].value_counts()<count_values>
donnees[(donnees['Port'] == 'S')&(donnees['Classe'] == 3)].Fare.median()
print(F'Maximum amount of photos in {train["PhotoAmt"].max() }')
 train['PhotoAmt'].value_counts().head()<train_on_grid>
coefficient = 1.2
ageCalc = donnees[~donnees.Age.isna() ].groupby(['Sex','Classe'] ).agg({'Age':['mean','std']})
 ageCalc.columns = ['_'.join(col ).rstrip('_')for col in ageCalc.columns]
ageCalc.reset_index(inplace=True)
 ageCalc['borneMin'] = ageCalc.Age_mean - ageCalc.Age_std*coefficient
ageCalc['borneMax'] = ageCalc.Age_mean + ageCalc.Age_std*coefficient
ageCalc.drop(columns=['Age_mean','Age_std'],inplace=True)
 ageCalcCount = donnees[donnees.Age.isna() ].groupby(['Sex','Classe'] ).agg({'Title':['count']} ).reset_index()
ageCalcCount.columns = ['Sex','Classe','nb']
ageCalc = ageCalc.merge(ageCalcCount,on=['Sex','Classe'])
 ageCalc
tokenizer = TweetTokenizer()
vectorizer = TfidfVectorizer(ngram_range=(1, 2), tokenizer=tokenizer.tokenize)
 vectorizer.fit(all_data['Description'].fillna('' ).values)
 X_train = vectorizer.transform(train['Description'].fillna(''))
rf = RandomForestClassifier(n_estimators=20)
 rf.fit(X_train, train['AdoptionSpeed'] )<feature_engineering>
ageRand = pd.DataFrame(columns=['Sex','Classe','Age'])
 for i in [(row.Sex,row.Classe,np.random.randint(round(row.borneMin),round(row.borneMax),size=row.nb))
for indx, row in ageCalc.iterrows() ]:
calc = pd.DataFrame(columns=['Sex','Classe','Age'])
 calc.Age = i[2]
calc.Sex = i[0]
calc.Classe = i[1]
ageRand = pd.concat([ageRand,calc] )
train['Description'] = train['Description'].fillna('')
 test['Description'] = test['Description'].fillna('')
 all_data['Description'] = all_data['Description'].fillna('')
 train['desc_length'] = train['Description'].apply(lambda x: len(x))
train['desc_words'] = train['Description'].apply(lambda x: len(x.split()))
test['desc_length'] = test['Description'].apply(lambda x: len(x))
test['desc_words'] = test['Description'].apply(lambda x: len(x.split()))
all_data['desc_length'] = all_data['Description'].apply(lambda x: len(x))
all_data['desc_words'] = all_data['Description'].apply(lambda x: len(x.split()))
train['averate_word_length'] = train['desc_length'] / train['desc_words']
test['averate_word_length'] = test['desc_length'] / test['desc_words']
all_data['averate_word_length'] = all_data['desc_length'] / all_data['desc_words']<feature_engineering>
donnees.Age01.isna().sum()
sentiment_dict = {}
for filename in os.listdir('.. /input/train_sentiment/'):
with open('.. /input/train_sentiment/' + filename, 'r')as f:
sentiment = json.load(f)
 pet_id = filename.split('.')[0]
sentiment_dict[pet_id] = {}
sentiment_dict[pet_id]['magnitude'] = sentiment['documentSentiment']['magnitude']
sentiment_dict[pet_id]['score'] = sentiment['documentSentiment']['score']
sentiment_dict[pet_id]['language'] = sentiment['language']
for filename in os.listdir('.. /input/test_sentiment/'):
with open('.. /input/test_sentiment/' + filename, 'r')as f:
sentiment = json.load(f)
 pet_id = filename.split('.')[0]
sentiment_dict[pet_id] = {}
sentiment_dict[pet_id]['magnitude'] = sentiment['documentSentiment']['magnitude']
sentiment_dict[pet_id]['score'] = sentiment['documentSentiment']['score']
sentiment_dict[pet_id]['language'] = sentiment['language']<feature_engineering>
donnees.Age = donnees.Age01.values
donnees.drop(columns=['AgeOld','Age01'],inplace=True )
train['lang'] = train['PetID'].apply(lambda x: sentiment_dict[x]['language'] if x in sentiment_dict else 'no')
 train['magnitude'] = train['PetID'].apply(lambda x: sentiment_dict[x]['magnitude'] if x in sentiment_dict else 0)
 train['score'] = train['PetID'].apply(lambda x: sentiment_dict[x]['score'] if x in sentiment_dict else 0)
 test['lang'] = test['PetID'].apply(lambda x: sentiment_dict[x]['language'] if x in sentiment_dict else 'no')
 test['magnitude'] = test['PetID'].apply(lambda x: sentiment_dict[x]['magnitude'] if x in sentiment_dict else 0)
 test['score'] = test['PetID'].apply(lambda x: sentiment_dict[x]['score'] if x in sentiment_dict else 0)
 all_data['lang'] = all_data['PetID'].apply(lambda x: sentiment_dict[x]['language'] if x in sentiment_dict else 'no')
 all_data['magnitude'] = all_data['PetID'].apply(lambda x: sentiment_dict[x]['magnitude'] if x in sentiment_dict else 0)
 all_data['score'] = all_data['PetID'].apply(lambda x: sentiment_dict[x]['score'] if x in sentiment_dict else 0 )<drop_column>
listeVariblesInitiales = donnees.drop(columns=['Name'] ).columns
donnees = donnees.set_index('PassengerId' ).sort_index()
cols_to_use = ['Type', 'Age', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State', 'RescuerID', 'health', 'Free', 'score',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'No_name', 'Pure_breed', 'desc_length', 'desc_words', 'averate_word_length', 'magnitude']
train = train[[col for col in cols_to_use if col in train.columns]]
test = test[[col for col in cols_to_use if col in test.columns]]<define_variables>
donnees['TitreFamille'] = donnees.apply(lambda ligne : 'Homme' if ligne['Title'] == 'Mr'
else 'Femme' if ligne['Sex'] == 'Femme'
else 'Garçon' if ligne['Title'] == 'Master'
else 'Homme' , axis=1)
 plt.figure(figsize=(14,12))
plt.title('La distribution du titre calculé pour le groupe famille',size=20)
 sns.countplot(x='TitreFamille',hue='Survived',data=donnees);
cat_cols = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'State', 'RescuerID',
'No_name', 'Pure_breed', 'health', 'Free']<data_type_conversions>
donnees['GroupFamille'] = donnees.Name+'-'+\
donnees.Classe.apply(lambda x: '%1d' % x)+'-'+\
donnees.Fare.apply(lambda x: '%.3f' % x)+'-'+\
donnees.Port+'-'+donnees.TicketNum
donnees.GroupFamille.unique() [:6]
more_cols = []
for col1 in cat_cols:
for col2 in cat_cols:
if col1 != col2 and col1 not in ['RescuerID', 'State'] and col2 not in ['RescuerID', 'State']:
train[col1 + '_' + col2] = train[col1].astype(str)+ '_' + train[col2].astype(str)
 test[col1 + '_' + col2] = test[col1].astype(str)+ '_' + test[col2].astype(str)
 more_cols.append(col1 + '_' + col2)
 cat_cols = cat_cols + more_cols<data_type_conversions>
donnees['GroupTicket'] = donnees.Classe.apply(lambda x: '%1d' % x)+'-'+\
donnees.Fare.apply(lambda x: '%.3f' % x)+'-'+\
donnees.Port+'-'+donnees.TicketNum
donnees.GroupTicket.unique() [:6]
%%time
indexer = {}
for col in cat_cols:
_, indexer[col] = pd.factorize(train[col].astype(str))
for col in tqdm_notebook(cat_cols):
train[col] = indexer[col].get_indexer(train[col].astype(str))
test[col] = indexer[col].get_indexer(test[col].astype(str)) 
<prepare_x_and_y>
donnees.drop(columns=['Name','TitreFamille','GroupFamille'], inplace=True)
 listeVariblesAvecGroups = donnees.columns
y = train['AdoptionSpeed']
train = train.drop(['AdoptionSpeed'], axis=1 )<choose_model_class>
def conversionVariableCategorielle(donnees,variable):
valeurs = list(donnees[variable].sort_values().unique())
 dicoVar = {nom:indx for indx,nom in enumerate(valeurs)}
dicoVarRev = {indx:nom for indx,nom in enumerate(valeurs)}
donnees[variable] = donnees[variable].apply(lambda x : dicoVar[x])
 return dicoVar,dicoVarRev
for col in donnees.dtypes[donnees.dtypes == 'object'].reset_index() ['index']:
_,_ = conversionVariableCategorielle(donnees,col)
 donnees.head(10 )
n_fold = 5
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=15 )<split>
apprentissage = donnees[~donnees.Survived.isnull() ]
apprentissage.Survived = apprentissage.Survived.astype('int8')
 X = apprentissage.drop(columns='Survived')
 y = apprentissage.Survived
apprentissage.head()
def train_model(X=train, X_test=test, y=y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, averaging='usual', make_oof=False):
result_dict = {}
if make_oof:
oof = np.zeros(( len(X), 5))
prediction = np.zeros(( len(X_test), 5))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
gc.collect()
print('Fold', fold_n + 1, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
train_data = lgb.Dataset(X_train, label=y_train, categorical_feature = cat_cols)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature = cat_cols)
 model = lgb.train(params,
train_data,
num_boost_round=20000,
valid_sets = [train_data, valid_data],
verbose_eval=500,
early_stopping_rounds = 200)
 del train_data, valid_data
y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 del X_valid
gc.collect()
y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test), ntree_limit=model.best_ntree_limit)
 if model_type == 'lcv':
model = LogisticRegressionCV(scoring='neg_log_loss', cv=3, multi_class='multinomial')
 model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=20000, loss_function='MultiClass', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test ).reshape(-1,)
 if make_oof:
oof[valid_index] = y_pred_valid
scores.append(kappa(y_valid, y_pred_valid.argmax(1)))
 print('Fold kappa:', kappa(y_valid, y_pred_valid.argmax(1)))
 print('')
 if averaging == 'usual':
prediction += y_pred
elif averaging == 'rank':
prediction += pd.Series(y_pred ).rank().values
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importance()
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= n_fold
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
result_dict['prediction'] = prediction
if make_oof:
result_dict['oof'] = oof
return result_dict<init_hyperparams>
test = donnees[donnees.Survived.isnull() ]
test.reset_index().head()
params = {'num_leaves': 512,
'objective': 'multiclass',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 3,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"random_state": 42,
"verbosity": -1,
"num_class": 5}<train_model>
X = apprentissage.drop(columns='Survived')
 y = apprentissage.Survived
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size=0.112, stratify = y, random_state = 101)
 print(X_train.shape, y_train.shape)
 print(X_test.shape, y_test.shape)
 plt.figure(figsize=(10,6))
plt.hist(y_train,label='apprentissage');
plt.hist(y_test,label='validation');
plt.legend() ;
result_dict_lgb = train_model(X=train, X_test=test, y=y, params=params, model_type='lgb', plot_feature_importance=True, make_oof=True )<train_model>
resultats = comparaisonsClassifieurs(classifieursArbresDict, X_train, X_test, y_train, y_test )
xgb_params = {'eta': 0.01, 'max_depth': 9, 'subsample': 0.9, 'colsample_bytree': 0.9,
'objective': 'multi:softprob', 'eval_metric': 'merror', 'silent': True, 'nthread': 4, 'num_class': 5}
result_dict_xgb = train_model(params=xgb_params, model_type='xgb', make_oof=True )<prepare_output>
classifieur = CatBoostClassifier(iterations=250, depth=3, silent=True )
prediction =(result_dict_lgb['prediction'] + result_dict_xgb['prediction'] ).argmax(1)
 submission = pd.DataFrame({'PetID': sub.PetID, 'AdoptionSpeed': [int(i)for i in prediction]})
 submission.head()<save_to_csv>
resultats = controleClassifieur(classifieur, X_train, X_test, y_train, y_test )
submission.to_csv('submission.csv', index=False )<save_to_csv>
resultatsFinaux,classifieursCV = effectueValidationCroisee(classifieur,
X,
y,
n_splits = 15 )
submission.to_csv('submission.csv', index=False )<set_options>
output = pd.DataFrame({'PassengerId':test.index, 'Survived':np.zeros(test.shape[0])})
 for i in range(len(classifieursCV)) :
output.Survived += classifieursCV[i].predict_proba(test)[:,1]
output.Survived /= len(classifieursCV)
 output.Survived = output.Survived.round().astype('int8')
 output.head()
<compute_test_metric><EOS> 
output.to_csv('submission008.csv',index=False )