Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.impute import KNNImputer
from imblearn.over_sampling import SMOTE
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import BaggingClassifier
from lightgbm import LGBMClassifier
lm = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3, pretrained=True )<find_best_params>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
lm.lr_find()<train_model>
train.drop('PassengerId',axis=1,inplace=True)
 ID = test['PassengerId']
test.drop('PassengerId',axis=1,inplace=True )
lm.fit_one_cycle(1, 4e-3, callbacks=[SaveModelCallback(lm, name="best_lm")], moms=(0.8,0.7))<something_strange>
def visualNA(df,perc=0):
NAN = [(c, df[c].isna().mean() *100)for c in df]
NAN = pd.DataFrame(NAN, columns=["column_name", "percentage"])
 NAN = NAN[NAN.percentage > perc]
print(NAN.sort_values("percentage", ascending=False))
lm.unfreeze()<train_model>
train['Embarked'].value_counts()
lm.fit_one_cycle(5, 4e-3, callbacks=[SaveModelCallback(lm, name="best_lm")], moms=(0.8,0.7))<load_pretrained>
train['Embarked'].fillna(value='S',inplace=True )
lm.load("best_lm" )<categorify>
train[['Ticket_Class','Ticket_Number']]=train['Ticket'].str.split(" ",expand=True,n=1)
 train['Ticket_Class'],train['Ticket_Number'] = zip(*train[['Ticket_Class','Ticket_Number']].apply(( lambda x :(None,x['Ticket_Class'])if x['Ticket_Number'] is None else(x['Ticket_Class'],x['Ticket_Number'])) ,axis=1))
lm.save_encoder("enc" )<split>
train['Title'] = train['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
data_clf =(TextList.from_df(pd.concat([data_train, data_valid]), vocab=data_lm.vocab, cols=["full"] ).
 split_from_df("is_valid" ).
 label_from_df("label" ).
 add_test(data_test["full"] ).
 databunch()
)<choose_model_class>
train['Cabin'].fillna(value='0',inplace=True)
 train['Ticket_Class'].fillna(value='0',inplace=True )
clf = text_classifier_learner(data_clf, AWD_LSTM, drop_mult=0.3 )<find_best_params>
les = LabelEncoder()
train['Sex'] = les.fit_transform(train['Sex'])
 let = LabelEncoder()
train['Title'] = let.fit_transform(train['Title'].astype(str))
letn = LabelEncoder()
train['Ticket_Number'] = letn.fit_transform(train['Ticket_Number'].astype(str))
letc = LabelEncoder()
train['Ticket_Class'] = letc.fit_transform(train['Ticket_Class'].astype(str))
lec = LabelEncoder()
train['Cabin'] = lec.fit_transform(train['Cabin'].astype(str))
lee = LabelEncoder()
train['Embarked'] = lee.fit_transform(train['Embarked'].astype(str))
del lm
torch.cuda.empty_cache()<categorify>
train['Cabin'].replace(0,np.NaN,inplace=True )
clf.load_encoder("enc" )<find_best_params>
X = train[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch',
'Fare', 'Cabin', 'Embarked', 'Ticket_Class','Ticket_Number','Title']]
y = train['Survived']
clf.lr_find()<train_model>
imputer = KNNImputer(n_neighbors=5)
 X = imputer.fit_transform(X )
clf.fit(3, 2e-3, callbacks=[SaveModelCallback(clf, name="best_clf")] )<load_pretrained>
sm = SMOTE(random_state=42)
 X_res, y_res = sm.fit_resample(X, y )
clf.load("best_clf" )<load_pretrained>
model_bc = BaggingClassifier(RandomForestClassifier(n_estimators=300, max_depth=5, random_state=42),n_estimators=100)
 model_bc.fit(X_res,y_res )
clf.unfreeze()<train_model>
feature_importances = np.mean([
tree.feature_importances_ for tree in model_bc.estimators_
], axis=0 )
clf.fit(1, 3e-4, callbacks=[SaveModelCallback(clf, name="best_clf_ft1")] )<train_model>
test[['Ticket_Class','Ticket_Number']]=test['Ticket'].str.split(" ",expand=True,n=1)
 test['Ticket_Class'],test['Ticket_Number'] = zip(*test[['Ticket_Class','Ticket_Number']].apply(( lambda x :(None,x['Ticket_Class'])if x['Ticket_Number'] is None else(x['Ticket_Class'],x['Ticket_Number'])) ,axis=1))
clf.fit(1, 3e-4, callbacks=[SaveModelCallback(clf, name="best_clf_ft2")] )<predict_on_test>
test['Title'] = test['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
pred_val = clf.get_preds(DatasetType.Valid, ordered=True )<prepare_output>
for i,lechange in [('Ticket_Class',letc),('Cabin',lec),('Embarked',lee), 
('Ticket_Number',letn),('Title',let)]:
test[i] = test[i].map(lambda s: '<unknown>' if s not in lechange.classes_ else s)
 le_classes = lechange.classes_.tolist()
le_classes.insert(len(le_classes), '<unknown>')
 lechange.classes_ = le_classes
pred_val_l = pred_val[0].argmax(1 )<import_modules>
test['Ticket_Class'].fillna(value='0',inplace=True)
 test['Sex'] = les.transform(test['Sex'])
 test['Ticket_Class'] = letc.transform(test['Ticket_Class'].astype(str))
test['Cabin'] = lec.transform(test['Cabin'].astype(str))
test['Embarked'] = lee.transform(test['Embarked'].astype(str))
test['Ticket_Number'] = letn.transform(test['Ticket_Number'].astype(str))
test['Title'] = let.transform(test['Title'].astype(str))
from sklearn.metrics import classification_report<compute_test_metric>
X_test = test[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch',
'Fare', 'Cabin', 'Embarked', 'Ticket_Class','Ticket_Number','Title']]
print(classification_report(pred_val[1], pred_val_l))<predict_on_test>
X_test = imputer.transform(X_test )
pred_test, label_test = clf.get_preds(DatasetType.Test, ordered=True )<prepare_output>
y_pred_bc = model_bc.predict(X_test )
pred_test_ = pred_test.argmax(1)
 pred_test_l = [data_clf.train_ds.y.classes[n] for n in pred_test_]<rename_columns>
lgb = LGBMClassifier()
res.index.name = "id"<save_to_csv>
lgb.fit(X_res,y_res )
pd.DataFrame(res ).to_csv("submission.csv" )<import_modules>
y_pred_lgb = lgb.predict(X_test )
from types import SimpleNamespace
from collections import Counter
import os
import re
import pathlib
import array
import pickle
import numpy as np
import torch
import torch.nn as nn
import pandas as pd<define_variables>
Proba_bc=model_bc.predict_proba(X_test)
 Proba_lgb=lgb.predict_proba(X_test )
DATASET_VERSION = 'ca-100'
COMPETITION_ROOT = '.. /input/vectors'
DATASET_ROOT = f'.. /input/cbow-preprocessing/data/{DATASET_VERSION}'
WORKING_ROOT = f'data/{DATASET_VERSION}'
DATASET_PREFIX = 'ca.wiki'<define_variables>
def argmax(iterable):
return max(enumerate(iterable), key=lambda x: x[1])[0]
params = SimpleNamespace(
 embedding_dim = 100,
window_size = 5,
batch_size = 1000,
epochs = 4,
preprocessed = f'{DATASET_ROOT}/{DATASET_PREFIX}',
working = f'{WORKING_ROOT}/{DATASET_PREFIX}',
modelname = f'{WORKING_ROOT}/{DATASET_VERSION}.pt',
train = True
)<categorify>
def custom_pred(bc,lgb):
preds = list()
diff_index = [i for i, x in enumerate(y_pred_bc==y_pred_lgb)if not x]
for index,(i,j)in enumerate(zip(bc,lgb)) :
if index in diff_index:
preds.append(y_pred_bc[index])
 else:
preds.append(argmax(i+j)%2)
 return preds
class Vocabulary(object):
def __init__(self, pad_token='<pad>', unk_token='<unk>', eos_token='<eos>'):
self.token2idx = {}
self.idx2token = []
self.pad_token = pad_token
self.unk_token = unk_token
self.eos_token = eos_token
if pad_token is not None:
self.pad_index = self.add_token(pad_token)
 if unk_token is not None:
self.unk_index = self.add_token(unk_token)
 if eos_token is not None:
self.eos_index = self.add_token(eos_token)
 def add_token(self, token):
if token not in self.token2idx:
self.idx2token.append(token)
 self.token2idx[token] = len(self.idx2token)- 1
return self.token2idx[token]
def get_index(self, token):
if isinstance(token, str):
return self.token2idx.get(token, self.unk_index)
 else:
return [self.token2idx.get(t, self.unk_index)for t in token]
def __len__(self):
return len(self.idx2token)
 def save(self, filename):
with open(filename, 'wb')as f:
pickle.dump(self.__dict__, f)
 def load(self, filename):
with open(filename, 'rb')as f:
self.__dict__.update(pickle.load(f))<define_variables>
preds = custom_pred(Proba_bc,Proba_lgb )
def batch_generator(idata, target, batch_size, shuffle=True):
nsamples = len(idata)
 if shuffle:
perm = np.random.permutation(nsamples)
 else:
perm = range(nsamples)
 for i in range(0, nsamples, batch_size):
batch_idx = perm[i:i+batch_size]
if target is not None:
yield idata[batch_idx], target[batch_idx]
else:
yield idata[batch_idx], None<categorify>
submission = pd.DataFrame({
"PassengerId": ID,
"Survived": preds
})
 submission.to_csv('submission.csv', index=False )