Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<merge>
%matplotlib inline
warnings.filterwarnings('ignore')
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
df_train = pd.read_csv('.. /input/titanic/train.csv')
 df_test = pd.read_csv('.. /input/titanic/test.csv')
 combined = [df_train,df_test]
print("Ok!" )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 print('NaN structure:
{}'.format(np.sum(pd.isnull(X))))<define_variables>
for dataset in combined:
missing_data = dataset.isnull().sum().sort_values(ascending=False)
 missing_percent =(dataset.isnull().sum() * 100 / dataset.shape[0] ).sort_values(ascending=False)
 df_missing = pd.concat([missing_data,missing_percent], axis = 1, keys = ['Ausentes','%'])
 print(df_missing)
 print('-'*50 )
column_types = X.dtypes
int_cols = column_types[column_types == 'int']
float_cols = column_types[column_types == 'float']
cat_cols = column_types[column_types == 'object']
print('\tinteger columns:
{}'.format(int_cols))
print('
\tfloat columns:
{}'.format(float_cols))
print('
\tto encode categorical columns:
{}'.format(cat_cols))<define_variables>
for dataset in combined:
dataset['Cabin'][~dataset['Cabin'].isnull() ] = 1
dataset['Cabin'][dataset['Cabin'].isnull() ] = 0
print("Ok!" )
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID'] 
<merge>
for dataset in combined:
media = dataset['Age'].mean()
dataset['Age'] = dataset['Age'].fillna(media)
 print("Ok!" )
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<normalization>
bebe = [0,5,0]
crianca = [6,13,1]
jovem = [14,18,2]
jovemAdulto =[19,25,3]
Adulto = [26,60,4]
Idoso = [61,85,5]
idades = [bebe,crianca,jovem,jovemAdulto,Adulto,Idoso]
for dataset in combined:
dataset['AgeClass'] = dataset['Age'].astype(int)
 for i in range(len(dataset['Age'])) :
for idade in idades:
if(dataset['Age'][i] >= idade[0] and dataset['Age'][i] <= idade[1]):
dataset['AgeClass'][i] = idade[2]
break
print("Ok!" )
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<data_type_conversions>
for dataset in combined:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 print("Ok!" )
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>' )<feature_engineering>
for dataset in combined:
dataset['Embarked'] = dataset['Embarked'].map({'Q': 0, 'S': 1, 'C': 2} ).astype(int)
 print('Ok!' )
n_components = 5
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 nmf_ = NMF(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_{}_'.format(i))
text_features.append(svd_col)
 text_features.append(nmf_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<count_missing_values>
for dataset in combined:
dataset['Fare'] = dataset['Fare'].fillna(dataset['Fare'].median())
 print('Ok!' )
np.sum(pd.isnull(X_train))<count_missing_values>
muitoBaixo = [0,7.99,1]
baixo = [8,14.99,2]
medio = [15,31.99,3]
alto =[32,513,4]
fares = [muitoBaixo,baixo,medio,alto]
for dataset in combined:
dataset['FareClass'] = dataset['Fare'].astype(int)
 for i in range(len(dataset['Fare'])) :
for fare in fares:
if(dataset['Fare'][i] >= fare[0] and dataset['Fare'][i] <= fare[1]):
dataset['FareClass'][i] = fare[2]
break
print("Ok!" )
np.sum(pd.isnull(X_test))<import_modules>
for dataset in combined:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.',expand=False)
 print("Ok!" )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<init_hyperparams>
for dataset in combined:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 print("Ok!" )
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
early_stop = 500
verbose_eval = 100
num_rounds = 90000
n_splits = 10<prepare_x_and_y>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combined:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 print("Ok!" )
kfold = StratifiedKFold(n_splits=n_splits, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_index, valid_index in kfold.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_index, :]
X_val = X_train.iloc[valid_index, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train[valid_index] = val_pred
oof_test[:, i] = test_pred
i += 1<features_selection>
for dataset in combined:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 print("Ok!" )
importance_type= "split"
idx_sort = np.argsort(model.feature_importance(importance_type=importance_type)) [::-1]
names_sorted = np.array(model.feature_name())[idx_sort]
imports_sorted = model.feature_importance(importance_type=importance_type)[idx_sort]
for n, im in zip(names_sorted, imports_sorted):
print(n, im )<statistical_test>
drop_elements = ['Name', 'Ticket','PassengerId']
combined[0] = combined[0].drop(drop_elements,1)
 print("Ok!" )
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk )<predict_on_test>
drop_elements = ['Name', 'Ticket','PassengerId']
combined[1] = combined[1].drop(drop_elements,1)
 print("Ok!" )
coefficients_ = coefficients.copy()
coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = optR.predict(oof_train, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions)))
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions)) )<count_values>
drop_elements = ['Age', 'Fare']
combined[0] = combined[0].drop(drop_elements,1)
 combined[1] = combined[1].drop(drop_elements,1)
 print("Ok!" )
print("True Distribution:")
 print(pd.value_counts(X_train['AdoptionSpeed'], normalize=True ).sort_index())
 print("
Train Predicted Distribution:")
 print(pd.value_counts(train_predictions, normalize=True ).sort_index())
 print("
Test Predicted Distribution:")
 print(pd.value_counts(test_predictions, normalize=True ).sort_index() )<save_to_csv>
df_train = combined[0]
df_test = combined[1]
df_train['Cabin'] = df_train['Cabin'].astype(int)
 df_test['Cabin'] = df_test['Cabin'].astype(int)
 y_train = df_train['Survived']
x_train = df_train.drop('Survived',1)
 x_test = df_test
print("Ok!" )
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions.astype(np.int32)})
 submission.head()
submission.to_csv('submission.csv', index=False )<import_modules>
decisiontree = DecisionTreeClassifier()
scores = -1 * cross_val_score(decisiontree, x_train, y_train,
cv=5,
scoring='neg_mean_absolute_error')
 print("MAE:
", scores.mean() )
import numpy as np
import pandas as pd
import os
from fastai import *
from fastai.tabular import *
from sklearn.metrics import cohen_kappa_score<set_options>
randomforest = RandomForestClassifier()
scores = -1 * cross_val_score(randomforest, x_train, y_train,
cv=5,
scoring='neg_mean_absolute_error')
 print("MAE:
", scores.mean() )
seed = 42
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
if torch.cuda.is_available() : torch.cuda.manual_seed_all(seed )<load_from_csv>
gaussianNb = GaussianNB()
scores = -1 * cross_val_score(gaussianNb, x_train, y_train,
cv=5,
scoring='neg_mean_absolute_error')
 print("MAE:
", scores.mean() )
train_csv = pd.read_csv('.. /input/train/train.csv', low_memory=False)
 test_csv = pd.read_csv('.. /input/test/test.csv', low_memory=False)
 def preprocess(csv):
csv['Description_len'] = [len(str(tt)) for tt in csv['Description']]
csv['Name_len'] = [len(str(tt)) for tt in csv['Name']]
return csv
train_csv = preprocess(train_csv)
 test_csv = preprocess(test_csv )<define_variables>
linearDA = LinearDiscriminantAnalysis()
scores = -1 * cross_val_score(linearDA, x_train, y_train,
cv=5,
scoring='neg_mean_absolute_error')
 print("MAE:
", scores.mean() )
cat_names = ['Type','Breed1','Breed2','Gender','Color1','Color2','State','Color3','FurLength', 'Vaccinated','Dewormed','Sterilized','Health']
cont_names = ['Age', 'MaturitySize', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'Description_len', 'Name_len']<load_from_csv>
logreg = LogisticRegression()
scores = -1 * cross_val_score(logreg, x_train, y_train,
cv=5,
scoring='neg_mean_absolute_error')
 print("MAE:
", scores.mean() )
bs = len(train_csv)
 procs = [FillMissing, Categorify, Normalize]
df = TabularList.from_df(train_csv, path='.. /input', cat_names=cat_names, cont_names=cont_names, procs=procs)\ 
.no_split() \ 
.label_from_df(cols='AdoptionSpeed')
 df_test = TabularList.from_df(test_csv, path='.. /input', cat_names=cat_names, cont_names=cont_names, processor=df.train.x.processor)
 data = df.add_test(df_test ).databunch(num_workers=0, bs=bs )<choose_model_class>
svmsvc = svm.SVC()
scores = -1 * cross_val_score(svmsvc, x_train, y_train,
cv=5,
scoring='neg_mean_absolute_error')
 print("MAE:
", scores.mean() )
def bn_drop_lin(n_in:int, n_out:int, bn:bool=True, p:float=0., actn:Optional[nn.Module]=None):
"Sequence of batchnorm(if `bn`), dropout(with `p`)and linear(`n_in`,`n_out`)layers followed by `actn`."
layers = [nn.BatchNorm1d(n_in, track_running_stats=False)] if bn else []
if p != 0: layers.append(nn.Dropout(p))
layers.append(nn.Linear(n_in, n_out))
if actn is not None: layers.append(actn)
 return layers
class TabularModel(nn.Module):
"Basic model for tabular data."
def __init__(self, emb_szs:ListSizes, n_cont:int, out_sz:int, layers:Collection[int], ps:Collection[float]=None,
emb_drop:float=0., y_range:OptRange=None, use_bn:bool=True, bn_final:bool=False):
super().__init__()
ps = ifnone(ps, [0]*len(layers))
ps = listify(ps, layers)
 self.embeds = nn.ModuleList([embedding(ni, nf)for ni,nf in emb_szs])
 self.emb_drop = nn.Dropout(emb_drop)
 self.bn_cont = nn.BatchNorm1d(n_cont, track_running_stats=False)
 n_emb = sum(e.embedding_dim for e in self.embeds)
 self.n_emb,self.n_cont,self.y_range = n_emb,n_cont,y_range
sizes = self.get_sizes(layers, out_sz)
 actns = [nn.ReLU(inplace=True)] *(len(sizes)-2)+ [None]
layers = []
for i,(n_in,n_out,dp,act)in enumerate(zip(sizes[:-1],sizes[1:],[0.]+ps,actns)) :
layers += bn_drop_lin(n_in, n_out, bn=use_bn and i!=0, p=dp, actn=act)
 if bn_final: layers.append(nn.BatchNorm1d(sizes[-1]))
layers.append(nn.Softmax(dim=1))
self.layers = nn.Sequential(*layers)
 def get_sizes(self, layers, out_sz):
return [self.n_emb + self.n_cont] + layers + [out_sz]
def forward(self, x_cat:Tensor, x_cont:Tensor)-> Tensor:
if self.n_emb != 0:
x = [e(x_cat[:,i])for i,e in enumerate(self.embeds)]
x = torch.cat(x, 1)
 x = self.emb_drop(x)
 if self.n_cont != 0:
x_cont = self.bn_cont(x_cont)
 x = torch.cat([x, x_cont], 1)if self.n_emb != 0 else x_cont
x = self.layers(x)
 if self.y_range is not None:
x =(self.y_range[1]-self.y_range[0])* torch.sigmoid(x)+ self.y_range[0]
return x<normalization>
clf = svm.SVC()
clf.fit(x_train, y_train)
 y_pred = clf.predict(x_test)
 y_pred = np.rint(y_pred)
 y_pred = abs(y_pred)
 print("Ok!" )
<choose_model_class><EOS> 
gender_submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 gender_submission['Survived'] = y_pred
gender_submission['Survived'] = gender_submission['Survived'].astype(int)
 gender_submission.to_csv('submission.csv', index=False)
 print("Ok!" )