Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
%matplotlib inline
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
data_train = pd.read_csv('.. /input/train.csv')
 data_test = pd.read_csv('.. /input/test.csv' )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
def simplify_ages(df):
unknow = df['Age'].isnull()
master_title = df['Title'] == 'Master'
mr_title = df['Title'] == 'Mr'
mrs_title = df['Title'] == 'Mrs'
miss_title = df['Title'] == 'Miss'
dr_title = df['Title'] == 'Dr'
df.loc[master_title & unknow,'Age'] = df[master_title]['Age'].mean()
df.loc[mr_title & unknow,'Age'] = df[mr_title]['Age'].mean()
df.loc[mrs_title & unknow,'Age'] = df[mrs_title]['Age'].mean()
df.loc[miss_title & unknow,'Age'] = df[miss_title]['Age'].mean()
df.loc[dr_title & unknow,'Age'] = df[dr_title]['Age'].mean()
df.Age = df.Age.fillna(-0.5)
 return df
def simplify_cabins(df):
df.Cabin = df.Cabin.fillna('N')
 df.Cabin = df.Cabin.apply(lambda x: x[0])
 return df
def simplify_fares(df):
df.Fare = df.Fare.fillna(-0.5)
 return df
def format_name(df):
df['Lname'] = df.Name.apply(lambda x: x.split(',')[0])
 df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)
 return df
def simplify_family(df):
df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
return df
def simplify_embarks(df):
df.Embarked = df.Embarked.fillna('N')
 df.Embarked = df.Embarked.apply(lambda x: x[0])
 return df
def simplify_titles(df):
df['Title'] = df['Title'].replace(['Col','Dr','Major'], 'MidNoble')
 df['Title'] = df['Title'].replace(['Mlle','Ms','Mme','Countess','Sir', 'Lady'], 'Noble')
 df['Title'] = df['Title'].replace(['Don','Jonkheer'], 'Mr')
 df['Title'] = df['Title'].replace(['Capt','Rev'], 'Worker')
 return df
def create_sex_class(df):
male = df['Sex']=='male'
female = df['Sex']=='female'
Class1 = df['Pclass']==1
Class2 = df['Pclass']==2
Class3 = df['Pclass']==3
df.loc[female & Class1,'Class'] =(0)
 df.loc[female & Class2,'Class'] =(1)
 df.loc[female & Class3,'Class'] =(2)
 df.loc[male & Class1,'Class'] =(3)
 df.loc[male & Class2,'Class'] =(4)
 df.loc[male & Class3,'Class'] =(5)
 return df
def drop_features(df):
return df.drop(['Ticket', 'SibSp', 'Parch', 'Name', 'Lname', 'Pclass'], axis=1)
 def transform_features(df):
df = format_name(df)
 df = simplify_ages(df)
 df = simplify_cabins(df)
 df = simplify_fares(df)
 df = simplify_family(df)
 df = simplify_embarks(df)
 df = simplify_titles(df)
 df = create_sex_class(df)
 df = drop_features(df)
 return df
def encode_features(df_train, df_test):
features = ['Cabin', 'Title', 'Embarked', 'Sex']
df_combined = pd.concat([df_train[features], df_test[features]])
 for feature in features:
le = preprocessing.LabelEncoder()
le = le.fit(df_combined[feature])
 df_train[feature] = le.transform(df_train[feature])
 df_test[feature] = le.transform(df_test[feature])
 return df_train, df_test
import numpy as np
import pandas as pd
import time
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import mean_squared_error
from sklearn.metrics import log_loss<load_from_csv>
data_train = transform_features(data_train)
 data_test = transform_features(data_test )
%%time
df_train = pd.read_csv('.. /input/predicting-outliers-to-improve-your-score/train_clean.csv')
 df_test = pd.read_csv('.. /input/predicting-outliers-to-improve-your-score/test_clean.csv' )<drop_column>
data_train, data_test = encode_features(data_train, data_test )
df_train = df_train[df_train['outliers'] == 0]
target = df_train['target']
del df_train['target']
features = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','outliers']]
categorical_feats = [c for c in features if 'feature_' in c]<init_hyperparams>
X_all = data_train.drop(['Survived', 'PassengerId'], axis=1)
 y_all = data_train['Survived']
param = {'objective':'regression',
'num_leaves': 31,
'min_data_in_leaf': 25,
'max_depth': 7,
'learning_rate': 0.01,
'lambda_l1':0.13,
"boosting": "gbdt",
"feature_fraction":0.85,
'bagging_freq':8,
"bagging_fraction": 0.9 ,
"metric": 'rmse',
"verbosity": -1,
"random_state": 2333}<split>
def MultivariateGaussian(x,y):
k = 2
d =(x.shape)[1]
mu = np.zeros(( k,d))
sigma = np.zeros(( k,d,d))
pi = np.zeros(k)
 for label in range(2):
indices =(y == label)
 mu[label] = np.mean(x[indices,:], axis=0)
 sigma[label] = np.cov(x[indices,:], rowvar=0, bias=1)
 pi[label] = float(sum(indices)) /float(len(y))
return mu, sigma, pi
%%time
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=2333)
 oof = np.zeros(len(df_train))
predictions = np.zeros(len(df_test))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(df_train,df_train['outliers'].values)) :
print("fold {}".format(fold_))
trn_data = lgb.Dataset(df_train.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(df_train.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval= 100, early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(df_train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(df_test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(mean_squared_error(oof, target)**0.5))<create_dataframe>
mu, sigma, pi = MultivariateGaussian(X_all.values,y_all.values )
model_without_outliers = pd.DataFrame({"card_id":df_test["card_id"].values})
 model_without_outliers["target"] = predictions<load_from_csv>
def test_model(mu, sigma, pi, features, tx, ty):
preds = []
errors = 0
for x,y in zip(tx,ty):
piP_list = []
for label in range(2):
S = np.linalg.inv(sigma[label])
 dS = np.linalg.det(sigma[label])
 xTsigmax=0
for i in features:
for j in features:
xTsigmax += S[i][j]*(x[i]-mu[label][i])*(x[j]-mu[label][j])
 piP_list.append(pi[label]*np.exp(-0.5*xTsigmax)/np.sqrt(dS))
predict = np.argmax(piP_list)
 preds.append(predict)
 if predict != y:
errors+=1
return errors, preds
%%time
df_train = pd.read_csv('.. /input/predicting-outliers-to-improve-your-score/train_clean.csv')
 df_test = pd.read_csv('.. /input/predicting-outliers-to-improve-your-score/test_clean.csv' )<drop_column>
errors, preds = test_model(mu, sigma, pi, [0,1,2,3,4,5,6,7], X_all.values, y_all.values)
 print('Accuracy for training data:',(1-errors/len(y_all)) *100, '%' )
target = df_train['outliers']
del df_train['outliers']
del df_train['target']<define_variables>
def write_predictions(mu, sigma, pi, features, tx):
preds = []
for x in tx:
piP_list = []
for label in range(2):
S = np.linalg.inv(sigma[label])
 dS = np.linalg.det(sigma[label])
 xTsigmax=0
for i in features:
for j in features:
xTsigmax += S[i][j]*(x[i]-mu[label][i])*(x[j]-mu[label][j])
 piP_list.append(pi[label]*np.exp(-0.5*xTsigmax)/np.sqrt(dS))
predict = np.argmax(piP_list)
 preds.append(predict)
 return preds
features = [c for c in df_train.columns if c not in ['card_id', 'first_active_month']]
categorical_feats = [c for c in features if 'feature_' in c]<init_hyperparams>
preds = write_predictions(mu, sigma, pi, [0,1,2,3,4,5,6,7], data_test.drop('PassengerId',axis=1 ).values )
param = {'num_leaves': 31,
'min_data_in_leaf': 30,
'objective':'binary',
'max_depth': 6,
'learning_rate': 0.01,
"boosting": "rf",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": 11,
"metric": 'binary_logloss',
"lambda_l1": 0.1,
"verbosity": -1,
"random_state": 2333}<split>
data_pred = pd.DataFrame()
data_pred['PassengerId']=data_test['PassengerId']
data_pred['Survived']=preds
<prepare_output><EOS> 
data_pred.to_csv('predictions_generative_multGauss.csv', index = False )