Kaggle competition: Personality Profile Prediction
<SOS> metric: meancolumnwiselogloss Kaggle data source: personality-profile-prediction<train_model>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
drop_cols = ['Name','PassengerId','Ticket','Survived']
train = df[:len(train)]
X = train.drop(drop_cols, axis=1)
 y = train['Survived']
X_train, X_test, y_train, y_test = train_test_split(
 X, y, test_size=0.2, random_state=0)
 model = RandomForestClassifier()
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred)
 fti = model.feature_importances_
print(accuracy )<predict_on_test>
%matplotlib inline
drop_cols = ['Name','PassengerId','Ticket','Survived']
test = df[len(train):]
test_x = test.drop(drop_cols, axis=1)
 sub_y_RF = model.predict(test_x)
 sub['sub_y_RF'] = sub_y_RF<prepare_x_and_y>
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from nltk.tokenize import TreebankWordTokenizer
import re
import string
drop_cols = ['Name','PassengerId','Ticket','Survived','Fare','Cabin','Embarked',
'Age','SibSp','Parch']
x_NN = df.drop(drop_cols, axis=1)
 X_dummies_train = x_NN.iloc[0:890]
X_dummies_test = x_NN.iloc[891:]
Y = df.iloc[0:890]["Survived"]<choose_model_class>
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn import metrics
def create_neural_net(in_shape, lyrs=[4], act='relu', opt='Adam', dr=0.0):
seed(37556)
 tf.random.set_seed(37556)
 model = Sequential()
model.add(Dense(lyrs[0], input_dim=in_shape, activation=act))
for i in range(1,len(lyrs)) :
model.add(Dense(lyrs[i], activation=act))
model.add(Dropout(dr))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer=opt,
metrics=['accuracy'])
 return model<choose_model_class>
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import ComplementNB
single_net = create_neural_net(X_dummies_train.shape[1], lyrs =[4])
 single_net.summary()<train_model>
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
training = single_net.fit(X_dummies_train, Y, epochs=100, batch_size=32,
validation_split=0.25, verbose=0)
 val_acc = np.mean(training.history['val_accuracy'])
 print("
%s: %.2f%%" %('val_acc', val_acc*100))<train_model>
pd.options.display.max_columns = 1000
pd.options.display.max_rows = 1000
nn = KerasClassifier(build_fn=create_neural_net,
in_shape = X_dummies_train.shape[1],
lyrs=[12, 8, 4], epochs=50, dr=0.1, batch_size=1,
verbose=0)
 nn.fit(X_dummies_train, Y)
 sub['sub_y_NN'] = nn.predict(X_dummies_test ).astype('int32' )<save_to_csv>
train = pd.read_csv('.. /input/train.csv', encoding='utf-8')
 test = pd.read_csv('.. /input/test.csv', encoding='utf-8', index_col='id')
 example = pd.read_csv('.. /input/random_example.csv', encoding='utf-8', index_col='id' )
model_cols = ['sub_y_lgb','sub_y_RF','sub_y_NN','sub_y_LR']
sub['Survived'] = np.sum(sub[model_cols], axis=1)
 sub['Survived'] =(sub['Survived'] >= 3 ).astype(int)
 sub.drop(model_cols, axis=1, inplace=True)
 sub.to_csv('sub_title_ensembled.csv', index=False)
 sub.head()<import_modules>
print('Training data size: ', train.shape[0])
 print('Test data size: ', test.shape[0] )
import numpy as np
import pandas as pd
import os
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F
from sklearn.preprocessing import LabelEncoder 
<load_from_csv>
train = train.reset_index()
train = train.rename(columns={'index': 'id'})
 train['id'] = train['id']+test.shape[0]+1
test['type'] = '????'
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
combined = pd.concat([test.reset_index() , train], sort=False)
 combined.set_index('id', inplace=True )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<concatenate>
combined.isna().sum()
all_df = pd.concat([train, test], sort=False )<categorify>
labels = ['mind','energy','nature','tactics']
label_letters = ['E','N','T','J']
alt_label_letters = ['I', 'S', 'F', 'P']
def preprocess(df, cat_cols):
df = df.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1)
 for cat_col in cat_cols:
if cat_col in ['Embarked']:
df[cat_col] = LabelEncoder().fit_transform(df[cat_col].astype(str))
else:
df[cat_col] = LabelEncoder().fit_transform(df[cat_col])
 df = df.fillna(df.mean())
 return df<categorify>
def convert_type_to_int(df):

 for i in range(len(labels)) :
df[labels[i]] = df['type'].apply\ 
(lambda x: x[i] == label_letters[i] ).astype('int')
 return df
cat_cols = ['Pclass', 'Sex', 'SibSp', 'Parch', 'Embarked']
all_df = preprocess(all_df, cat_cols)
 all_df.head()<data_type_conversions>
data = combined
class TabularDataset(Dataset):
def __init__(self, df, categorical_columns, output_column=None):
super().__init__()
self.len = df.shape[0]
self.categorical_columns = categorical_columns
self.continous_columns = [col for col in df.columns if col not in self.categorical_columns + [output_column]]
if self.continous_columns:
self.cont_X = df[self.continous_columns].astype(np.float32 ).values
else:
self.cont_X = np.zeros(( self.len, 1))
if self.categorical_columns:
self.cat_X = df[self.categorical_columns].astype(np.int64 ).values
else:
self.cat_X = np.zeros(( self.len, 1))
if output_column != None:
self.has_label = True
self.label = df[output_column].astype(np.float32 ).values.reshape(-1, 1)
 else:
self.has_label = False
def __len__(self):
return self.len
def __getitem__(self, index):
if self.has_label:
return [self.label[index], self.cont_X[index], self.cat_X[index]]
else:
return [self.cont_X[index], self.cat_X[index]]<create_dataframe>
data = convert_type_to_int(data )
train_ds = TabularDataset(train_df, cat_cols, 'Survived')
 train_dl = DataLoader(train_ds, 64, shuffle=True )<choose_model_class>
def separate_posts(post):
return ' '.join(post.split('|||'))
class TitanicNet(nn.Module):
def __init__(self, emb_dims, n_cont, lin_layer_sizes, output_size):
super().__init__()
self.emb_layers = nn.ModuleList([nn.Embedding(x, y)for x, y in emb_dims])
 self.n_embs = sum([y for x, y in emb_dims])
 self.n_cont = n_cont
first_lin_layer = nn.Linear(self.n_embs + self.n_cont, lin_layer_sizes[0])
 self.lin_layers = nn.ModuleList(
 [first_lin_layer] +
[nn.Linear(lin_layer_sizes[i], lin_layer_sizes[i + 1])for i in range(len(lin_layer_sizes)- 1)] 
)
 self.output_layer = nn.Linear(lin_layer_sizes[-1], output_size)
 nn.init.kaiming_normal_(self.output_layer.weight.data)
 self.first_bn_layer = nn.BatchNorm1d(self.n_cont)
 self.bn_layers = nn.ModuleList([nn.BatchNorm1d(size)for size in lin_layer_sizes])
 def forward(self, cont_data, cat_data):
if self.n_embs != 0:
x = [emb_layer(cat_data[:, i])for i, emb_layer in enumerate(self.emb_layers)]
x = torch.cat(x, 1)
 if self.n_cont != 0:
normalized_cont_data = self.first_bn_layer(cont_data)
 if self.n_embs != 0:
x = torch.cat([x, normalized_cont_data], 1)
 else:
x = cont_data
for lin_layer, bn_layer in zip(self.lin_layers, self.bn_layers):
x = torch.relu(lin_layer(x))
x = bn_layer(x)
 x = self.output_layer(x)
 x = torch.sigmoid(x)
 return x<count_unique_values>
def remove_urls(post):
pattern_url = r'http[s]?://(?:[A-Za-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9A-Fa-f][0-9A-Fa-f])) +'
subs_url = r' '
return re.sub(pattern_url, subs_url, post )
cat_dims = [int(all_df[col].nunique())for col in cat_cols]
cat_dims<define_variables>
def remove_numbers(post):
p_numbers = '0123456789'
return ''.join([l for l in post if l not in p_numbers] )
emb_dims = [(x, min(50,(x + 1)// 2)) for x in cat_dims]
emb_dims<set_options>
def remove_punctuation(post):
return ''.join([l for l in post if l not in string.punctuation] )
torch.manual_seed(2 )<choose_model_class>
def remove_strange(post):
return post.replace("‘", '' ).replace("’", '' ).replace("'", '' )
model = TitanicNet(emb_dims, n_cont=2, lin_layer_sizes=[50, 100, 50], output_size=1)
 optimizer = torch.optim.Adam(model.parameters() , lr=0.003)
 no_of_epochs = 10
criterion = nn.BCELoss()
for epoch in range(no_of_epochs):
epoch_loss = 0
epoch_accuracy = 0
i = 0
for y, cont_x, cat_x in train_dl:
preds = model(cont_x, cat_x)
 loss = criterion(preds, y)
 epoch_loss += loss
accuracy =(( preds > 0.5 ).float() == y ).float().mean()
epoch_accuracy += accuracy
optimizer.zero_grad()
loss.backward()
optimizer.step()
print("Epoch ", epoch, ", loss: ", epoch_loss.item() /len(train_dl), "accuracy: ", epoch_accuracy.item() /len(train_dl)) 
<create_dataframe>
def lower_case(post):
return post.lower()
test_df = all_df.tail(test.shape[0])
 test_ds = TabularDataset(test_df, cat_cols, 'Survived')
 test_dl = DataLoader(test_ds, len(test_ds))<find_best_params>
def remove_extra_spaces(post):
return re.sub('\s+', ' ', post )
with torch.no_grad() :
for _, cont_x, cat_x in test_dl:
preds = model(cont_x, cat_x)
 preds =(preds > 0.5 )<create_dataframe>
def tokenizer_func(post):
return TreebankWordTokenizer().tokenize(post )
output_df = pd.DataFrame({'PassengerId':test['PassengerId'],'Survived':preds.flatten().numpy() } )<save_to_csv>
custom_stop_words = [remove_punctuation(word)for word in stopwords.words('english')[39:]]
output_df.to_csv('titanic_preds.csv', index=False )<set_options>
def remove_stop_words(tokens, stop_words=custom_stop_words):
return [token for token in tokens if(token not in stop_words)and(len(token)<15)]
pd.plotting.register_matplotlib_converters()
%matplotlib inline
%matplotlib inline
%matplotlib inline
plt.style.use('seaborn-whitegrid')
 warnings.filterwarnings('ignore' )<load_from_csv>
def lem_func(words, lemma = WordNetLemmatizer()):
return [lemma.lemmatize(word)for word in words if word not in custom_stop_words]
train=pd.read_csv("/kaggle/input/titanic/train.csv")
 X_test=pd.read_csv("/kaggle/input/titanic/test.csv")
 X_test.copy()
train.info()<import_modules>
def join_tokens(post):
return ' '.join(post )
from fastai.tabular import *<prepare_x_and_y>
def clean_data(df, col, cleaning_funcs):

 df['posts_processed'] = df[col].apply(separate_posts)
 for func in cleaning_funcs:
df['posts_processed'] = df['posts_processed'].apply(func)
 return df
procs = [FillMissing, Categorify, Normalize]
dep_var = 'Survived'
cat_names = ['Sex', 'Cabin', 'Embarked']
cont_names = ['PassengerId', 'Pclass', 'Age', 'SibSp', 'Parch', 'SibSp']
X=train[cont_names]
X=train[cat_names]<split>
for label in labels:
train['naive_'+label] = train[label].value_counts().index[0]
data =(TabularList.from_df(train, procs=procs, cont_names=cont_names, cat_names=cat_names)
.split_by_idx(valid_idx=range(int(len(train)*0.9),len(train)))
.label_from_df(cols=dep_var)
.add_test(TabularList.from_df(X_test, cat_names=cat_names, cont_names=cont_names, procs=procs)) 
.databunch())
 print(data.train_ds.cont_names)
 print(data.train_ds.cat_names )<choose_model_class>
naive_results = pd.DataFrame(data=[], index = ['Naive_Accuracy'], columns = labels)
 for label in labels:
naive_results[label] = metrics.accuracy_score(train[label], train['naive_'+label] )
learn = tabular_learner(data, layers=[1000,500], metrics=accuracy, wd=0.1 )<train_model>
for label in labels:
print('
Confusion matrix for ' + label + ' is:')
 print(metrics.confusion_matrix(train[label], train['naive_'+label]))
learn.fit_one_cycle(5, 2.5e-2 )<train_model>
data = clean_data(data, 'posts', cleaning_funcs )
learn.fit_one_cycle(5, 2.6e-9 )<save_model>
countV = CountVectorizer()
X_countV = countV.fit_transform(data['posts_processed'] )
learn.save('stage-1' )<predict_on_test>
tfidf = TfidfVectorizer()
X_tfidfV = tfidf.fit_transform(data['posts_processed'] )
preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 pred_prob, pred_class = preds.max(1 )<create_dataframe>
X1 = X_countV[len(test):]
X2 = X_tfidfV[len(test):]
y = data[len(test):]
submission = pd.DataFrame({'PassengerId': X_test.PassengerId, 'Survived':pred_class} )<save_to_csv>
logR = LogisticRegression(solver='lbfgs',n_jobs=-1)
 evaluate(logR, [X1,X2], y )
submission.to_csv('my_submission.csv', index=False )<import_modules>
logR = LogisticRegression(class_weight='balanced', solver='lbfgs', n_jobs=-1)
 evaluate(logR, [X1, X2], y )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import preprocessing
from keras.models import Sequential
from keras.layers import Dense, Dropout<load_from_csv>
knn = KNeighborsClassifier()
evaluate(knn, [0, X2], y )
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 train_data_copy = train_data.copy()
test_data_copy = test_data.copy()<categorify>
tfidf = TfidfVectorizer(max_features=1000, min_df=2, max_df=0.9)
 X_tfidfV = tfidf.fit_transform(data['posts_processed'] )
test_data['Test'] = 1
train_data['Test'] = 0
data = train_data.append(test_data, sort = False)
 drop_cols = list()
one_hot_encoding_cols = list()
normalization_cols = list()<drop_column>
X3 = X_tfidfV[len(test):]
data.drop('PassengerId', axis = 1, inplace = True )<count_values>
ks = [3, 5, 10]
param_grid = {'n_neighbors': ks}
grid_knn = GridSearchCV(KNeighborsClassifier() , param_grid, scoring='f1', cv=5, return_train_score=False )
data['Pclass'].value_counts(dropna = False)
 normalization_cols.append('Pclass' )<drop_column>
grid_knn.fit(X3, y[labels[0]] )
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 data.drop('Name', axis = 1, inplace = True )<count_values>
pd.DataFrame(grid_knn.cv_results_)[['params', 'mean_test_score', 'rank_test_score']]
data['Title'].value_counts()<define_variables>
randF = RandomForestClassifier(n_estimators=10)
 evaluate(randF, [0, X3], y )
mapping = {'Col': 'Army', 'Mlle' : 'Miss', 'Major' : 'Army', 'Sir': 'Royal',
'Mme': 'Mrs', 'Capt' : 'Army', 'Don' : 'Royal', 'Jonkheer' : 'Royal',
'Ms' : 'Miss', 'Countess' : 'Royal', 'Lady': 'Royal'}
data.replace({'Title': mapping}, inplace=True )<count_values>
cleaning_funcs = [remove_urls,
remove_numbers,
remove_extra_spaces]
data['Sex'].value_counts(dropna = False )<categorify>
data = clean_data(data, 'posts', cleaning_funcs )
label_encoder = preprocessing.LabelEncoder()
data['Sex'] = label_encoder.fit_transform(data['Sex'])
 data.head()<groupby>
tfidf = TfidfVectorizer(stop_words='english',
max_df=0.8,
min_df=2)
 X_tfidfV = tfidf.fit_transform(data['posts_processed'] )
data[data['Test'] == 0].groupby(['Title', 'Sex'] ).Age.mean()<feature_engineering>
X4 = X_tfidfV[len(test):]
X_sub = X_tfidfV[:len(test)]
y = data[len(test):]
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Army'), 'Age'] = 56.60
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Dr'), 'Age'] = 49.00
data.loc[(data.Age.isna())&(data.Sex == 0)&(data.Title == 'Dr'), 'Age'] = 40.60
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Master'), 'Age'] = 4.57
data.loc[(data.Age.isna())&(data.Sex == 0)&(data.Title == 'Miss'), 'Age'] = 21.85
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Mr'), 'Age'] = 32.36
data.loc[(data.Age.isna())&(data.Sex == 0)&(data.Title == 'Mrs'), 'Age'] = 35.78
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Rev'), 'Age'] = 43.16
data.loc[(data.Age.isna())&(data.Sex == 0)&(data.Title == 'Royal'), 'Age'] = 40.50
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Royal'), 'Age'] = 42.33<count_values>
logR = LogisticRegression(class_weight='balanced', max_iter=1000)
 logR_parameters = {'C': [0.01, 0.1, 1.0, 10],
'solver':('liblinear','saga','lbfgs')
 }
logR_grid = GridSearchCV(estimator=logR,
param_grid=logR_parameters,
cv=5,
return_train_score=False, )
data['SibSp'].value_counts()
normalization_cols.append('SibSp' )<count_values>
sub = example
data['Parch'].value_counts()
normalization_cols.append('Parch')
 data.head()<drop_column>
for label in labels:
y_train = y[label]
X_train = X4
logR_grid.fit(X_train, y_train)
 sub[label] = logR_grid.predict(X_sub)
 results = pd.DataFrame(logR_grid.cv_results_)
 print(results[results['rank_test_score']==1][['params', 'mean_test_score']] )
data.drop('Ticket', axis =1, inplace = True )<concatenate>
sub.to_csv('submission.csv' )