Kaggle competition: Personality Profile Prediction
normalization_cols.append('Fare')
<drop_column>
%matplotlib inline
data['HasCabin'] = data['Cabin'].isnull() == False
data['HasCabin'].replace(False, 0, inplace = True)
 data['HasCabin'].replace(True, 1, inplace = True)
 data.drop('Cabin', axis =1, inplace = True )<concatenate>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('Train info:')
 train.info()
print('
Test info:')
 test.info()
one_hot_encoding_cols.append('Embarked' )<categorify>
train['mind'] = train['type'].str[0]
train['energy'] = train['type'].str[1]
train['nature'] = train['type'].str[2]
train['tactics'] = train['type'].str[3]
data = pd.get_dummies(data = data, columns = one_hot_encoding_cols )<feature_engineering>
train['mind'] = train['mind'].apply(lambda x: 0 if x == 'I' else 1)
 train['energy'] = train['energy'].apply(lambda x: 0 if x == 'S' else 1)
 train['nature'] = train['nature'].apply(lambda x: 0 if x == 'F' else 1)
 train['tactics'] = train['tactics'].apply(lambda x: 0 if x == 'P' else 1)
 train.head()
std = data[data['Test'] == 0][normalization_cols].std(axis = 0)
 mean = data[data['Test'] == 0][normalization_cols].mean(axis = 0)
 data[normalization_cols] =(data[normalization_cols] - mean)/ std<drop_column>
def word_replace(df):

 df['posts'] = df['posts'].str.replace(r'http.?://[^\s]+[\s]?', 'url ')
 df['posts'] = df['posts'].str.replace(r'n't', ' not')
 df['posts'] = df['posts'].str.replace(r''s', ' is')
 df['posts'] = df['posts'].str.replace(r''m', ' am')
 df['posts'] = df['posts'].str.replace(r''re', ' are')
 df['posts'] = df['posts'].str.replace(r''ve', ' have')
 df['posts'] = df['posts'].str.replace(r''ll', ' will')
 df['posts'] = df['posts'].str.replace(r''d', ' would')
 df['posts'] = df['posts'].str.replace(r'
df['posts'] = df['posts'].str.replace(r"[',.():|-]", " ")
 df['posts'] = df['posts'].str.lower()
return df
train_data = data[data['Test'] == 0].drop(columns = ['Test'])
 test_data = data[data['Test'] == 1].drop(columns = ['Survived', 'Test'] )<data_type_conversions>
train_clean = word_replace(train.copy())
 test_clean = word_replace(test.copy() )
X = train_data.iloc[: , 1:].to_numpy()
y = train_data.iloc[:, 0].to_numpy()
print(str(X.shape))
print(str(y.shape))<choose_model_class>
def create_model() :
model = Sequential()
model.add(Dense(14, input_dim = 19, activation = 'relu'))
model.add(Dropout(0.3))
model.add(Dense(8, activation = 'relu'))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(loss='binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 return model<train_model>
get_tokens = TweetTokenizer()
get_lemmas = WordNetLemmatizer()
def tokenize_lemmatize(df):

 df['posts'] = df.apply(lambda row: [get_lemmas.lemmatize(w)for w in get_tokens.tokenize(row['posts'])], axis=1)
 return df
epochs = 20
model = create_model()
history = model.fit(X, y, epochs=epochs, validation_split = 0.3, batch_size=10 )<train_model>
train_clean = tokenize_lemmatize(train_clean.copy())
 test_clean = tokenize_lemmatize(test_clean.copy() )
epochs = 20
model = create_model()
history = model.fit(X, y, epochs=epochs, batch_size=10, verbose = 0 )<data_type_conversions>
emojies = set(emoji.UNICODE_EMOJI.keys())
 def swap_emoji(word):

 if word in emojies:
return 'emoji'
return word
def emoji_convert(df):

 df['posts'] = df['posts'].apply(lambda row: [swap_emoji(word)for word in row])
 return df
X_test = test_data.to_numpy()<predict_on_test>
train_clean = emoji_convert(train_clean.copy())
 test_clean = emoji_convert(test_clean.copy() )
prediction = model.predict(X_test )<prepare_output>
stop_words = stopwords.words('english')
 stop_words = stop_words + ['could', 'would']
stop_words = [w for w in stop_words if w not in ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves']]
def remove_words(df):

 df['posts'] = df['posts'].apply(lambda row: [word for word in row if word not in stop_words])
 return df
submission = pd.DataFrame(test_data_copy[['PassengerId']])
 submission['Survived'] = prediction
submission['Survived'] = submission['Survived'].apply(lambda x: 0 if x < 0.5 else 1 )<save_to_csv>
train_clean = remove_words(train_clean.copy())
 test_clean = remove_words(test_clean.copy() )
submission.to_csv('submission.csv', index = False )<set_options>
E = train_clean['mind'][train_clean['mind'] == 1].count() / train_clean['mind'].count()
print('E:I = {} : {}'.format(round(E, 2), 1-round(E, 2)))
 N = train_clean['energy'][train_clean['energy'] == 1].count() / train_clean['energy'].count()
print('N:S = {} : {}'.format(round(N, 2), 1-round(N, 2)))
 T = train_clean['nature'][train_clean['nature'] == 1].count() / train_clean['nature'].count()
print('T:F = {} : {}'.format(round(T, 2), 1-round(T, 2)))
 J = train_clean['tactics'][train_clean['tactics'] == 1].count() / train_clean['tactics'].count()
print('J:P = {} : {}'.format(round(J, 2), 1-round(J, 2)) )
%matplotlib inline<load_from_csv>
tt = TfidfVectorizer(preprocessor=list, tokenizer=list, ngram_range=(1,2), min_df=2, smooth_idf=False)
 def vectorise(train_set, test_set):

 tt.fit(train_set)
 train_vect = tt.transform(train_set)
 test_vect = tt.transform(test_set)
 return train_vect, test_vect
X_full = pd.read_csv('.. /input/titanic/train.csv')
 X_test_full = pd.read_csv('.. /input/titanic/test.csv')
 X_full.dropna(axis=0, subset=['Survived'], inplace=True)
 y = X_full.Survived
X_full.drop(['PassengerId'], axis=1, inplace=True)
 X_train_full, X_valid_full, y_train, y_valid = train_test_split(X_full, y,
train_size=0.7, test_size=0.3,
random_state=2)
 fig,(train, test)= plt.subplots(ncols=2, sharey=True, figsize=(15,5))
sns.heatmap(data=X_full.isnull() , ax=train)
 sns.heatmap(data=X_test_full.isnull() , ax=test )<prepare_output>
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import log_loss, confusion_matrix, accuracy_score
from sklearn.linear_model import LogisticRegression
categorical_cols = ['Sex', 'Embarked']
numerical_cols = ['Pclass', 'SibSp', 'Parch']
age_col = ['Age']
fare_col = ['Fare']
complex_cols = ['Name','Ticket','Cabin']
my_cols = categorical_cols + numerical_cols + age_col + fare_col + complex_cols
X_train = X_train_full[my_cols].copy()
X_valid = X_valid_full[my_cols].copy()
X_test = X_test_full[my_cols].copy()
X_train.head()<groupby>
X_train_vect, X_test_vect = vectorise(train_clean['posts'], test_clean['posts'])
 X_train_vect.shape
avg_survival_chance = X_full['Survived'].mean()
avg_survival_chance_male = X_full[X_full['Sex']=='male']['Survived'].mean()
avg_survival_chance_female = X_full[X_full['Sex']=='female']['Survived'].mean()
print('all: {}, male {}, female {}'.format(avg_survival_chance, avg_survival_chance_male, avg_survival_chance_female))<groupby>
cm = LogisticRegression(penalty='l1', solver='liblinear', random_state=11)
 params = {'C': [0.1, 0.5, 1, 5, 10, 50, 100]}
grid = GridSearchCV(cm, param_grid=params, scoring='neg_log_loss', n_jobs=-1, cv=4)
 categories = ['mind', 'energy', 'nature', 'tactics']
for cat in categories:
grid.fit(X_train_vect, train_clean[cat])
 print(cat + ':')
 print('Best score: ', -grid.best_score_)
 print('Best paramaters: ', grid.best_params_)
 print('
' )
ticket_survival = X_full[['Ticket','Survived']].copy()
ticket_survival['TicketCode'] = ticket_survival['Ticket'].map(lambda c : c.split() [0])
 ticket_survival['TicketCode'] = ticket_survival['TicketCode'].map(lambda c : 'NUM' if c.isdigit() else c)
 ticket_survival = ticket_survival.groupby('TicketCode' ).mean().sort_values(by='Survived')['Survived']
ticket_survival<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(train_clean['posts'], train_clean[['mind', 'energy', 'nature', 'tactics']], random_state=11)
 X_train_vect, X_test_vect = vectorise(X_train, X_test)
 X_train_vect.shape
code_dict = {}
def split_ticket_codes(surv):
if surv < 0.05:
return 1
elif surv < 0.15:
return 2
elif surv < 0.3:
return 3
elif surv < 0.4:
return 4
elif surv < 0.48:
return 5
elif surv < 0.6:
return 6
elif surv < 0.8:
return 7
return 8
for i in ticket_survival.index:
code_dict[i] = split_ticket_codes(ticket_survival[i])
 code_dict['Y'] = split_ticket_codes(avg_survival_chance )<sort_values>
def build_model(X_train_vect_f, X_test_vect_f, y_train_f, y_test_f, model):

 model.fit(X_train_vect_f, y_train_f)
 train_pred = model.predict(X_train_vect_f)
 test_pred = model.predict(X_test_vect_f)
 train_proba = model.predict_proba(X_train_vect_f)
 test_proba = model.predict_proba(X_test_vect_f)
 train_loss = log_loss(y_train_f, train_proba, eps=1e-15)
 test_loss = log_loss(y_test_f, test_proba, eps=1e-15)
 test_accuracy = accuracy_score(y_test_f, test_pred)
 test_matrix = confusion_matrix(y_test_f, test_pred)
 return train_pred, test_pred, train_loss, test_loss, test_accuracy, test_matrix
cabin_survival = X_full[['Cabin','Survived']].copy()
cabin_survival['Code'] = cabin_survival['Cabin'].map(lambda c : str(c)[0])
 cabin_survival = cabin_survival.groupby('Code' ).mean().sort_values(by='Survived')
 cabin_survival<define_variables>
ll_scores = []
ac_scores = []
cabin_dict = {'T':1, 'n':2, 'A': 3, 'G':4, 'C':5, 'F':6, 'B':7, 'E':8, 'D':9}<define_variables>
cm = LogisticRegression(penalty='l1', solver='liblinear', C=4, random_state=11)
 pred_train, pred_test, loss_train, loss_test, acc_test, matrix_test = build_model(X_train_vect, X_test_vect, y_train['mind'], y_test['mind'], cm)
 ll_scores.append(loss_test)
 ac_scores.append(acc_test)
 print('Mind:')
 print('Log loss on training set: ',loss_train)
 print('Log loss on test set: ', loss_test)
 print('Accuracy on test set:', acc_test)
 print('Test set confusion matrix:
', matrix_test )
title_dict = {'Capt':0, 'Don':1, 'Jonkheer':2, 'Rev':3,
'Mr':4,
'Dr':7,
'Col':8, 'Major':9, 'Master':10,
'Miss':11,
'Mrs':15,
'Mme':16, 'Sir':17, 'Ms':18, 'Lady':19, 'Mlle':20, 'the Countess':21,
'Don':5, 'Jonkheer':6,
'Dona':12, 'Lady':13}<concatenate>
cm = LogisticRegression(penalty='l1', solver='liblinear', C=5, random_state=11)
 pred_train, pred_test, loss_train, loss_test, acc_test, matrix_test = build_model(X_train_vect, X_test_vect, y_train['energy'], y_test['energy'], cm)
 ll_scores.append(loss_test)
 ac_scores.append(acc_test)
 print('Energy:')
 print('Log loss on training set: ',loss_train)
 print('Log loss on test set: ', loss_test)
 print('Accuracy on test set:', acc_test)
 print('Test set confusion matrix:
', matrix_test )
train_age = X_train[['Age','Name']].append(X_test[['Age','Name']] )<feature_engineering>
cm = LogisticRegression(penalty='l1', solver='liblinear', C=5, random_state=11)
 pred_train, pred_test, loss_train, loss_test, acc_test, matrix_test = build_model(X_train_vect, X_test_vect, y_train['nature'], y_test['nature'], cm)
 ll_scores.append(loss_test)
 ac_scores.append(acc_test)
 print('Nature:')
 print('Log loss on training set: ',loss_train)
 print('Log loss on test set: ', loss_test)
 print('Accuracy on test set:', acc_test)
 print('Test set confusion matrix:
', matrix_test )
def get_title(name):
return name.split(',')[1].split('.')[0].strip()
def get_title_id(title):
return title_dict[title] if title in title_dict else 9
def get_age_per_title(x):
x['Title'] = x.Name.map(lambda x : get_title(x))
return x.groupby('Title' ).Age.mean()
avg_age_per_title = get_age_per_title(train_age )<concatenate>
cm = LogisticRegression(penalty='l1', solver='liblinear', C=3, random_state=11)
 pred_train, pred_test, loss_train, loss_test, acc_test, matrix_test = build_model(X_train_vect, X_test_vect, y_train['tactics'], y_test['tactics'], cm)
 ll_scores.append(loss_test)
 ac_scores.append(acc_test)
 print('Tactics:')
 print('Log loss on training set: ',loss_train)
 print('Log loss on test set: ', loss_test)
 print('Accuracy on test set:', acc_test)
 print('Test set confusion matrix:
', matrix_test )
train_fare = X_train[['Fare','Pclass']].append(X_test[['Fare','Pclass']] )<groupby>
print('Average log loss across personality categories: ', round(np.mean(ll_scores), 3))
print('Average accuracy across personality categories: ', round(np.mean(ac_scores), 3))
def get_fare_per_pclass(x):
avg_fare = []
for i in range(1,4):
avg_fare.append(x[x['Pclass']==i]['Fare'].mean())
 print('avg_fare: {}'.format(avg_fare))
return avg_fare
avg_fare = get_fare_per_pclass(train_fare )<train_model>
results = test[['id']]
X_train_vect, X_test_vect = vectorise(train_clean['posts'], test_clean['posts'])
 X_train_vect.shape
class NameAgeFeaturesTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, x, y=None):
return self
def omit_garbage(self, name):
name = name.replace('(', '(')
 name = name.replace(')', ')')
 parts = name.split()
idx_start = -1
idx_end = -1
has_nick = 0
for p in parts:
i = p.find('(')
 j = p.find(')')
 if i >= 0:
idx_start = i
if j >= 0:
idx_end = j
i = p.find('"')
 if i >= 0:
has_nick = 1
comp_len = idx_end - idx_start
return len(parts)- comp_len - has_nick
def transform(self, x):
x['Title'] = x['Name'].map(lambda name : get_title(name))
x['NameLen'] = x['Name'].map(lambda name : self.omit_garbage(name))
x['Age'] = x.apply(lambda x : avg_age_per_title[x['Title']] if pd.isnull(x['Age'])else x['Age'], axis=1)
 x['Title'] = x['Title'].map(lambda title : title_dict[title])
 x.drop('Name', axis=1, inplace=True)
 return x.values<categorify>
c_vals = [4, 5, 5, 3]
class1_proba = []
for i in range(4):
cm = LogisticRegression(penalty='l1', solver='liblinear', C=c_vals[i], random_state=11)
 cm.fit(X_train_vect, train[categories[i]].values)
 pred_test = cm.predict(X_test_vect)
 results[categories[i]] = pred_test
probability = cm.predict_proba(X_test_vect)
 class1_proba.append(probability[:,1] )
class CabinFeaturesTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, x, y=None):
return self
def transform(self, x):
x.Cabin.fillna('n0', inplace=True)
 x['CabinCode'] = x['Cabin'].map(lambda c : str(c)[0])
 x['CabinCode'] = x['CabinCode'].map(lambda c : int(cabin_dict[c]))
x.drop('Cabin', axis=1, inplace=True)
 return x.values<categorify>
results.to_csv('MBTI_l1_liblinear_C4553_split_tt_all_pp_emoji_lib_handles.csv', index=False)
class FareFeaturesTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, x, y=None):
return self
def fillEmpty(self, fare, pclass):
if pd.isnull(fare):
return avg_fare[int(pclass)-1]
return fare
def transform(self, x):
x['Fare'] = x.apply(lambda c : self.fillEmpty(c['Fare'], c['Pclass']), axis=1)
 x.drop('Pclass', axis=1, inplace=True)
 return x.values<categorify>
for i in range(4):
results[categories[i]] = class1_proba[i]
<categorify><EOS> 
results.to_csv('MBTI_l1_liblinear_C4553_split_tt_all_pp_emoji_lib_handles_proba.csv', index=False)