Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<merge>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
tf_idf = df["product_name"].map(calculate_score)
 tf_idf = np.nan_to_num(np.array(tf_idf.tolist()))
max_value, min_value = tf_idf.max() , tf_idf.min()
tf_idf =(tf_idf - min_value)/(max_value - min_value)
 df = df.merge(pd.DataFrame(tf_idf.tolist() , columns=["dept_"+str(i)for i in range(n_department)]), left_index=True, right_index=True)
 df = df.reset_index()<load_pretrained>
dataset = pd.read_csv('.. /input/train.csv' )
model_ft = gensim.models.KeyedVectors.load_word2vec_format('.. /input/ykc-2nd/wiki-news-300d-1M.vec/wiki-news-300d-1M.vec' )<install_modules>
dataset.isnull().sum()
!pip install hyphenate<import_modules>
dataset['Age'] = dataset['Age'].fillna(dataset['Age'].median() )
from hyphenate import hyphenate_word<categorify>
dataset['Embarked'].value_counts()
unused_words = defaultdict(int)
 def to_vec(x, model_ft):
v = np.zeros(model_ft.vector_size)
 for w in x:
try:
v += model_ft[w]
except:
hw = hyphenate_word(w)
 for w2 in hw:
try:
v += model_ft[w2]
except:
unused_words[w2] += 1
v = v /(np.sqrt(np.sum(v ** 2)) + 1e-16)
 return v
def to_vec_max(x, model_ft):
v = [np.zeros(model_ft.vector_size), np.zeros(model_ft.vector_size)]
for w in x:
try:
v.append(model_ft[w])
 except:
hw = hyphenate_word(w)
 for w2 in hw:
try:
v.append(model_ft[w2])
 except:
pass
v = np.max(v, axis=0)
 v = v /(np.sqrt(np.sum(v ** 2)) + 1e-16)
 return v
vecs = df["product_name"].apply(lambda x : to_vec(x, model_ft))
vecs = np.vstack(vecs)
 vecs = np.nan_to_num(vecs)
 vecs_max = df["product_name"].apply(lambda x : to_vec_max(x, model_ft))
vecs_max = np.vstack(vecs_max)
 vecs_max = np.nan_to_num(vecs_max)
 fasttext_pretrain_cols = [f"fasttext_pretrain_vec{k}" for k in range(vecs.shape[1])]
fasttext_pretrain_cols_max = [f"fasttext_pretrain_vec_max{k}" for k in range(vecs.shape[1])]
vec_df = pd.DataFrame(vecs, columns=fasttext_pretrain_cols)
 vec_max_df = pd.DataFrame(vecs_max, columns=fasttext_pretrain_cols_max)
 df = pd.concat([df, vec_df], axis = 1)
 df = pd.concat([df, vec_max_df], axis = 1)
 df.head()<split>
dataset['Embarked'].fillna('S', inplace = True )
df_train = df[~df.department_id.isna() ]
df_test = df[df.department_id.isna() ]<define_variables>
dataset['Sex'].value_counts()
np.random.seed(42)
 trainCols = fasttext_pretrain_cols + fasttext_pretrain_cols_max + \
["order_rate", "order_dow_mode", "order_hour_of_day_mode"]
trainCols2 = ["dept_"+str(i)for i in range(n_department)] + ["order_rate", "order_dow_mode", "order_hour_of_day_mode"]
nn_cols = ["dept_"+str(i)for i in range(n_department)]
nn_cols2 = fasttext_pretrain_cols + fasttext_pretrain_cols_max + ["dept_"+str(i)for i in range(n_department)]
X_train = df_train[trainCols]
X2_train = df_train[trainCols2]
Xnn_train = df_train[nn_cols]
Xnn2_train = df_train[nn_cols2]
Y_train = df_train["department_id"]
Ynn_train = pd.get_dummies(df_train, columns=["department_id"])[[f"department_id_{i}.0" for i in range(21)]]
validInds = np.random.choice(X_train.index.values, 4000, replace = False)
 trainInds = np.setdiff1d(X_train.index.values, validInds )<init_hyperparams>
dataset['Sex'] = dataset['Sex'].apply(lambda x: 1 if x=='male' else 0 )
params = {
'lambda_l1': 0.0,
'lambda_l2': 0.0,
'num_leaves': 200,
'feature_fraction': 0.9840000000000001,
'bagging_fraction': 1.0,
'bagging_freq': 0,
'min_child_samples': 20,
'task': 'train',
'boosting_type': 'dart',
'objective': 'multiclass',
'num_class': 21,
'metric': 'multi_error',
'verbose': 1,
'learning_rate': 0.1,
'num_iterations': 2000,
'max_depth': 7
}<train_model>
cat_columns = ['Embarked']
dataset = pd.get_dummies(dataset,prefix_sep = '__', columns = cat_columns )
lgb_train = lgb.Dataset(X_train.iloc[trainInds], Y_train.iloc[trainInds])
 lgb_val = lgb.Dataset(X_train.iloc[validInds], Y_train.iloc[validInds])
 best_params, history = {}, []
lgb_model = lgb.train(params, lgb_train,
valid_sets=lgb_val,
verbose_eval=10 )<predict_on_test>
dataset.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True )
pred_val_lgb = lgb_model.predict(X_train.iloc[validInds])
 score = {
"logloss" : log_loss(Y_train.iloc[validInds], pred_val_lgb),
"f1_micro" : f1_score(Y_train.iloc[validInds], np.argmax(pred_val_lgb, axis = 1), average = "micro")}
pred_test_lgb = lgb_model.predict(df_test[trainCols] )<init_hyperparams>
x = dataset.iloc[:, 1:].values
y = dataset.iloc[:,:1].values
params = {
'lambda_l1': 0.0,
'lambda_l2': 0.0,
'num_leaves': 111,
'feature_fraction': 0.9840000000000001,
'bagging_fraction': 1.0,
'bagging_freq': 0,
'min_child_samples': 20,
'task': 'train',
'boosting_type': 'dart',
'objective': 'multiclass',
'num_class': 21,
'metric': 'multi_error',
'verbose': 1,
'learning_rate': 0.05,
'num_iterations': 1000
}<train_model>
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.2 )
lgb_train2 = lgb.Dataset(X2_train.iloc[trainInds], Y_train.iloc[trainInds])
 lgb_val2 = lgb.Dataset(X2_train.iloc[validInds], Y_train.iloc[validInds])
 best_params, history = {}, []
lgb_model2 = lgb.train(params, lgb_train2,
valid_sets=lgb_val2,
verbose_eval=10 )<predict_on_test>
N,D = x.shape
M = 10
pred_val_lgb2 = lgb_model2.predict(X2_train.iloc[validInds])
 score = {
"logloss" : log_loss(Y_train.iloc[validInds], pred_val_lgb2),
"f1_micro" : f1_score(Y_train.iloc[validInds], np.argmax(pred_val_lgb2, axis = 1), average = "micro")}
pred_test_lgb2 = lgb_model2.predict(df_test[trainCols2] )<choose_model_class>
w1= np.random.randn(D,M)/np.sqrt(D+M)
 b1 = np.zeros(M)
 w2 = np.random.randn(M,1)/np.sqrt(M+1)
 b2 = 0
nn_model1 = tf.keras.Sequential([
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(32, activation='relu'),
tf.keras.layers.Dense(21, activation='softmax')
 ] )<choose_model_class>
def sigmoid(z):
return 1/(1+np.exp(-z))
nn_model1.compile(optimizer=tf.optimizers.Adam(lr=0.001),
loss='categorical_crossentropy',
metrics=['accuracy'])
 es_cb = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='auto' )<train_model>
d_relu = lambda x:(x>0 ).astype(x.dtype )
history = nn_model1.fit(Xnn2_train.iloc[trainInds], Ynn_train.iloc[trainInds], validation_data=(Xnn2_train.iloc[validInds], Ynn_train.iloc[validInds]), epochs=100, batch_size=16, verbose=2, callbacks=[es_cb] )<predict_on_test>
def forward(x,w1,b1,w2,b2):
z = np.maximum(( x.dot(w1)+b1), 0)
 return sigmoid(z.dot(w2)+b2), z
pred_val_nn1 = nn_model1.predict(Xnn2_train.iloc[validInds])
 pred_test_nn1 = nn_model1.predict(df_test[nn_cols2] )<compute_test_metric>
def cross_en(T,Y):
return -np.mean(T*np.log(Y)+(1-T)*np.log(1-Y))
log_loss(Y_train.iloc[validInds], pred_val_nn1), f1_score(Y_train.iloc[validInds], np.argmax(pred_val_nn1, axis = 1), average = "micro" )<choose_model_class>
cost_train = []
cost_test = []
lr = 0.001
nn_model = tf.keras.Sequential([
tf.keras.layers.Dense(512, activation='relu'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(21, activation='softmax')
 ] )<choose_model_class>
for i in range(100000):
yprep, ztrain = forward(x_train,w1,b1,w2,b2)
 train_cost = cross_en(y_train, yprep)
 yprep_test, ztest = forward(x_test, w1,b1,w2,b2)
 test_cost = cross_en(y_test, yprep_test)
 if i%5000==0:
print('train_cost = {}'.format(train_cost))
print('test_cost = {}'.format(test_cost))
E =(yprep-y_train)
 w2-= lr*(ztrain.T.dot(E)/len(x_train))
b2-= lr*(E.sum() /len(x_train))
dz = E.dot(w2.T)*d_relu(ztrain)
 w1-= lr*(x_train.T.dot(dz)/len(x_train))
b1-= lr*(dz.sum() /len(x_train))
cost_train.append(train_cost)
 cost_test.append(test_cost)
nn_model.compile(optimizer=tf.optimizers.Adam(lr=0.001),
loss='categorical_crossentropy',
metrics=['accuracy'])
 es_cb = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='auto' )<prepare_x_and_y>
test_dataset = pd.read_csv('.. /input/test.csv' )
nn_cols = ["dept_"+str(i)for i in range(n_department)]
Xnn_train = df_train[nn_cols]
Y_train = df_train["department_id"] 
<train_model>
test_dataset.isnull().sum()
history = nn_model.fit(Xnn_train.iloc[trainInds], Ynn_train.iloc[trainInds], validation_data=(Xnn_train.iloc[validInds], Ynn_train.iloc[validInds]), epochs=100, batch_size=16, verbose=2, callbacks=[es_cb] )<predict_on_test>
test_dataset['Age']= test_dataset['Age'].fillna(test_dataset['Age'].median())
 test_dataset['Fare']= test_dataset['Fare'].fillna(test_dataset['Fare'].median() )
pred_val_nn = nn_model.predict(Xnn_train.iloc[validInds])
 Xnn_test = df_test[nn_cols]
pred_test_nn = nn_model.predict(Xnn_test )<compute_test_metric>
test_dataset['Sex'] = test_dataset['Sex'].apply(lambda x: 1 if x=='male' else 0 )
log_loss(Y_train.iloc[validInds], pred_val_nn), f1_score(Y_train.iloc[validInds], np.argmax(pred_val_nn, axis = 1), average = "micro" )<train_model>
test_dataset = pd.get_dummies(test_dataset,prefix_sep = '__', columns = cat_columns )
rfc = RandomForestClassifier(random_state=0, n_estimators=1000, n_jobs=-1, verbose=1)
 rfc.fit(Xnn_train.iloc[trainInds], Y_train.iloc[trainInds] )<predict_on_test>
passengerID = test_dataset['PassengerId']
test_dataset.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1, inplace=True )
pred_val_rfc = rfc.predict_proba(Xnn_train.iloc[validInds])
 pred_test_rfc = rfc.predict_proba(Xnn_test )<compute_test_metric>
test_data_pred,z_test_data = forward(test_dataset,w1,b1,w2,b2)
 test_data_pred =(np.round(test_data_pred))
submission = pd.DataFrame({'PassengerId': passengerID, 'Survived':test_data_pred[0]} ).astype(int )
log_loss(Y_train.iloc[validInds], pred_val_rfc), f1_score(Y_train.iloc[validInds], np.argmax(pred_val_rfc, axis = 1), average = "micro" )<train_model>
submission.to_csv('submission.csv', index=False)