Kaggle competition: Titanic - Machine Learning from Disaster
def remove_double_links(series):
rgx = '(https?(?!.+https? ).+)'
tmp = series.copy()
tmp['article_link'] = tmp['article_link'].str.extract(rgx)
 return tmp<feature_engineering>
df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv' )
def get_source(series):
rgx = '(( ?!https?:)\/\/.+?\.. +?\/)'
tmp = series.copy()
tmp['article_link'] = tmp['article_link'].str.extract(rgx)
 tmp['article_link'] = tmp['article_link'].str.strip(to_strip="/w")
 tmp['article_link'] = tmp['article_link'].str.strip(to_strip=r'^\.')
 return tmp<create_dataframe>
df['Sex'].value_counts()
def rename_article_link(dataframe):
return dataframe.rename(columns={'article_link': 'source'} )<feature_engineering>
complete_df = pd.concat([df, test_df] )
train['headline_and_source'] = train.source + ' ' + train.headline<split>
complete_df.isnull().sum()
X_train, X_test, y_train, y_test = train_test_split(train['headline_and_source'], train.is_sarcastic)
 sarcasm_classfication.fit(X_train, y_train )<compute_train_metric>
complete_df[complete_df['Embarked'].isnull() ]
print(roc_auc_score(y_train, sarcasm_classfication.decision_function(X_train)))
 print(roc_auc_score(y_test, sarcasm_classfication.decision_function(X_test)))
 cross_validate(sarcasm_classfication, train.headline_and_source, train.is_sarcastic, cv=25, scoring='roc_auc', return_train_score=True )<train_model>
complete_df['Embarked'] = complete_df['Embarked'].fillna('C' )
tmp = train.copy()
X_tmp_train, X_tmp_test, y_tmp_train, y_tmp_test = train_test_split(tmp.source, tmp.is_sarcastic)
 tmp_model = sarcasm_classfication
tmp_model.fit(X_tmp_train, y_tmp_train)
 print(roc_auc_score(y_tmp_train, tmp_model.decision_function(X_tmp_train)))
 print(roc_auc_score(y_tmp_test, tmp_model.decision_function(X_tmp_test)))
 cross_validate(sarcasm_classfication, X_tmp_train, y_tmp_train, cv=5, scoring='roc_auc', return_train_score=True )<find_best_model_class>
complete_df[complete_df['Fare'].isnull() ]
tmp = train.copy()
tmp = tmp.drop(columns=['ID', 'headline', 'headline_and_source'])
 tmp = pd.get_dummies(data=tmp, columns=['source'])
 X_tmp_train, X_tmp_test, y_tmp_train, y_tmp_test = train_test_split(tmp.drop(columns=['is_sarcastic']), tmp.is_sarcastic)
 tmp_model = RandomForestClassifier(n_jobs=-1, n_estimators=100)
 tmp_model.fit(X_tmp_train, y_tmp_train)
 prob = tmp_model.predict_proba(X_tmp_train)
 prob = prob[:, 1]
print(roc_auc_score(y_tmp_train, prob))
prob = tmp_model.predict_proba(X_tmp_test)
 prob = prob[:, 1]
print(roc_auc_score(y_tmp_test, prob))
cross_validate(tmp_model, X_tmp_train, y_tmp_train, cv=5, scoring='roc_auc', return_train_score=True )<drop_column>
complete_df['Fare'] = complete_df.groupby('Pclass')['Fare'].transform(lambda val: val.fillna(val.median()))
valid = remove_double_links(valid)
 valid = get_source(valid)
 valid = valid.rename(columns={'article_link': 'source'} )<save_to_csv>
complete_df.loc[complete_df['Sex']=='female','Age'] = complete_df[complete_df['Sex']=='female']['Age'].transform(lambda val: val.fillna(val.median()))
complete_df.loc[complete_df['Sex']=='male', 'Age'] = complete_df[ complete_df['Sex']=='male' ]['Age'].transform(lambda val: val.fillna(val.median()))
predicted = sarcasm_classfication.predict(valid.source)
 prediction_dataframe = pd.DataFrame({'ID': valid.ID, 'is_sarcastic': predicted})
 prediction_dataframe.to_csv('output.csv', index=False )<import_modules>
complete_df.isnull().sum()
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, roc_auc_score
from sklearn.tree import DecisionTreeClassifier<load_from_csv>
X = complete_df[:891].drop(['PassengerId', 'Survived', 'Name', 'Ticket', 'Cabin'] ,axis=1)
 X
X_train = pd.read_csv(".. /input/nctu-bdalab-2020-onboard/data_train.csv", index_col="index")
 Y_train = pd.read_csv(".. /input/nctu-bdalab-2020-onboard/answer_train.csv", index_col="index")
 X_train, X_test, Y_train, Y_test = train_test_split(X_train, Y_train, test_size=0.2 )<train_model>
X = pd.get_dummies(X)
 X
model = DecisionTreeClassifier()
model.fit(X_train, Y_train )<compute_train_metric>
y = complete_df[:891]['Survived']
train_pred = model.predict(X_train)
 print(classification_report(Y_train, train_pred))
print(roc_auc_score(Y_train, train_pred))<predict_on_test>
from sklearn.model_selection import train_test_split
test_pred = model.predict(X_test)
 print(classification_report(Y_test, test_pred))
print(roc_auc_score(Y_test, test_pred))<save_to_csv>
from sklearn.model_selection import train_test_split
X_submit = pd.read_csv(".. /input/nctu-bdalab-2020-onboard/data_test.csv", index_col="index")
 pred = model.predict(X_submit)
 pred_df = pd.DataFrame(data={'default.payment.next.month': pred} ).reset_index()
pred_df.to_csv("./pred.csv", index=False )<define_variables>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42 )
nrows = None<load_from_csv>
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
%%time
train = pd.read_csv(".. /input/ykc-cup-2nd/train.csv", nrows = nrows)
 test = pd.read_csv(".. /input/ykc-cup-2nd/test.csv", nrows = nrows)
 sub = pd.read_csv(".. /input/ykc-cup-2nd/sample_submission.csv" )<concatenate>
param_grid = {'max_depth':[4,5,6,7,8,9,10]}
df = pd.concat([train, test])
 df = df.reset_index(drop = True)
 df.shape<feature_engineering>
forest = RandomForestClassifier(random_state=42)
 grid = GridSearchCV(forest, param_grid, cv=10)
 grid.fit(X_train, y_train )
def clear(x, punct, rep = ""):
for p in punct:
x = x.replace(p, rep)
 return x
punct = ["ñ","\",<categorify>
grid.best_params_
unuse_words = defaultdict(int)
 def get_vec(x):
vs = []
for xx in x:
if len(vs)>= n_length:
break
try:
vs.append(model.wv[xx])
 except:
flg = False
for i in range(1, len(xx)) :
try:
v1 = model.wv[xx[:i]]
v2 = model.wv[xx[i:]]
vs.append(v1)
 vs.append(v2)
 flg = True
break
except:
pass
if flg == False:
for i in range(1, len(xx)) :
try:
v1 = model.wv[xx[:i]]
vs.append(v1)
 break
except:
pass
if len(vs)>= n_length:
break
for i in range(1, len(xx)) :
try:
v2 = model.wv[xx[i:]]
vs.append(v2)
 break
except:
pass
unuse_words[xx] += 1
if len(vs)< n_length:
for i in range(n_length - len(vs)) :
vs.append(np.zeros(model.vector_size))
if len(vs)> n_length:
vs = vs[:n_length]
vs = np.array(vs)
 if len(vs)== 0:
vs = np.zeros([1, model.vector_size])
 return vs<load_pretrained>
from sklearn.metrics import accuracy_score, classification_report
%%time
model = pd.read_pickle(".. /input/ykc-cup-2nd-save-fasttext/fasttext_gensim_model.pkl" )<categorify>
test_predictions = grid.predict(X_test)
 print(accuracy_score(y_test, test_predictions))
def to_vec(x):
vs = get_vec(x)
 return vs
n_length = 15
vecs = df["product_name"].apply(lambda x : to_vec(x))
vecs = np.stack(vecs)
 del model<sort_values>
print(classification_report(y_test, test_predictions))
sorted(unuse_words.items() , key=lambda x: x[1], reverse = True)[:100]<categorify>
X_final = complete_df[891:].drop(['PassengerId', 'Survived', 'Name', 'Ticket', 'Cabin'] ,axis=1)
 X_final = pd.get_dummies(X_final )
additional_feats = []
additional_words = [k for k, v in unuse_words.items() if v >= 5 and len(k)> 2]
print(additional_words)
 additional_word_cols = []
for w in additional_words:
c = f"is_{w}"
df[c] = df["product_name"].apply(lambda x : w in x)
 additional_word_cols.append(c)
 additional_feats.append(df[additional_word_cols])
 additional_feats.append(pd.get_dummies(df["order_dow_mode"]))
additional_feats.append(pd.get_dummies(df["order_hour_of_day_mode"]))
additional_feats.append(StandardScaler().fit_transform(df[["order_rate"]]))
additional_feats = np.hstack(additional_feats)
 additional_feats.shape<prepare_x_and_y>
forest = RandomForestClassifier(max_depth=6, random_state=42)
 forest.fit(X,y)
 final_preds = forest.predict(X_final )
target = "department_id"
train_wv = vecs[~df[target].isna() ]
test_wv = vecs[df[target].isna() ]
train_x = additional_feats[~df[target].isna() ]
test_x = additional_feats[df[target].isna() ]
y_train = df[~df[target].isna() ][target].values
y_train_ohe = to_categorical(y_train, num_classes=None)
 del vecs, additional_feats, df
train_wv.shape, test_wv.shape, train_x.shape, test_x.shape, y_train.shape, y_train_ohe.shape<import_modules>
submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 submission
from tensorflow import keras
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.layers import *
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.optimizers import Adam, SGD, Nadam
from tensorflow.keras import backend as K 
<predict_on_test>
submission['Survived'] = final_preds
submission
class F1(keras.callbacks.Callback):
def __init__(self, model, inputs, targets, epoch_max):
self.model = model
self.inputs = inputs
self.targets = targets
self.epoch = 0
self.epoch_max = epoch_max
self.eval_freq = 5
def on_epoch_end(self, epoch, logs):
if(self.epoch % self.eval_freq == 0)or(self.epoch_max - self.epoch < 3):
p = self.model.predict(self.inputs)
 score = get_score(self.targets, p)
 print(f"{self.epoch}, : ")
 print(score)
 self.epoch += 1
def get_score(t, p):
score = {
"logloss" : log_loss(t, p),
"f1_micro" : f1_score(t, np.argmax(p, axis = 1), average = "micro")}
return score<compute_test_metric>
submission['Survived'] = submission['Survived'].astype(int)
 submission
def get_model(param):
inp_wv = Input(( n_length, 300))
inp = Input(( train_x.shape[1],))
mask = Lambda(lambda x : 1 /(1 + K.exp(-100 * K.std(x, axis = 2, keepdims = True))))(inp_wv)
 def calc_mean(x, mask, axis = 1, keepdims = False):
return K.sum(x * mask, axis = axis, keepdims = keepdims)/ K.sum(mask, axis = axis, keepdims = keepdims)
 aggs = []
mean = Lambda(lambda x, mask : calc_mean(x, mask, axis = 1))(inp_wv, mask)
 maximum = Lambda(lambda x : K.max(x, axis = 1))(inp_wv)
 aggs.append(BatchNormalization()(mean))
aggs.append(BatchNormalization()(maximum))
h = inp_wv
for f in param["n_units"]:
h = keras.layers.Dense(f, "relu" )(h)
 h = Dropout(param["p_drop"] )(h)
 mean = Lambda(lambda x, mask : calc_mean(x, mask, axis = 1))(h, mask)
 maximum = Lambda(lambda x : K.max(x, axis = 1))(h)
 aggs.append(BatchNormalization()(mean))
aggs.append(BatchNormalization()(maximum))
h = Concatenate()(aggs)
 h = Dropout(param["p_drop_fc"] )(h)
 h = Concatenate()([h, inp])
 for f in param["n_units_fc"]:
h = Dense(f, "relu" )(h)
 h = Dropout(param["p_drop_fc"] )(h)
 out = Dense(21, "softmax" )(h)
 model = keras.models.Model(inputs = [inp_wv, inp], outputs = out)
 return model 
<train_model>
submission.to_csv('submission.csv', index=False )
def trainNN(param, x_tr, y_tr, x_va, y_va, y_va_label, verbose = 0):
param_base = {
"lr" : 1e-3,
"lr_min" : 1e-5,
"lr_reduce_factor" : 0.5,
"lr_reduce_patience" : 5,
"epochs" : 100,
}
param.update(param_base)
 param["n_units"] = [int(param["n_unit"] *(param["n_unit_scale"] ** k)) for k in range(param["n_layer"])]
param["n_units_fc"] = [int(param["n_unit_fc"] *(param["n_unit_fc_scale"] ** k)) for k in range(param["n_layer_fc"])]
param["n_units_fc"] = [n for n in param["n_units_fc"] if n >=64]
print(param)
 model = get_model(param)
 model.compile(loss=keras.losses.CategoricalCrossentropy() , optimizer=keras.optimizers.Adam(learning_rate=param["lr"], decay = param["decay"]))
cb_schedule = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=param["lr_reduce_factor"], patience=param["lr_reduce_patience"],
verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=param["lr_min"])
 cb_earlystop = keras.callbacks.EarlyStopping(monitor='val_loss',patience=10, restore_best_weights=True)
 cb_save = keras.callbacks.ModelCheckpoint(f'model.hdf5', save_best_only=True, monitor='val_loss', mode='min')
 callbacks = [cb_schedule, cb_earlystop, cb_save, F1(model, x_va, y_va_label, param["epochs"])]
model.fit(x_tr, y_tr,
batch_size = param["batch_size"],
epochs=param["epochs"],
callbacks = callbacks,
validation_data=(x_va, y_va), verbose = verbose)
 loss_history = model.history.history["val_loss"]
return model<split>
submission.to_csv('submission.csv', index=False )