Kaggle competition: Titanic - Machine Learning from Disaster
train = pd.read_csv(".. /input/ykc-cup-2nd/train.csv")
 test = pd.read_csv(".. /input/ykc-cup-2nd/test.csv")
 sub = pd.read_csv(".. /input/ykc-cup-2nd/sample_submission.csv")
 train.shape, test.shape, sub.shape<concatenate>
dataset_train = pd.read_csv("/kaggle/input/titanic/train.csv")
 dataset_train.head()
df = pd.concat([train, test])
 df = df.reset_index(drop=True)
 df.shape<choose_model_class>
dataset_test = pd.read_csv("/kaggle/input/titanic/test.csv")
 dataset_test.head()
target = "department_id"
n_split = 5
kfold = StratifiedKFold(n_splits=n_split, shuffle = True, random_state=42 )<feature_engineering>
gender = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 gender.head()
df["product_name"] = df["product_name"].apply(lambda words : words.lower().replace(",", "" ).replace("&", "" ).split(" "))
df.head()<define_variables>
dataset_train.isnull().sum()
model_names = ["crawl-300d-2M", "crawl-300d-2M-subword", "wiki-news-300d-1M", "wiki-news-300d-1M-subword"]<load_pretrained>
dataset_test.isnull().sum()
fasttext_pretrain_cols = []
unused_words = defaultdict(int)
 for model_name in model_names:
model = pd.read_pickle(f".. /input/fasttext/{model_name}.pkl")
 def to_mean_vec(x, model):
v = np.zeros(model.vector_size)
 for w in x:
try:
v += model[w]
except:
unused_words[w] += 1
v = v /(np.sqrt(np.sum(v ** 2)) + 1e-16)
 return v
def to_max_vec(x, model):
v = np.zeros(model.vector_size)
 for w in x:
try:
v = np.maximum(v, model[w])
 except:
pass
return v
mean_vecs = df["product_name"].apply(lambda x : to_mean_vec(x, model))
mean_vecs = np.vstack(mean_vecs)
 cols = [f"fasttext_pretrain_{model_name}_mean_vec{k}" for k in range(mean_vecs.shape[1])]
fasttext_pretrain_cols += cols
mean_vec_df = pd.DataFrame(mean_vecs, columns=cols)
 df = pd.concat([df, mean_vec_df], axis = 1)
 max_vecs = df["product_name"].apply(lambda x : to_max_vec(x, model))
max_vecs = np.vstack(max_vecs)
 cols = [f"fasttext_pretrain_{model_name}_max_vec{k}" for k in range(max_vecs.shape[1])]
fasttext_pretrain_cols += cols
max_vec_df = pd.DataFrame(max_vecs, columns=cols)
 df = pd.concat([df, max_vec_df], axis = 1)
 df.head()<split>
dataset_train['Age'] = dataset_train['Age'].fillna(dataset_train['Age'].mean())
 mean_train = int(dataset_train['Age'].mean())
 mean_test = int(dataset_test['Age'].mean())
 dataset_train['Embarked'] = dataset_train['Embarked'].fillna(np.random.choice(['C','Q','S']))
dataset_test['Age'] = dataset_test['Age'].fillna(dataset_test['Age'].mean())
 dataset_test['Fare'] = dataset_test['Fare'].fillna(dataset_test['Fare'].mean() )
train = df[~df[target].isna() ]
test = df[df[target].isna() ]<feature_engineering>
temp_title.value_counts()
def to_weighted_count_vec(x, word_sets):
v = np.zeros(21)
 for w in x:
hits = []
for i in range(21):
if w in word_sets[i]:
hits.append(i)
 for i in hits:
v[i] += 1.0 / len(hits)
 return v
weighted_count_cols = [f"weighted_count_vec{k}" for k in range(21)]<create_dataframe>
fare_cabin = dataset_train[['Cabin', 'Fare']].groupby(by = 'Cabin' ).mean()
fare_cabin = fare_cabin.sort_values(by = 'Fare', ascending = False)
 fare_cabin = fare_cabin[:16][:]
fare_cabin = fare_cabin.reset_index()
fare_cabin = fare_cabin.sort_values(by = 'Cabin', ascending = True)
 def label_encoder(string):
num = re.findall('\d+', string)
 alpha = string[0]
if len(num)>1:
label = label = alpha + '(' + '-'.join(num)+ ')'
else:
label = alpha + '-'.join(num)
 return label
fare_cabin['Cabin'] = fare_cabin['Cabin'].apply(label_encoder )
train_weighted_count = pd.DataFrame(index=train.index, columns=weighted_count_cols, dtype=np.float32)
 for i_fold,(train_idx, valid_idx)in enumerate(kfold.split(train, train[target])) :
tr = train.loc[train_idx]
va = train.loc[valid_idx]
word_sets = [set(sum(tr[tr["department_id"] == i]["product_name"], [])) for i in range(21)]
vecs = va["product_name"].apply(lambda x : to_weighted_count_vec(x, word_sets))
vecs = np.vstack(vecs)
 vec_df = pd.DataFrame(vecs, index=va.index, columns=weighted_count_cols)
 train_weighted_count.loc[valid_idx, :] = vec_df
train = pd.concat([train, train_weighted_count], axis=1 )<create_dataframe>
dataset_train['Ticket'] = dataset_train['Ticket'].apply(lambda x: len(x))
dataset_train['Title'] = dataset_train['Name'].apply(get_title)
 dataset_train['Title'] = dataset_train.apply(replace_titles, axis = 1)
 drop_cols = ['PassengerId','Name','Cabin', 'Title']
encode_cols = ['Sex','Embarked', 'Title']
encode_after = pd.get_dummies(dataset_train[encode_cols])
 fin_data = dataset_train.copy()
fin_data = fin_data.drop(drop_cols, axis = 1)
 fin_data = pd.concat([fin_data, encode_after], axis = 1)
 print(fin_data.columns)
 fin_data.drop(['Sex', 'Embarked'], axis = 1, inplace = True )
test_weighted_count = pd.DataFrame(index=test.index, columns=weighted_count_cols, dtype=np.float32)
 word_sets = [set(sum(train[train["department_id"] == i]["product_name"], [])) for i in range(21)]
vecs = test["product_name"].apply(lambda x : to_weighted_count_vec(x, word_sets))
vecs = np.vstack(vecs)
 test_weighted_count.loc[:, :] = pd.DataFrame(vecs, index=test.index, columns=weighted_count_cols)
 test = pd.concat([test, test_weighted_count], axis=1 )<define_variables>
dataset_test['Title'] = dataset_test['Name'].apply(get_title)
 dataset_test['Ticket'] = dataset_test['Ticket'].apply(lambda x: str(x))
dataset_test['Ticket'] = dataset_test['Ticket'].apply(lambda x: len(x))
dataset_test['Title'] = dataset_test.apply(replace_titles, axis = 1)
 encode_cols_teset = pd.get_dummies(dataset_test[encode_cols])
 fin_data_test = dataset_test.copy()
fin_data_test = fin_data_test.drop(['PassengerId','Name','Cabin', 'Embarked','Sex'],axis =1)
 fin_data_test = pd.concat([fin_data_test, encode_cols_teset], axis = 1)
 fin_data_test = fin_data_test[[ 'Pclass', 'Age', 'SibSp', 'Parch', 'Ticket', 'Fare',
'Sex_female', 'Sex_male', 'Embarked_C', 'Embarked_Q',
'Embarked_S', 'Title_Master', 'Title_Miss', 'Title_Mr',
'Title_Mrs']]
features = fasttext_pretrain_cols + weighted_count_cols + ["order_rate", "order_dow_mode", "order_hour_of_day_mode"]<drop_column>
X_train = fin_data.values[:,1:]
Y_train = fin_data.values[:,0]
X_test = fin_data_test.values[:,:]
Y_test = gender.values[:,1:]
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<train_model>
params = {'penalty':['l1','l2'], 'C':[0.01,0.1,1,10,100]}
lr = LogisticRegression(solver = 'liblinear')
 grid = GridSearchCV(lr, param_grid=params, scoring ='f1', cv = 10, n_jobs=-1)
 grid.fit(X_train, Y_train)
 grid.best_params_
preds_test = []
scores = []
oof = np.zeros(( len(train), 21))
for i_fold,(train_idx, valid_idx)in enumerate(kfold.split(train, train[target])) :
print(f"--------fold {i_fold}-------")
 x_tr = train.loc[train_idx, features]
y_tr = train.loc[train_idx, target]
x_va = train.loc[valid_idx, features]
y_va = train.loc[valid_idx, target]
lgb = LGBMClassifier(objective='multiclass', colsample_bytree=0.2, n_estimators=500)
 lgb.fit(x_tr, y_tr, eval_set=[(x_va, y_va)], early_stopping_rounds=10, verbose=10)
 def predict_proba(x):
return lgb.predict_proba(x)
 pred_val = predict_proba(x_va)
 oof[valid_idx] += pred_val
score = {
"logloss" : log_loss(y_va, pred_val),
"f1_micro" : f1_score(y_va, np.argmax(pred_val, axis = 1), average = "micro")}
print(score)
 scores.append(score)
 pred_test = predict_proba(test[features])
 preds_test.append(pred_test )<create_dataframe>
lr = LogisticRegression(C = 1, penalty='l2', solver='liblinear')
 lr.fit(X_train, Y_train)
 predict_lr = lr.predict(X_test )
score_df = pd.DataFrame(scores)
 score_df<save_to_csv>
print("Accuracy = {0}%".format(round(accuracy_score(Y_test, predict_lr)*100, 2)))
 print(classification_report(Y_test, predict_lr))
print("Score = {0}".format(f1_score(Y_test, predict_lr)))
 print(confusion_matrix(Y_test, predict_lr))
oof_df = pd.DataFrame(oof)
 oof_df.to_csv("oof_lgb.csv", index = False )<save_to_csv>
rf = RandomForestClassifier()
params_rf = {'n_estimators':list(range(1,20)) , 'max_depth':list(range(1,10)) , 'criterion':['gini', 'entropy']}
grid_rf = GridSearchCV(rf, param_grid=params_rf, cv = 5, scoring='accuracy', n_jobs = -1)
 grid_rf.fit(X_train,Y_train)
 print('HyperParameter optimization')
 grid_rf.best_params_
for i in range(len(preds_test)) :
pred_df = pd.DataFrame(preds_test[i])
 pred_df.to_csv(f"pred_{i}_lgb.csv", index = False )<prepare_output>
rf = RandomForestClassifier(criterion = 'entropy', max_depth = 7, n_estimators = 16, bootstrap=False)
 rf.fit(X_train, Y_train)
 predict_rf = rf.predict(X_test)
 print("Accuracy Score = {} %".format(rf.score(X_test, Y_test)*100))
print(classification_report(Y_test, predict_rf))
print('F1 Score = {}'.format(f1_score(Y_test, predict_rf)) )
pred_test_final = np.array(preds_test ).mean(axis = 0)
 pred_test_final = np.argmax(pred_test_final, axis = 1 )<save_to_csv>
def model() :
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(120, activation = tf.nn.relu, kernel_initializer = 'normal'))
model.add(tf.keras.layers.Dense(120, activation = tf.nn.relu, kernel_initializer = 'normal'))
model.add(tf.keras.layers.Dense(120, activation = tf.nn.relu, kernel_initializer = 'normal'))
model.add(tf.keras.layers.Dense(1, activation = tf.nn.sigmoid))
model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 return model
sub["department_id"] = pred_test_final
sub.to_csv("submission_lgb.csv", index = False)
 sub.head()<set_options>
model = model()
model.fit(X_train, Y_train, epochs = 500, verbose = 0 )
%matplotlib inline
plt.style.use('seaborn' )<load_from_csv>
val_loss, val_acc = model.evaluate(X_train, Y_train )
print(check_output(["ls", ".. /input/adultb"] ).decode("utf8"))
df_train = pd.read_csv(".. /input/adultb/train_data.csv", na_values = '?')
 df_train.set_index('Id',inplace=True)
 df_train.head()<sort_values>
val_loss, val_acc = model.evaluate(X_test, Y_test )
total = df_train.isnull().sum().sort_values(ascending = False)
 percent =(( df_train.isnull().sum() /df_train.isnull().count())*100 ).sort_values(ascending = False)
 missing_data = pd.concat([total, percent], axis = 1, keys = ['Total', '%'])
 missing_data.head()<sort_values>
predict_nn = model.predict([X_test])
 predict_nn_fin = []
for i in range(len(predict_nn)) :
if predict_nn[i]>=0.7:
predict_nn_fin.append(1)
 else:
predict_nn_fin.append(0 )
total = df_train.isnull().sum().sort_values(ascending = False)
 percent =(( df_train.isnull().sum() /df_train.isnull().count())*100 ).sort_values(ascending = False)
 missing_data = pd.concat([total, percent], axis = 1, keys = ['Total', '%'])
 missing_data.head()<count_values>
print("Accuracy = {0}%".format(round(accuracy_score(Y_test, predict_nn_fin)*100, 2)))
 print(classification_report(Y_test, predict_nn_fin))
print("Score = {0}".format(f1_score(Y_test, predict_nn_fin)))
 print(confusion_matrix(Y_test, predict_nn_fin))
<define_variables><EOS>