Kaggle competition: Natural Language Processing with Disaster Tweets
train_df.loc[train_df['ps_car_12'] == -1, 'ps_car_12'] = train_df[train_df['ps_car_12'] != -1]['ps_car_12'].mean()<predict_on_test>
import tensorflow as tf
from tensorflow import keras
import numpy as np
import pandas as pd
import tensorflow_hub as hub
from sklearn.model_selection import train_test_split
feature_mod = linear_model.LinearRegression()
feature_xgb = XGBRegressor(n_estimators=200)
 corr_list = ['ps_car_12', 'ps_car_13']
feature_mod.fit(train_df[train_df['ps_car_14'] != -1][corr_list], train_df[train_df['ps_car_14'] != -1]['ps_car_14'])
 train_df.loc[train_df['ps_car_14'] == -1, 'ps_car_14'] = feature_mod.predict(train_df[train_df['ps_car_14'] == -1][corr_list])
 test_df.loc[test_df['ps_car_14'] == -1, 'ps_car_14'] = feature_mod.predict(test_df[test_df['ps_car_14'] == -1][corr_list])
 corr_list = ['ps_reg_02', 'ps_car_13', 'ps_car_12', 'ps_ind_01']
feature_xgb.fit(train_df[train_df['ps_reg_03'] != -1][corr_list], train_df[train_df['ps_reg_03'] != -1]['ps_reg_03'])
 train_df.loc[train_df['ps_reg_03'] == -1, 'ps_reg_03'] = feature_xgb.predict(train_df[train_df['ps_reg_03'] == -1][corr_list])
 test_df.loc[test_df['ps_reg_03'] == -1, 'ps_reg_03'] = feature_xgb.predict(test_df[test_df['ps_reg_03'] == -1][corr_list] )<categorify>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
cat_cols = [x for x in potential_features if '_cat' in x ]
cat_cols = list(set(cat_cols)- set(['ps_car_11_cat']))
train_df = pd.get_dummies(data=train_df, columns=cat_cols, drop_first=True)
 test_df = pd.get_dummies(data=test_df, columns=cat_cols, drop_first=True )<data_type_conversions>
X_train, X_test, y_train, y_test = train_test_split(train.text, train.target, test_size=0.2,random_state =42 )
columns_needed = max(train_df['ps_car_11_cat'].max().item().bit_length() ,
test_df['ps_car_11_cat'].max().item().bit_length())
 format_string = '0>'+ str(columns_needed)+ 'b'
bin_cols = train_df['ps_car_11_cat'].apply(lambda x: format(x, format_string)).str.extractall('(\d)' ).unstack().astype(np.int8 ).add_prefix('ps_car_11_cat_b')
 bin_cols.columns = bin_cols.columns.droplevel()
train_df = pd.concat([train_df, bin_cols], axis=1)
 train_df.drop('ps_car_11_cat', inplace=True, axis=1)
 bin_cols = test_df['ps_car_11_cat'].apply(lambda x: format(x, format_string)).str.extractall('(\d)' ).unstack().astype(np.int8 ).add_prefix('ps_car_11_cat_b')
 bin_cols.columns = bin_cols.columns.droplevel()
test_df = pd.concat([test_df, bin_cols], axis=1)
 test_df.drop('ps_car_11_cat', inplace=True, axis=1 )<choose_model_class>
X_test_lst = test.text.to_list()
MAX_ROUNDS = 400
LEARNING_RATE = 0.07
model = XGBClassifier(
 n_estimators=MAX_ROUNDS,
max_depth=4,
objective="binary:logistic",
learning_rate=LEARNING_RATE,
subsample=.8,
min_child_weight=6,
colsample_bytree=.8,
gamma=10,
reg_alpha=8,
reg_lambda=1.3
)<prepare_x_and_y>
train_dataset = tf.data.Dataset.from_tensor_slices(( X_train, y_train))
K = 5
kf = KFold(n_splits = K, random_state = 1, shuffle = True)
 y_valid_preds = 0 * train_df['target']
y_test_preds = 0
for i,(train_index, valid_index)in enumerate(kf.split(train_df)) :
print("
Starting fold {}".format(i+1))
start = time.time()
y_train = train_df['target'].loc[train_index]
X_train = train_df.drop('target', axis=1 ).loc[train_index]
y_valid = train_df['target'].loc[valid_index]
X_valid = train_df.drop('target', axis=1 ).loc[valid_index]
fit_model = model.fit(X_train, y_train, verbose=True)
 preds = fit_model.predict_proba(X_valid)[:, 1]
gini = eval_gini(y_valid, preds)
 y_valid_preds.loc[valid_index] = preds
y_test_preds += fit_model.predict_proba(test_df)[:, 1]
print("Fold {} Gini score: {}".format(i, gini))
print("Completed fold {} in {:.2f} minutes
".format(i+1,(time.time() - start)/60))
y_test_preds /= K
print("
Gini for full training set:")
 eval_gini(train_df['target'], y_valid_preds )<save_to_csv>
validation_dataset = tf.data.Dataset.from_tensor_slices(( X_test, y_test))
sub = pd.DataFrame()
sub['id'] = test_ids
sub['target'] = y_test_preds
sub.to_csv('xgb_submit_3.csv', float_format='%.6f', index=False )<set_options>
train_examples_batch, train_labels_batch = next(iter(train_dataset.batch(10)))
 train_examples_batch
warnings.filterwarnings('ignore' )<load_from_csv>
embedding = "https://tfhub.dev/google/universal-sentence-encoder-large/5"
hub_layer = hub.KerasLayer(embedding, input_shape=[],
dtype=tf.string, trainable=False)
 hub_layer(tf.convert_to_tensor(train_examples_batch[:3]))
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<categorify>
model = tf.keras.Sequential()
model.add(hub_layer)
 model.add(tf.keras.layers.Dense(128, activation='relu'))
model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
model.summary()
id_test = test['id'].values
target_train = train['target'].values
train = train.drop(['target','id'], axis = 1)
 test = test.drop(['id'], axis = 1)
 col_to_drop = train.columns[train.columns.str.startswith('ps_calc_')]
train = train.drop(col_to_drop, axis=1)
 test = test.drop(col_to_drop, axis=1)
 train = train.replace(-1, np.nan)
 test = test.replace(-1, np.nan)
 cat_features = [a for a in train.columns if a.endswith('cat')]
for column in cat_features:
temp = pd.get_dummies(pd.Series(train[column]))
train = pd.concat([train,temp],axis=1)
 train = train.drop([column],axis=1)
 for column in cat_features:
temp = pd.get_dummies(pd.Series(test[column]))
test = pd.concat([test,temp],axis=1)
 test = test.drop([column],axis=1)
 print(train.values.shape, test.values.shape )<choose_model_class>
model.compile(optimizer='adam',
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'] )
class Ensemble(object):
def __init__(self, mode, n_splits, stacker_2, stacker_1, base_models):
self.mode = mode
self.n_splits = n_splits
self.stacker_2 = stacker_2
self.stacker_1 = stacker_1
self.base_models = base_models
def fit_predict(self, X, y, T):
X = np.array(X)
 y = np.array(y)
 T = np.array(T)
 folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True,
random_state=2016 ).split(X, y))
OOF_columns = []
S_train = np.zeros(( X.shape[0], len(self.base_models)))
 S_test = np.zeros(( T.shape[0], len(self.base_models)))
 for i, clf in enumerate(self.base_models):
S_test_i = np.zeros(( T.shape[0], self.n_splits))
for j,(train_idx, test_idx)in enumerate(folds):
X_train = X[train_idx]
y_train = y[train_idx]
X_holdout = X[test_idx]
print("Fit %s_%d fold %d" %(str(clf ).split("(")[0], i+1, j+1))
clf.fit(X_train, y_train)
 S_train[test_idx, i] = clf.predict_proba(X_holdout)[:,1]
S_test_i[:, j] = clf.predict_proba(T)[:,1]
S_test[:, i] = S_test_i.mean(axis=1)
 print(" Base model_%d score: %.5f
" %(i+1, roc_auc_score(y, S_train[:,i])))
 OOF_columns.append('Base model_'+str(i+1))
OOF_S_train = pd.DataFrame(S_train, columns = OOF_columns)
 print('
')
 print('Correlation between out-of-fold predictions from Base models:')
 print('
')
 print(OOF_S_train.corr())
 print('
')
 if self.mode==1:
folds_2 = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True,
random_state=2016 ).split(S_train, y))
OOF_columns = []
S_train_2 = np.zeros(( S_train.shape[0], len(self.stacker_1)))
 S_test_2 = np.zeros(( S_test.shape[0], len(self.stacker_1)))
 for i, clf in enumerate(self.stacker_1):
S_test_i_2 = np.zeros(( S_test.shape[0], self.n_splits))
for j,(train_idx, test_idx)in enumerate(folds_2):
X_train_2 = S_train[train_idx]
y_train_2 = y[train_idx]
X_holdout_2 = S_train[test_idx]
print("Fit %s_%d fold %d" %(str(clf ).split("(")[0], i+1, j+1))
clf.fit(X_train_2, y_train_2)
 S_train_2[test_idx, i] = clf.predict_proba(X_holdout_2)[:,1]
S_test_i_2[:, j] = clf.predict_proba(S_test)[:,1]
S_test_2[:, i] = S_test_i_2.mean(axis=1)
 print(" 1st level model_%d score: %.5f
"%(i+1,
roc_auc_score(y, S_train_2.mean(axis=1))))
OOF_columns.append('1st level model_'+str(i+1))
OOF_S_train = pd.DataFrame(S_train_2, columns = OOF_columns)
 print('
')
 print('Correlation between out-of-fold predictions from 1st level models:')
 print('
')
 print(OOF_S_train.corr())
 print('
')
 if self.mode==2:
WOC_columns = []
S_train_2 = np.zeros(( S_train.shape[0], len(self.stacker_1)))
 S_test_2 = np.zeros(( S_test.shape[0], len(self.stacker_1)))
 for i, clf in enumerate(self.stacker_1):
S_train_i_2= np.zeros(( S_train.shape[0], S_train.shape[1]))
S_test_i_2 = np.zeros(( S_test.shape[0], S_train.shape[1]))
for j in range(S_train.shape[1]):
S_tr = S_train[:,np.arange(S_train.shape[1])!=j]
S_te = S_test[:,np.arange(S_test.shape[1])!=j]
print("Fit %s_%d subset %d" %(str(clf ).split("(")[0], i+1, j+1))
clf.fit(S_tr, y)
 S_train_i_2[:, j] = clf.predict_proba(S_tr)[:,1]
S_test_i_2[:, j] = clf.predict_proba(S_te)[:,1]
S_train_2[:, i] = S_train_i_2.mean(axis=1)
 S_test_2[:, i] = S_test_i_2.mean(axis=1)
 print(" 1st level model_%d score: %.5f
"%(i+1,
roc_auc_score(y, S_train_2.mean(axis=1))))
WOC_columns.append('1st level model_'+str(i+1))
WOC_S_train = pd.DataFrame(S_train_2, columns = WOC_columns)
 print('
')
 print('Correlation between without-one-column predictions from 1st level models:')
 print('
')
 print(WOC_S_train.corr())
 print('
')
 try:
num_models = len(self.stacker_2)
 if self.stacker_2==(et_model):
num_models=1
except TypeError:
num_models = len([self.stacker_2])
 if num_models==1:
print("Fit %s for final
" %(str(self.stacker_2 ).split("(")[0]))
self.stacker_2.fit(S_train_2, y)
 stack_res = self.stacker_2.predict_proba(S_test_2)[:,1]
stack_score = self.stacker_2.predict_proba(S_train_2)[:,1]
print("2nd level model final score: %.5f" %(roc_auc_score(y, stack_score)))
 else:
F_columns = []
stack_score = np.zeros(( S_train_2.shape[0], len(self.stacker_2)))
 res = np.zeros(( S_test_2.shape[0], len(self.stacker_2)))
 for i, clf in enumerate(self.stacker_2):
print("Fit %s_%d" %(str(clf ).split("(")[0], i+1))
clf.fit(S_train_2, y)
 stack_score[:, i] = clf.predict_proba(S_train_2)[:,1]
print(" 2nd level model_%d score: %.5f
"%(i+1,roc_auc_score(y, stack_score[:, i])))
 res[:, i] = clf.predict_proba(S_test_2)[:,1]
F_columns.append('2nd level model_'+str(i+1))
F_S_train = pd.DataFrame(stack_score, columns = F_columns)
 print('
')
 print('Correlation between final predictions from 2nd level models:')
 print('
')
 print(F_S_train.corr())
 print('
')
 stack_res = res.mean(axis=1)
 print("2nd level models final score: %.5f" %(roc_auc_score(y, stack_score.mean(axis=1))))
return stack_res<init_hyperparams>
history = model.fit(train_dataset.shuffle(10000 ).batch(512),
epochs=40,
validation_data=validation_dataset.batch(512),
verbose=1 )
lgb_params_1 = {
'learning_rate': 0.02,
'n_estimators': 750,
'subsample': 0.8,
'subsample_freq': 10,
'colsample_bytree': 0.8,
'max_bin': 10,
'min_child_samples': 500,
'seed': 99
}
lgb_params_2 = {
'learning_rate': 0.02,
'n_estimators': 1200,
'subsample': 0.7,
'subsample_freq': 2,
'colsample_bytree': 0.3,
'num_leaves': 16,
'seed': 99
}
lgb_params_3 = {
'learning_rate': 0.02,
'n_estimators': 475,
'subsample': 0.4,
'subsample_freq': 1,
'colsample_bytree': 0.9,
'num_leaves': 28,
'max_bin': 10,
'min_child_samples': 700,
'seed': 99
}<choose_model_class>
y_pred_list = []
for text in X_test_lst:
y_pred = model.predict(tf.expand_dims(text, 0))
y_pred_list.append(y_pred.tolist() [0] )
lgb_model_1 = LGBMClassifier(**lgb_params_1)
 lgb_model_2 = LGBMClassifier(**lgb_params_2)
 lgb_model_3 = LGBMClassifier(**lgb_params_3 )<choose_model_class>
y_pred_boolean = tf.greater(y_pred_list,.7 )
log_model = LogisticRegression()
et_model = ExtraTreesClassifier(n_estimators=100, max_depth=6, min_samples_split=10, random_state=10)
 mlp_model = MLPClassifier(max_iter=7, random_state=42 )<predict_on_test>
final=[]
for data in y_pred_boolean:
if data == True:
final.append(1)
 else:
final.append(0 )
stack = Ensemble(mode=2,
n_splits=3,
stacker_2 =(log_model, et_model),
stacker_1 =(log_model, et_model, mlp_model),
base_models =(lgb_model_1, lgb_model_2, lgb_model_3))
y_pred = stack.fit_predict(train, target_train, test )<save_to_csv>
output = pd.DataFrame({'id':test['id'],'target':final} )
<import_modules><EOS> 
output.to_csv("output.csv",index=False )