Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<save_to_csv>
!pip install transformers
submission['time_to_failure'] = 0.8 * lgbm_predictions + 0.2 * prediction1
submission.to_csv('0.8lgbm_ensemble_0.2submission.csv' )<import_modules>
pd.set_option('max_colwidth', 500)
 pd.set_option('display.float_format', lambda x: '%.4f' % x)
 tf.random.set_seed(42)
 random.seed(42)
 warnings.filterwarnings("ignore" )
import time
import gc
import os
import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
from numba import jit<define_variables>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train.loc[train['id'].isin(ids_with_target_error),'target'] = 0
N_SPLITS = 4<load_from_csv>
%%time
tokenizer = BertTokenizer.from_pretrained('bert-large-uncased', do_lower_case=True)
 def greed_encode(data, max_len):
input_ids = []
attention_masks = []
for i in range(len(data.text)) :
encoded = tokenizer.encode_plus(data.text[i], add_special_tokens=True, max_length=max_len, pad_to_max_length=True)
 tok_len = sum(encoded['attention_mask'])
 if tok_len > max_len*.8:
all_encode = tokenizer.encode_plus(data.text[i], add_special_tokens=True)
 all_ids = all_encode['input_ids']
all_attention = all_encode['attention_mask']
max_len_half = int(max_len/2)
 input_ids.append(all_ids[:max_len_half] + all_ids[-max_len_half:])
 attention_masks.append(all_attention[:max_len_half] + all_attention[-max_len_half:])
 else:
input_ids.append(encoded['input_ids'])
 attention_masks.append(encoded['attention_mask'])
 return np.array(input_ids),np.array(attention_masks)
 train_input_ids,train_attention_masks = greed_encode(train,50)
 test_input_ids,test_attention_masks = greed_encode(test,50)
 y_train = train.target
src_data_path = ".. /input/porto-seguro-safe-driver-prediction"
df_train = pd.read_csv(os.path.join(src_data_path, "train.csv"))
df_test = pd.read_csv(os.path.join(src_data_path, "test.csv"))<drop_column>
N_SAMPLES = 8
def create_model(bert_model, MAX_LEN=50):
input_ids = layers.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_ids')
 attention_mask = layers.Input(shape=(MAX_LEN,), dtype=tf.int32, name='attention_mask')
 last_hidden_state, _ = bert_model({'input_ids': input_ids, 'attention_mask': attention_mask})
 last_hidden_state = Dropout(0.1 )(last_hidden_state)
 x_avg = layers.GlobalAveragePooling1D()(last_hidden_state)
 x_max = layers.GlobalMaxPooling1D()(last_hidden_state)
 x = layers.Concatenate()([x_avg, x_max])
 samples = []
for n in range(N_SAMPLES):
sample_mask = layers.Dense(64, activation='relu', name = f'dense_{n}')
 sample = layers.Dropout (.5 )(x)
 sample = sample_mask(sample)
 sample = layers.Dense(1, activation='sigmoid', name=f'sample_{n}' )(sample)
 samples.append(sample)
 output = layers.Average(name='output' )(samples)
 model = Model(inputs=[input_ids, attention_mask], outputs=output)
 model.compile(Adam(lr=1e-5), loss = BinaryCrossentropy(label_smoothing=0.1), metrics=['accuracy'])
 return model
id_test = df_test['id'].values
target_train = df_train['target'].values
df_train = df_train.drop(['target', 'id'], axis=1)
 df_test = df_test.drop(['id'], axis=1 )<drop_column>
bert_model = TFBertModel.from_pretrained('bert-large-uncased')
 model = create_model(bert_model)
 model.summary()
incomplete_columns = ['ps_car_03_cat', 'ps_car_05_cat']
df_train = df_train.drop(incomplete_columns, axis=1)
 df_test = df_test.drop(incomplete_columns, axis=1 )<categorify>
def lgb_svc_cv(model):
cls_layer_model = Model(model.input, outputs=model.get_layer(f'dense_1' ).output)
 X_train = cls_layer_model.predict([train_input_ids,train_attention_masks])
 X_test = cls_layer_model.predict([test_input_ids,test_attention_masks])
 y_train = train.target.values
N_FOLDS = 5
print(f'LGBM')
 folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=42)
 oof = np.zeros(len(X_train))
sub = np.zeros(len(X_test))
params = {'boosting_type': 'dart'}
for fold_,(train_idx, val_idx)in enumerate(folds.split(X_train, y_train)) :
X_train_cv, y_train_cv = pd.DataFrame(X_train ).loc[train_idx], pd.DataFrame(y_train ).loc[train_idx]
X_val, y_val = pd.DataFrame(X_train ).loc[val_idx], pd.DataFrame(y_train ).loc[val_idx]
train_data = lgb.Dataset(X_train_cv, label=y_train_cv)
 val_data = lgb.Dataset(X_val, label=y_val)
 watchlist = [train_data, val_data]
clf = lgb.train(params, train_set = train_data, valid_sets=watchlist)
 oof[val_idx] = clf.predict(X_val)
 sub += clf.predict(X_test)/folds.n_splits
sub_all_lgb = sub
print(accuracy_score(y_train, np.round(oof ).astype(int)) ,'
')
 print(f'SVC')
 folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=42)
 oof = np.zeros(len(X_train))
sub = np.zeros(len(X_test))
scores = [0 for _ in range(folds.n_splits)]
for fold_,(train_idx, val_idx)in enumerate(folds.split(X_train, y_train)) :
X_train_cv, y_train_cv = pd.DataFrame(X_train ).loc[train_idx], pd.DataFrame(y_train ).loc[train_idx]
X_val, y_val = pd.DataFrame(X_train ).loc[val_idx], pd.DataFrame(y_train ).loc[val_idx]
clf = SVC(kernel='rbf', C=1.75, gamma = 0.1, probability = True ).fit(X_train_cv, y_train_cv)
 oof[val_idx] = clf.predict_proba(X_val)[:,1]
sub += clf.predict_proba(X_test)[:,1]/folds.n_splits
sub_all_svc = sub
print(accuracy_score(y_train, np.round(oof ).astype(int)) ,'
')
 return sub_all_lgb, sub_all_svc
df_train = df_train.replace(-1, np.nan)
 df_test = df_test.replace(-1, np.nan )<categorify>
oof_preds = np.zeros(train_input_ids.shape[0])
 test_preds = np.zeros(test.shape[0])
 all_preds = pd.DataFrame()
fold_hist = {}
n_splits = 2
folds = KFold(n_splits=n_splits, shuffle=True, random_state=42)
 for i,(trn_idx, val_idx)in enumerate(folds.split(train_input_ids)) :
modelstart = time.time()
bert_model = TFBertModel.from_pretrained('bert-large-uncased')
 model = create_model(bert_model)
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.001, patience=5, verbose=1,
mode='min', baseline=None, restore_best_weights=True)
 def lr_sc(epoch):
return 1.5e-5/(epoch + 1)
 scheduler = LearningRateScheduler(lr_sc)
 history = model.fit(
 x=[train_input_ids[trn_idx], train_attention_masks[trn_idx]], y=y_train[trn_idx],
validation_data=([train_input_ids[val_idx], train_attention_masks[val_idx]], y_train[val_idx]),
epochs=3,
batch_size=16,
callbacks=[scheduler, es] 
)
 best_index = np.argmin(history.history['val_loss'])
 fold_hist[i] = history
oof_preds[val_idx] = model.predict([train_input_ids[val_idx], train_attention_masks[val_idx]] ).ravel()
all_preds[str(i)+ '_fold_NN'] = model.predict([test_input_ids,test_attention_masks] ).reshape(-1)
 sc = accuracy_score(y_train[val_idx],(oof_preds[val_idx] > 0.5 ).astype(int))
print("
FOLFD {} in {:.1f} min - Avg Acc NN {:.5f} - Best Epoch {}".format(i,(time.time() - modelstart)/60, sc, best_index + 1),'
')
 all_preds[str(i)+ '_fold_lgb'], all_preds[str(i)+ '_fold_svc'] = lgb_svc_cv(model)
 del model
K.clear_session()
gc.collect()
cat_features = [a for a in df_train.columns if a.endswith('cat')]
def preprocess_cat_feature(df, cat_feats):
for column in cat_feats:
temp = pd.get_dummies(pd.Series(df[column]), prefix=column)
 df = pd.concat([df, temp],axis=1)
 df = df.drop([column],axis=1)
 return df
df_train = preprocess_cat_feature(df_train, cat_features)
 df_test = preprocess_cat_feature(df_test, cat_features )<drop_column>
test_pred = all_preds[['0_fold_NN','1_fold_NN']].mean(axis = 1)*.8 + all_preds[['0_fold_lgb','1_fold_lgb']].mean(axis = 1)*.1 + all_preds[['0_fold_svc','1_fold_svc']].mean(axis = 1)*.1
sorted_columns_by_fi = sorted(zip(model.feature_importances_, X.columns))
col_to_drop = [x[1] for x in sorted_columns_by_fi if x[0] < 100]
df_train = df_train.drop(col_to_drop, axis=1)
 df_test = df_test.drop(col_to_drop, axis=1 )<choose_model_class>
submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission['target'] = np.round(test_pred ).astype(int)
 submission.head(10 )
class model_builder(object):
def __init__(self, n_splits):
self.n_splits = n_splits
self.stacker = LogisticRegression()
pass
def fit_predict(self, X, y, T, params_list):
y_valid_pred = 0 * y
y_submit_pred = 0
X = np.array(X)
 y = np.array(y)
 T = np.array(T)
 stack_train = np.zeros(( X.shape[0], len(params_list)))
 stack_test = np.zeros(( T.shape[0], len(params_list)))
 for param_index, params in enumerate(params_list):
kfold = KFold(n_splits = self.n_splits, random_state = 1, shuffle = True)
 model = lgb.LGBMClassifier(**params)
 stack_test_i = np.zeros(( T.shape[0], self.n_splits))
for fold_index,(train_index, val_index)in enumerate(kfold.split(X)) :
print("model ", param_index, " fold ", fold_index)
 X_train = X[train_index]
y_train = y[train_index]
X_valid = X[val_index]
model.fit(X_train, y_train, verbose=True)
 y_pred = model.predict_proba(X_valid)[:, 1]
stack_train[val_index, param_index] = y_pred
pred = model.predict_proba(T)
 stack_test_i[:, fold_index] = pred[:, 1]
stack_test[:, param_index] = stack_test_i.mean(axis=1)
 self.stacker.fit(stack_train, y)
 pred = self.stacker.predict_proba(stack_test)[:, 1]
return pred
params_list = [
{
'learning_rate' : 0.02,
'n_estimators' : 650,
'max_bin' : 10,
'subsample' : 0.8,
'subsample_freq' : 10,
'colsample_bytree' : 0.8,
'min_child_samples' : 500,
'seed' : 99
},
{
'learning_rate' : 0.02,
'n_estimators' : 1100,
'subsample' : 0.7,
'subsample_freq' : 2,
'colsample_bytree' : 0.3,
'num_leaves' : 16,
'seed' : 99
},
{
'n_estimators' : 1100,
'max_depth' : 4,
'learning_rate' : 0.02,
'seed' : 99
},
]
builder = model_builder(n_splits=N_SPLITS)
 y_test_pred = builder.fit_predict(X=df_train, y=target_train, T=df_test, params_list=params_list )<save_to_csv>
submission.to_csv('submission.csv', index=False )
<set_options><EOS> 
submission.to_csv('submission.csv', index=False )