Kaggle competition: Tabular Playground Series - Jan 2021
train_set = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_set = pd.read_csv('.. /input/nlp-getting-started/test.csv' )<import_modules>
train = pd.read_csv(input_path / 'train.csv')
 display(train.head() )
import re<drop_column>
test = pd.read_csv(input_path / 'test.csv')
 display(test.head() )
def clean_url(x):
cleaned_x = re.sub(r'http\S{0,}', r'', x)
 return cleaned_x<feature_engineering>
submission = pd.read_csv(input_path / 'sample_submission.csv')
 display(submission.head() )
train_set['text'] = train_set['text'].apply(clean_url)
 test_set['text'] = test_set['text'].apply(clean_url )<drop_column>
sns.set_style('whitegrid')
 %matplotlib inline
def clean_new_line(x):
return re.sub(r"
", " ", x )<drop_column>
target = 'target'
Id_Cols = ['id' ]
features = [x for x in train.columns if x not in [target]+Id_Cols]
X = train[features]
y = train['target']
X
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )<feature_engineering>
def RunForAll(algo , k , train , test , features , params):
val_set_pred = np.zeros(len(train))
test_set_pred = np.zeros(len(test))
X = train[features]
y = train['target']
kf = KFold(n_splits=k)
 for fold_,(train_index, val_index)in enumerate(kf.split(train , train['target'])) :
print(f'
---------------------Fold {fold_ + 1}-----------------')
 target = train['target']
X_train , y_train = train[features].iloc[train_index] , target.iloc[train_index]
X_val , y_val = train[features].iloc[val_index] , target.iloc[val_index]
X_train = X_train.abs()
X_test = test[features].abs()
_ = algo.fit(X_train , y_train , eval_set = [(X_val , y_val)] , **params)
 prediction_val = algo.predict(X_val)
 kf_score = np.sqrt(mean_squared_error(y_val , prediction_val))
print(f'
Score For Validation Sample is {kf_score}')
 val_set_pred[val_index] = prediction_val
y_val = target.iloc[val_index]
prediction_test = algo.predict(X_test)
 test_set_pred += prediction_test / k
val_score = np.sqrt(mean_squared_error(target,val_set_pred))
print(f'
Score for Validation set is {val_score}')
 return val_set_pred , test_set_pred , target
train_set['text'] = train_set['text'].apply(clean_new_line)
 train_set['text'] = train_set['text'].apply(remove_emoji)
 train_set['text'] = train_set['text'].apply(lower_case )<feature_engineering>
xgb=XGBRegressor(n_estimators = 4000 , learning_rate = 0.011)
 params = {'verbose' : False , 'early_stopping_rounds' : 100}
target = 'target'
Id_Cols = ['id' ]
features = [x for x in train.columns if x not in [target]+Id_Cols]
xgb_val_pred , xgb_test_preds , y_val_xgb = RunForAll(xgb,5, train , test , features , params )
test_set['text'] = test_set['text'].apply(clean_new_line)
 test_set['text'] = test_set['text'].apply(remove_emoji)
 test_set['text'] = test_set['text'].apply(lower_case )<load_from_url>
xgb_val_pred , xgb_test_preds , y_val_xgb
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
lgb = LGBMRegressor(n_estimators = 100000, metric = 'rmse' , learning_rate = 0.01 , boosting_type = 'gbdt' ,
num_leaves = 200 , feature_fraction = 0.6 , lambda_l1 = 2 , lambda_l2 = 2 , min_child_samples = 50 , bagging_fraction = 0.4 ,
bagging_freq = 1 , verbosity=-1 , max_depth = 12 , max_bin = 200 ,
objective = 'regression')
 params = {'verbose' : 100 , 'early_stopping_rounds' : 1000 }
target = 'target'
Id_Cols = ['id' ]
features = [x for x in train.columns if x not in [target]+Id_Cols]
lgb_val_pred , lgb_test_preds , y_val_lgb = RunForAll(lgb,5, train , test , features , params )
import tokenization
from sklearn.model_selection import StratifiedKFold, StratifiedShuffleSplit
from sklearn.metrics import precision_score, recall_score, f1_score
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow import keras
from tensorflow.keras.optimizers import Adam, SGD
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, Callback<choose_model_class>
cb = CatBoostRegressor(n_estimators = 5000 , learning_rate = 0.005)
 params = {'verbose' : False , 'early_stopping_rounds' : 100}
target = 'target'
Id_Cols = ['id' ]
features = [x for x in train.columns if x not in [target]+Id_Cols]
cb_val_pred , cb_test_preds , y_val_cb = RunForAll(cb,5, train , test , features , params )
bert_layer = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1', trainable=True )<choose_model_class>
test['target'] = cb_test_preds
output = pd.DataFrame({'id' : test.id , 'target' : test.target})
 output.to_csv('submission_cb.csv' , index=False)
 output.head()
K = 2
skf = StratifiedKFold(n_splits=K, shuffle=True )<prepare_x_and_y>
ensemble_df = pd.DataFrame(xgb_val_pred , columns=['xgboost'])
 ensemble_df['lgboost'] = lgb_val_pred
ensemble_df['cbboost'] = cb_val_pred
ensemble_df.head()
class ClassificationReport(Callback):
def __init__(self, train_data=() , validation_data=()):
super(Callback, self ).__init__()
self.X_train, self.y_train = train_data
self.train_precision_scores = []
self.train_recall_scores = []
self.train_f1_scores = []
self.X_val, self.y_val = validation_data
self.val_precision_scores = []
self.val_recall_scores = []
self.val_f1_scores = []
def on_epoch_end(self, epoch, logs={}):
train_predictions = np.round(self.model.predict(self.X_train, verbose=0))
train_precision = precision_score(self.y_train, train_predictions, average='macro')
 train_recall = recall_score(self.y_train, train_predictions, average='macro')
 train_f1 = f1_score(self.y_train, train_predictions, average='macro')
 self.train_precision_scores.append(train_precision)
 self.train_recall_scores.append(train_recall)
 self.train_f1_scores.append(train_f1)
 val_predictions = np.round(self.model.predict(self.X_val, verbose=0))
val_precision = precision_score(self.y_val, val_predictions, average='macro')
 val_recall = recall_score(self.y_val, val_predictions, average='macro')
 val_f1 = f1_score(self.y_val, val_predictions, average='macro')
 self.val_precision_scores.append(val_precision)
 self.val_recall_scores.append(val_recall)
 self.val_f1_scores.append(val_f1)
 print('
Epoch: {} - Training Precision: {:.6} - Training Recall: {:.6} - Training F1: {:.6}'.format(epoch + 1, train_precision, train_recall, train_f1))
print('Epoch: {} - Validation Precision: {:.6} - Validation Recall: {:.6} - Validation F1: {:.6}'.format(epoch + 1, val_precision, val_recall, val_f1))<choose_model_class>
ensemble_test_df = pd.DataFrame(xgb_test_preds , columns=['xgboost'])
 ensemble_test_df['lgboost'] = lgb_test_preds
ensemble_test_df['cbboost'] = cb_test_preds
ensemble_test_df.head()
class DisasterDetector:
def __init__(self, bert_layer, optimizer, max_seq_length=128, lr=0.0001, epochs=15, batch_size=32):
self.bert_layer = bert_layer
self.max_seq_length = max_seq_length
vocab_file = self.bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = self.bert_layer.resolved_object.do_lower_case.numpy()
self.tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 self.optimizer = optimizer
self.lr = lr
self.epochs = epochs
self.batch_size = batch_size
self.models = []
self.scores = {}
def preprocess(self, texts):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = self.tokenizer.tokenize(text)
 text = text[:self.max_seq_length - 2]
input_sequence = ['[CLS]'] + text + ['[SEP]']
pad_len = self.max_seq_length - len(input_sequence)
 tokens = self.tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * self.max_seq_length
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(self):
input_word_ids = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='input_word_ids')
 input_mask = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='input_mask')
 segment_ids = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='segment_ids')
 pooled_output, sequence_output = self.bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer=self.optimizer, metrics=['accuracy'])
 return model
def train(self, X, y):
for fold,(trn_idx, val_idx)in enumerate(skf.split(X, y)) :
print('
Fold {}
'.format(fold))
X_trn_encoded = self.preprocess(X.loc[trn_idx].str.lower())
 y_trn = y.loc[trn_idx]
X_val_encoded = self.preprocess(X.loc[val_idx].str.lower())
 y_val = y.loc[val_idx]
metrics = ClassificationReport(train_data=(X_trn_encoded, y_trn), validation_data=(X_val_encoded, y_val))
model = self.build_model()
model.fit(X_trn_encoded, y_trn, validation_data=(X_val_encoded, y_val), callbacks=[metrics], epochs=self.epochs,
batch_size=self.batch_size)
 self.models.append(model)
 self.scores[fold] = {
'train': {
'precision': metrics.train_precision_scores,
'recall': metrics.train_recall_scores,
'f1': metrics.train_f1_scores
},
'validation': {
'precision': metrics.val_precision_scores,
'recall': metrics.val_recall_scores,
'f1': metrics.val_f1_scores
}
}
def plot_learning_curve(self):
fig, axes = plt.subplots(nrows=K, ncols=2, figsize=(20, K * 6), dpi=100)
 for i in range(K):
sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[i].history.history['val_accuracy'], ax=axes[i][0], label='val_accuracy')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['precision'], ax=axes[i][0], label='val_precision')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['recall'], ax=axes[i][0], label='val_recall')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['f1'], ax=axes[i][0], label='val_f1')
 axes[i][0].legend()
axes[i][0].set_title('Fold {} Validation Classification Report'.format(i), fontsize=14)
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['loss'], ax=axes[i][1], label='train_loss')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['val_loss'], ax=axes[i][1], label='val_loss')
 axes[i][1].legend()
axes[i][1].set_title('Fold {} Train / Validation Loss'.format(i), fontsize=14)
 for j in range(2):
axes[i][j].set_xlabel('Epoch', size=12)
 axes[i][j].tick_params(axis='x', labelsize=12)
 axes[i][j].tick_params(axis='y', labelsize=12)
 plt.show()
def predict(self, X):
X_test_encoded = self.preprocess(X.str.lower())
 y_pred = np.zeros(( X_test_encoded[0].shape[0], 1))
for model in self.models:
y_pred += model.predict(X_test_encoded)/ len(self.models)
 return y_pred<train_on_grid>
from sklearn.linear_model import LinearRegression
sgd = SGD(1e-3)
 clf = DisasterDetector(bert_layer,sgd, max_seq_length=128, lr=0.0001, epochs=10, batch_size=32)
 clf.train(train_set['text'], train_set['target'] )<predict_on_test>
X = ensemble_df
y = y_val_xgb
reg = LinearRegression().fit(X, y)
 print(reg.score(X, y))
print(reg.coef_)
 print(reg.intercept_ )
y_pred = clf.predict(test_set['text'])
 y_pred_thres = [1 if pred[0] >=0.5 else 0 for pred in y_pred]
y_pred_df = pd.DataFrame(y_pred_thres, columns=['target'] )<load_from_csv>
ensemble_preds = reg.predict(ensemble_test_df)
 ensemble_preds
sample_subm = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 id = sample_subm.id<concatenate>
test['target'] = ensemble_preds
output = pd.DataFrame({'id' : test.id , 'target' : test.target})
 output.to_csv('submission.csv' , index=False)
 output.head()
subm = pd.concat([id, y_pred_df], axis=1 )<save_to_csv>
test['target'] = cb_test_preds
output = pd.DataFrame({'id' : test.id , 'target' : test.target})
 output.to_csv('submission_1.csv' , index=False)
 output.head()
subm.to_csv('sample_subm.csv', index=False, header = True )<set_options>
X = ensemble_df
y = y_val_xgb
dregr = DecisionTreeRegressor(max_depth=5)
 dregr.fit(X, y)
 ensemble_preds_dr = dregr.predict(ensemble_test_df)
nltk.download('stopwords', quiet=True)
 stopwords = stopwords.words('english')
 sns.set(style="white", font_scale=1.2)
 plt.rcParams["figure.figsize"] = [10,8]
pd.set_option.display_max_columns = 0
pd.set_option.display_max_rows = 0<load_from_csv>
test['target'] = ensemble_preds_dr
output = pd.DataFrame({'id' : test.id , 'target' : test.target})
 output.to_csv('submission_2.csv' , index=False)
 output.head()
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )<feature_engineering>
X = ensemble_df
y = y_val_xgb
rfregr = RandomForestRegressor(n_estimators = 200 , max_depth=5 , min_samples_leaf=100 , n_jobs=4)
 rfregr.fit(X, y)
 ensemble_preds_rf = rfregr.predict(ensemble_test_df )
<count_values><EOS> 
test['target'] = ensemble_preds_rf
output = pd.DataFrame({'id' : test.id , 'target' : test.target})
 output.to_csv('submission_3.csv' , index=False)
 output.head()