Kaggle competition: Natural Language Processing with Disaster Tweets
y = df_train["target"]
df_train.pop("target");
df_train.pop("id")
 colnames1 = df_train.columns 
<normalization>
!cp '/kaggle/input/nlp-getting-started/test.csv' '/kaggle/working/test.csv'
!cp '/kaggle/input/nlp-getting-started/train.csv' '/kaggle/working/train.csv'
scaler = StandardScaler()
scaler.fit(df_train)
 X = scaler.transform(df_train)
 df_train = pd.DataFrame(data = X, columns=colnames1 )<choose_model_class>
!pip install fastai --upgrade
random_state = 0
clf = LogisticRegression(random_state = random_state)
 param_grid = {'class_weight' : ['balanced'],
'penalty' : ['l1'],
'C' : [0.0001, 0.0005, 0.001,
0.005, 0.01, 0.05, 0.1, 0.5, 1,
10, 100, 1000, 1500, 2000, 2500,
2600, 2700, 2800, 2900, 3000, 3100, 3200
] ,
'max_iter' : [100, 1000, 2000, 5000, 10000] }
scorer = make_scorer(roc_auc_score)
 grid = GridSearchCV(estimator = clf, param_grid = param_grid ,
scoring = scorer, verbose = 1, cv=20,
n_jobs = -1)
 X = df_train.values
grid.fit(X,y)
 print("Best Score:" + str(grid.best_score_))
best_parameters = grid.best_params_ 
<find_best_params>
%reload_ext autoreload
%autoreload 2
path = Path('/kaggle/working/')
 path.ls()
best_clf = grid.best_estimator_
print(best_clf )<train_model>
train_df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'test.csv')
 print(train_df.head())
model = LogisticRegression(C=0.1, class_weight='balanced', dual=False,
fit_intercept=True, intercept_scaling=1, max_iter=100,
multi_class='warn', n_jobs=None, penalty='l1', random_state=0,
solver='warn', tol=0.0001, verbose=0, warm_start=False);
model.fit(X, y); 
<load_from_csv>
dls = TextDataLoaders.from_csv(path=path,
csv_fname='train.csv',
text_col='text',
label_col='target',
valid_pct=0.2)
 first(dls[0] )
df_test = pd.read_csv(".. /input/test.csv")
 df_test.pop("id");
X = df_test
X = scaler.transform(X)
 df_test = pd.DataFrame(data = X, columns=colnames1)
 X = df_test.values
y_pred = model.predict_proba(X)
 y_pred = y_pred[:,1]<save_to_csv>
learn = text_classifier_learner(dls, AWD_LSTM, drop_mult=0.5, metrics=F1Score())
 learn.model_dir = '/kaggle/working/'
learn.fine_tune(2, 1e-2 )
smpsb_df = pd.read_csv(".. /input/sample_submission.csv")
 smpsb_df["target"] = y_pred
smpsb_df.to_csv("submission.csv", index=None)
<load_from_csv>
learn.save('simple_NLP' )
df = pd.read_csv('.. /input/train.csv')
 df.head()
features = ['16', '33', '43', '45', '52', '63', '65', '73', '90', '91', '117', '133', '134', '149', '189', '199', '217', '237', '258', '295']<prepare_x_and_y>
learn.load('simple_NLP' )
X = df[features].values
y = df.values[:,1]
print(X.shape, y.shape )<choose_model_class>
print(learn.predict('It was crazy'))
print(learn.predict('It was a disaster'))
def simple_model(input_shape):

 inp = Input(shape=(input_shape[1],))
x = Dense(3, activation='sigmoid' )(inp)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=0.005), metrics=['accuracy'])
 return model<train_model>
print('Total number of needed predictions:',len(test_df))
predictions = []
ids = []
for i in range(len(test_df)) :
print(i)
 pred, _, _ = learn.predict(test_df.loc[i, 'text'])
 ids.append(test_df.loc[i, 'id'])
 predictions.append(pred)
N_SPLITS = 10
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True ).split(X, y))
preds_val = []
y_val = []
best_models = []
for idx,(train_idx, val_idx)in enumerate(splits):
print("Beginning fold {}".format(idx+1))
X_train, y_train, X_val, y_val = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = simple_model(X_train.shape)
 cb = ModelCheckpoint('weights.h5', monitor='val_acc', mode='max', save_best_only=True, save_weights_only=True)
 model.fit(X_train, y_train, epochs=200, validation_data=(X_val, y_val), callbacks=[cb], verbose=0)
 model.load_weights('weights.h5')
 score = roc_auc_score(y_val, model.predict(X_val))
print(( model, score))
best_models.append(( model, score))<load_from_csv>
submission = pd.DataFrame({'id': ids, 'target':predictions})
 print(submission)
 submission.to_csv("simple_submission.csv", index=False )
df_test = pd.read_csv('.. /input/test.csv')
 print(len(df_test))
df_test.head()<prepare_x_and_y>
df = pd.concat([train_df, test_df] )
X_test = df_test[features].values<predict_on_test>
dls_lm = TextDataLoaders.from_df(df, text_col='text', is_lm=True, valid_pct=0.0)
 dls_lm.show_batch(max_n=3 )
y_preds = []
for mod, score in best_models:
y_preds.append(mod.predict(X_test))
y_preds = np.concatenate(y_preds, axis=1)
 y_preds.shape<save_to_csv>
lm_learn = language_model_learner(dls_lm, AWD_LSTM, metrics=[accuracy, Perplexity() ], path=path, wd=0.1)
 lm_learn.model_dir = '/kaggle/working/'
subs = pd.read_csv('.. /input/sample_submission.csv')
 mean_preds = y_preds.mean(axis=1)
 subs['target'] = mean_preds
subs.to_csv('submission.csv', index=False )<set_options>
lm_learn.fit_one_cycle(1, 1e-2)
 lm_learn.save('1epoch' )
%matplotlib inline 
<import_modules>
lm_learn = lm_learn.load('1epoch')
 lm_learn.unfreeze()
lm_learn.fit_one_cycle(10, 1e-3)
 lm_learn.save_encoder('finetuned' )
Conv1D, Conv2D, Conv3D, \
Conv2DTranspose, \
AveragePooling1D, AveragePooling2D, \
MaxPooling1D, MaxPooling2D, MaxPooling3D, \
GlobalAveragePooling1D, GlobalAveragePooling2D, \
GlobalMaxPooling1D, GlobalMaxPooling2D, GlobalMaxPooling3D, \
LocallyConnected1D, LocallyConnected2D, \
concatenate, Flatten, Average, Activation, \
RepeatVector, Permute, Reshape, Dot, \
multiply, dot, add, \
PReLU, \
Bidirectional, TimeDistributed, \
SpatialDropout1D, \
BatchNormalization 
<import_modules>
dls_clas = TextDataLoaders.from_csv(path=path,
csv_fname='train.csv',
text_col='text',
label_col='target',
valid_pct=0.2,
text_vocab=dls_lm.vocab )
from PIL import Image
from zipfile import ZipFile
import h5py
import cv2
from tqdm import tqdm<load_from_csv>
adv_learn = text_classifier_learner(dls_clas, AWD_LSTM, drop_mult=0.5, metrics=F1Score())
 adv_learn.model_dir = '/kaggle/working/'
adv_learn = adv_learn.load_encoder('finetuned')
 adv_learn.fit_one_cycle(1, 2e-2 )
src_dir = '.. /input'
train_csv = pd.read_csv(os.path.join(src_dir, 'train.csv'))
print(train_csv.shape)
 train_csv.head()<categorify>
adv_learn.freeze_to(-2)
 adv_learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2))
y_cat_train0 = to_categorical(y_train0)
 print(y_cat_train0.shape)
 y_cat_train0[:5]<load_from_csv>
adv_learn.freeze_to(-3)
 adv_learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3))
test_csv = pd.read_csv(os.path.join(src_dir, 'test.csv'))
print(test_csv.shape)
 test_csv.head()<load_from_csv>
adv_learn.unfreeze()
adv_learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3))
sample_submission_csv = pd.read_csv(os.path.join(src_dir, 'sample_submission.csv'))
print(sample_submission_csv.shape)
 sample_submission_csv.head()<set_options>
test_df.loc[0, 'text']
warnings.filterwarnings('ignore' )<import_modules>
adv_learn.predict(test_df.loc[0, 'text'] )
from sklearn.linear_model import Lasso
from sklearn.feature_selection import RFECV
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import StratifiedShuffleSplit, GridSearchCV, validation_curve
from sklearn.metrics import mean_squared_error, mean_absolute_error, roc_auc_score, r2_score, make_scorer
from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances<init_hyperparams>
predictions = []
ids = []
for i in range(len(test_df)) :
print(i)
 pred, _, _ = adv_learn.predict(test_df.loc[i, 'text'])
 ids.append(test_df.loc[i, 'id'])
 predictions.append(pred )
rfe_min_features = 12
rfe_step = 15
rfe_cv = 20
sss_n_splits = 20
sss_test_size = 0.35
grid_search_cv = 20
noise_std = 0.01
r2_threshold = 0.185
random_seed = 213
np.random.seed(random_seed )<compute_test_metric>
submission = pd.DataFrame({'id': ids, 'target':predictions})
 print(submission)
 submission.to_csv("complex_submission.csv", index=False )
def scoring_roc_auc(y, y_pred):
try:
return roc_auc_score(y, y_pred)
 except:
return 0.5<normalization>
dls_lm_back = TextDataLoaders.from_df(df, text_col='text', is_lm=True, valid_pct=0, backward = True)
 dls_lm_back.show_batch(max_n=3 )
x = RobustScaler().fit_transform(np.concatenate(( x_train0, x_test), axis=0))
x_train0 = x[:250]
x_test = x[250:]<init_hyperparams>
lm_learn_back = language_model_learner(dls_lm_back, AWD_LSTM, metrics=[accuracy, Perplexity() ], path=path, wd=0.1)
 lm_learn_back.model_dir = '/kaggle/working/'
robust_roc_auc = make_scorer(scoring_roc_auc)
 param_grid = {
'alpha' : [0.022, 0.021, 0.02, 0.019, 0.023, 0.024, 0.025, 0.026, 0.027, 0.029, 0.031],
'tol' : [0.0013, 0.0014, 0.001, 0.0015, 0.0011, 0.0012, 0.0016, 0.0017]
} 
<choose_model_class>
lm_learn_back.fit_one_cycle(1, 1e-2)
 lm_learn_back.save('1epoch_back' )
class RFECV_wr(RFECV):
def __init__(self, alpha=1.0, tol=0.0001,
step=1, min_features_to_select=1, cv='warn',
scoring=None, verbose=0, n_jobs=None):
estimator = Lasso(alpha=0.031, tol=0.001,
random_state=random_seed, selection='random')
 super().__init__(estimator, step=step,
min_features_to_select=min_features_to_select,
cv=cv, scoring=scoring, verbose=verbose, n_jobs=n_jobs)
 def set_params(self, **params):
if 'alpha' in params:
self.estimator.set_params(alpha=params['alpha'])
 if 'tol' in params:
self.estimator.set_params(tol=params['tol'])
 return self
feature_selector2 = RFECV_wr(min_features_to_select=rfe_min_features,
scoring=robust_roc_auc,
step=rfe_step,
verbose=0,
cv=rfe_cv,
n_jobs=-1 )<find_best_params>
lm_learn_back = lm_learn_back.load('1epoch_back')
 lm_learn_back.unfreeze()
lm_learn_back.fit_one_cycle(10, 1e-3)
 lm_learn_back.save_encoder('finetuned_back' )
feature_selector2.get_params()<define_search_space>
dls_clas_back = TextDataLoaders.from_csv(path=path,
csv_fname='train.csv',
text_col='text',
label_col='target',
valid_pct=0.2,
text_vocab=dls_lm_back.vocab,
backwards = True )
param_range = np.logspace(-1.7, -1.5, 10)
 param_range<train_on_grid>
back_learn = text_classifier_learner(dls_clas_back, AWD_LSTM, drop_mult=0.5, metrics=F1Score())
 back_learn.model_dir = '/kaggle/working/'
back_learn = back_learn.load_encoder('finetuned_back')
 back_learn.fit_one_cycle(1, 2e-2 )
scorer = make_scorer(roc_auc_score)
 scorer
cv_splitter = StratifiedShuffleSplit(n_splits=5, test_size=0.35, random_state=0)
 cv_splitter.get_n_splits(x_train0, y_train0 )<find_best_params>
back_learn.freeze_to(-2)
 back_learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2))
train_scores, test_scores = validation_curve(
 feature_selector2, x_train0, y_train0,
param_name="alpha", param_range=param_range,
cv=cv_splitter, scoring=scorer, n_jobs=None, verbose=0 )<feature_engineering>
back_learn.freeze_to(-3)
 back_learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3))
train_scores_mean = np.mean(train_scores, axis=1)
 train_scores_std = np.std(train_scores, axis=1)
 test_scores_mean = np.mean(test_scores, axis=1)
 test_scores_std = np.std(test_scores, axis=1)
 train_scores_mean<split>
back_learn.unfreeze()
back_learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3))
print("counter | val_mse | val_mae | val_roc | val_r2 | alpha | feature_count ")
 print("-------------------------------------------------------------------------------------")
 importances = np.zeros(( 300,))
predictions = pd.DataFrame()
counter = 0
for train_index, val_index in StratifiedShuffleSplit(n_splits=sss_n_splits, test_size=sss_test_size, random_state=random_seed ).split(x_train0, y_train0):
X, val_X = x_train0[train_index], x_train0[val_index]
y, val_y = y_train0[train_index], y_train0[val_index]
grid_search = GridSearchCV(feature_selector2,
param_grid=param_grid,
verbose=0,
n_jobs=None,
scoring=robust_roc_auc,
cv=grid_search_cv)
 grid_search.fit(X, y)
 val_y_pred = grid_search.best_estimator_.predict(val_X)
 val_mse = mean_squared_error(val_y, val_y_pred)
 val_mae = mean_absolute_error(val_y, val_y_pred)
 val_roc = roc_auc_score(val_y, val_y_pred)
 val_cos = cosine_similarity(val_y.reshape(1, -1), val_y_pred.reshape(1, -1)) [0][0]
val_dst = euclidean_distances(val_y.reshape(1, -1), val_y_pred.reshape(1, -1)) [0][0]
val_r2 = r2_score(val_y, val_y_pred)
 if val_r2 > r2_threshold:
message = '<-- OK'
prediction = grid_search.best_estimator_.predict(x_test)
 predictions = pd.concat([predictions, pd.DataFrame(prediction)], axis=1)
 importances += grid_search.best_estimator_.support_.astype(int)
 else:
message = '<-- skipping'
print("{0:2} | {1:.4f} | {2:.4f} | {3:.4f} | {4:.4f} | {5:.4f} | {6:3} {7} ".format(
 counter,
val_mse,
val_mae,
val_roc,
val_r2,
grid_search.best_estimator_.estimator_.get_params() ['alpha'],
grid_search.best_estimator_.n_features_,
message))
counter += 1<find_best_params>
back_learn.predict(test_df.loc[0, 'text'] )
grid_search.best_estimator_.estimator_.get_params()<save_to_csv>
predictions = []
ids = []
for i in range(len(test_df)) :
print(i)
 _, _, forward_predict = adv_learn.predict(test_df.loc[i, 'text'])
 _, _, backwards_predict = back_learn.predict(test_df.loc[i, 'text'])
 pred = bool(forward_predict[1]+backwards_predict[1] > 1)
 ids.append(test_df.loc[i, 'id'])
 predictions.append(int(pred))
mean_pred = pd.DataFrame(predictions.mean(axis=1))
mean_pred.index += 250
mean_pred.columns = ['target']
mean_pred.to_csv('submission.csv', index_label='id', index=True)
 mean_pred.head()<load_from_csv>
submission = pd.DataFrame({'id': ids, 'target':predictions})
 print(submission)
 submission.to_csv("ensemble_submission.csv", index=False )