Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
import numpy as np
import pandas as pd
from fastai.text import *
from fastai.callbacks import *
diff = {}
for col in X_tr.columns:
save = X_tr[col].values.copy()
X_tr[col] = np.random.permutation(X_tr[col])
 diff[col] = model.validation_score(X_tr, y_tr)- benchmark
X_tr[col] = save<save_to_csv>
train = pd.read_csv(path/'train.csv')
 test = pd.read_csv(path/'test.csv' )
num_feats = scores.index(min(scores))
features = sorted(diff, key=diff.get, reverse=True)[:num_feats]
model_.train(X_tr[features], y_tr)
 submission['time_to_failure'] = model_.predict(X_test[features])
 submission.to_csv('submission_pi.csv' )<save_to_csv>
data_lm =(TextList.from_df(pd.concat([train[['text']], test[['text']]], ignore_index=True, axis=0)) 
.split_by_rand_pct(0.15)
.label_for_lm() 
.databunch(bs=128))
features = features[scores.index(min(scores)) ]
model_.train(X_tr[features], y_tr)
 submission['time_to_failure'] = model_.predict(X_test[features])
 submission.to_csv('submission_rfe.csv' )<train_model>
data_lm.show_batch()
estimator = RandomForestRegressor(n_estimators=500, n_jobs=4, max_depth=4)
 boruta_selector = BorutaPy(estimator, n_estimators='auto', verbose=0)
 boruta_selector.fit(X_tr.values, y_tr.values.flatten())
 boruta_df = pd.DataFrame({'feature': X_tr.columns, 'rank': boruta_selector.ranking_})
 feats = boruta_df[boruta_df['rank'] == 1].feature
model_.train(X_tr[feats], y_tr)
 print("Boruta validation score: {:.6f}".format(model_.validation_score(X_tr[feats], y_tr)) )<train_model>
learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.5 )
actual_importance = model.feat_importance[['feature', 'fold_mean']]
null_importance = pd.DataFrame()
for i in range(200):
shuffled_target = y_tr.copy().sample(frac=1.0)
 model_.train(X_tr, shuffled_target)
 importance = model_.feat_importance[['feature', 'fold_mean']]
importance['run'] = i + 1
null_importance = pd.concat([null_importance, importance])
 null_importance.head()<find_best_params>
learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))
feature_scores = []
for _f in actual_importance['feature'].unique() :
f_null_imps_gain = null_importance.loc[null_importance['feature'] == _f, 'fold_mean'].values
f_act_imps_gain = actual_importance.loc[actual_importance['feature'] == _f, 'fold_mean'].mean()
gain_score = np.log(1e-10 + f_act_imps_gain /(1 + np.percentile(f_null_imps_gain, 75)))
 feature_scores.append(( _f, gain_score))
scores_df = pd.DataFrame(feature_scores, columns=['feature', 'gain_score'])
 scores_df.sort_values('gain_score', ascending=False, inplace=True)
 for num_feats in range(6, 60, 2):
features = scores_df.loc[:num_feats, 'feature']
model_.train(X_tr[features], y_tr)
 scores.append(model_.validation_score(X_tr[features], y_tr))
lineplot(list(range(6, 60, 2)) , scores )<train_on_grid>
callback = SaveModelCallback(learn,monitor="accuracy", mode="max", name="best_lang_model" )
scores = []
for num_feats in range(6, 60, 2):
values = SelectFromModel(LassoCV(cv=5), max_features=num_feats,
threshold=-np.inf ).fit_transform(X_tr, y_tr)
 X = pd.DataFrame(values, columns=[i for i in range(num_feats)])
 model_.train(X, y_tr)
 scores.append(model_.validation_score(X, y_tr))
lineplot(list(range(6, 60, 2)) , scores, "Feature selection with Lasso" )<set_options>
learn.unfreeze()
learn.fit_one_cycle(10, 1e-3, moms=(0.8,0.7), callbacks=[callback] )
warnings.filterwarnings('ignore' )<load_from_csv>
learn.save('fine_tuned')
 learn.save_encoder('fine_tuned_enc' )
train_data = pd.read_csv('.. /input/train.csv', dtype = {'acoustic_data': np.int16, 'time_to_failure': np.float32} )<define_variables>
df = train[['text', 'target']]
seg_length = 150000
total_samples = int(np.floor(( train_data.shape[0])/ seg_length))
COLUMNS = ['AVERAGE', 'STD', 'MAX', 'MIN', 'SUM',
'AVERAGE_FIRST_10000', 'AVERAGE_LAST_10000', 'AVERAGE_FIRST_50000', 'AVERAGE_LAST_50000',
'STD_FIRST_10000', 'STD_LAST_10000', 'STD_FIRST_50000', 'STD_LAST_50000',
'ABS_AVERAGE', 'ABS_STD', 'ABS_MAX', 'ABS_MIN',
'10Q', '25Q', '50Q', '75Q', '90Q',
'ABS_1Q', 'ABS_5Q', 'ABS_30Q', 'ABS_60Q', 'ABS_95Q', 'ABS_99Q',
'KURTOSIS', 'SKEW', 'MEDIAN',
'HILBERT_MEAN', 'HANN_WINDOW_MEAN']
cols = COLUMNS
x_train = pd.DataFrame(index = range(total_samples), columns = cols, dtype = np.float64)
 y_train = pd.DataFrame(index = range(total_samples), columns = ['time_to_failure'], dtype = np.float64 )<feature_engineering>
df_test = test[['text']]
for value in tqdm_notebook(range(total_samples)) :
sample = train_data.iloc[value*seg_length : value*seg_length + seg_length]
x = pd.Series(sample['acoustic_data'].values)
 y = sample['time_to_failure'].values[-1]
y_train.loc[value, 'time_to_failure'] = y
x_train.loc[value, 'AVERAGE'] = x.mean()
x_train.loc[value, 'STD'] = x.std()
x_train.loc[value, 'MAX'] = x.max()
x_train.loc[value, 'MIN'] = x.min()
x_train.loc[value, 'SUM'] = x.sum()
x_train.loc[value, 'AVERAGE_FIRST_10000'] = x[:10000].mean()
x_train.loc[value, 'AVERAGE_LAST_10000'] = x[-10000:].mean()
x_train.loc[value, 'AVERAGE_FIRST_50000'] = x[:50000].mean()
x_train.loc[value, 'AVERAGE_LAST_50000'] = x[-50000:].mean()
x_train.loc[value, 'STD_FIRST_10000'] = x[:10000].std()
x_train.loc[value, 'STD_LAST_10000'] = x[-10000:].std()
x_train.loc[value, 'STD_FIRST_50000'] = x[:50000].std()
x_train.loc[value, 'STD_LAST_50000'] = x[-50000:].std()
x_train.loc[value, 'ABS_AVERAGE'] = np.abs(x ).mean()
x_train.loc[value, 'ABS_STD'] = np.abs(x ).std()
x_train.loc[value, 'ABS_MAX'] = np.abs(x ).max()
x_train.loc[value, 'ABS_MIN'] = np.abs(x ).min()
x_train.loc[value, '10Q'] = np.percentile(x, 0.10)
 x_train.loc[value, '25Q'] = np.percentile(x, 0.25)
 x_train.loc[value, '50Q'] = np.percentile(x, 0.50)
 x_train.loc[value, '75Q'] = np.percentile(x, 0.75)
 x_train.loc[value, '90Q'] = np.percentile(x, 0.90)
 x_train.loc[value, 'ABS_1Q'] = np.percentile(x, np.abs(0.01))
x_train.loc[value, 'ABS_5Q'] = np.percentile(x, np.abs(0.05))
x_train.loc[value, 'ABS_30Q'] = np.percentile(x, np.abs(0.30))
x_train.loc[value, 'ABS_60Q'] = np.percentile(x, np.abs(0.60))
x_train.loc[value, 'ABS_95Q'] = np.percentile(x, np.abs(0.95))
x_train.loc[value, 'ABS_99Q'] = np.percentile(x, np.abs(0.99))
x_train.loc[value, 'KURTOSIS'] = x.kurtosis()
x_train.loc[value, 'SKEW'] = x.skew()
x_train.loc[value, 'MEDIAN'] = x.median()
x_train.loc[value, 'HILBERT_MEAN'] = np.abs(hilbert(x)).mean()
x_train.loc[value, 'HANN_WINDOW_MEAN'] =(convolve(x, hann(150), mode = 'same')/ sum(hann(150)) ).mean()<import_modules>
data_clas =(TextList.from_df(df, vocab=data_lm.vocab)
.split_by_rand_pct(0.1)
.label_from_df('target')
.add_test(TextList.from_df(df_test, vocab=data_lm.vocab)) 
.databunch(bs=128))
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_absolute_error<normalization>
learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5, metrics=[accuracy, FBeta(beta=1)])
 learn.load_encoder('fine_tuned_enc' )
scaler = StandardScaler()
scaler.fit(x_train)
 x_train_scaled = scaler.transform(x_train)
 y_train_flatten = y_train.values.ravel()<prepare_output>
learn.fit_one_cycle(1, 1e-3, moms=(0.8,0.7))
x_train_dataframe = pd.DataFrame(x_train_scaled)
 x_train_dataframe.head(10 )<import_modules>
learn.freeze_to(-2)
 learn.fit_one_cycle(1, slice(1e-3/(2.6**4),1e-2), moms=(0.8,0.7))
import keras
from keras.layers import Dense
from keras.layers import BatchNormalization
from keras.layers import Activation
from keras.layers import Dropout
from keras.models import Sequential
from keras import optimizers<choose_model_class>
learn.freeze_to(-3)
 learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))
kernel_init = 'he_normal'
def Model(input_dim, activation, classes):
model = Sequential()
model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dense(32, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(classes, kernel_initializer = kernel_init))
model.add(Activation('linear'))
return model<choose_model_class>
callbacks = SaveModelCallback(learn,monitor="accuracy", mode="max", name="best_classification_model" )
input_dim = x_train.shape[1]
activation = 'tanh'
classes = 1
model = Model(input_dim = input_dim, activation = activation, classes = classes)
<choose_model_class>
learn.unfreeze()
learn.fit_one_cycle(15, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7), callbacks=[callbacks] )
optim = optimizers.Adam(lr = 0.001)
 model.compile(loss = 'mean_absolute_error', optimizer = optim, metrics = ['mean_absolute_error'] )<load_from_csv>
preds, _ = learn.get_preds(ds_type=DatasetType.Test, ordered=True)
 preds = preds.argmax(dim=-1)
 id = test['id']
submission = pd.read_csv('.. /input/sample_submission.csv', index_col = 'seg_id')
 X_test = pd.DataFrame(columns = x_train.columns, dtype = np.float64, index = submission.index)
 for id in X_test.index:
seg = pd.read_csv('.. /input/test/' + id + '.csv')
 x = pd.Series(seg['acoustic_data'].values)
 X_test.loc[id, 'AVERAGE'] = x.mean()
X_test.loc[id, 'STD'] = x.std()
X_test.loc[id, 'MAX'] = x.max()
X_test.loc[id, 'MIN'] = x.min()
X_test.loc[id, 'SUM'] = x.sum()
X_test.loc[id, 'AVERAGE_FIRST_10000'] = x[:10000].mean()
X_test.loc[id, 'AVERAGE_LAST_10000'] = x[-10000:].mean()
X_test.loc[id, 'AVERAGE_FIRST_50000'] = x[:50000].mean()
X_test.loc[id, 'AVERAGE_LAST_50000'] = x[-50000:].mean()
X_test.loc[id, 'STD_FIRST_10000'] = x[:10000].std()
X_test.loc[id, 'STD_LAST_10000'] = x[-10000:].std()
X_test.loc[id, 'STD_FIRST_50000'] = x[:50000].std()
X_test.loc[id, 'STD_LAST_50000'] = x[-50000:].std()
X_test.loc[id, 'ABS_AVERAGE'] = np.abs(x ).mean()
X_test.loc[id, 'ABS_STD'] = np.abs(x ).std()
X_test.loc[id, 'ABS_MAX'] = np.abs(x ).max()
X_test.loc[id, 'ABS_MIN'] = np.abs(x ).min()
X_test.loc[id, '10Q'] = np.percentile(x, 0.10)
 X_test.loc[id, '25Q'] = np.percentile(x, 0.25)
 X_test.loc[id, '50Q'] = np.percentile(x, 0.50)
 X_test.loc[id, '75Q'] = np.percentile(x, 0.75)
 X_test.loc[id, '90Q'] = np.percentile(x, 0.90)
 X_test.loc[id, 'ABS_1Q'] = np.percentile(x, np.abs(0.01))
X_test.loc[id, 'ABS_5Q'] = np.percentile(x, np.abs(0.05))
X_test.loc[id, 'ABS_30Q'] = np.percentile(x, np.abs(0.30))
X_test.loc[id, 'ABS_60Q'] = np.percentile(x, np.abs(0.60))
X_test.loc[id, 'ABS_95Q'] = np.percentile(x, np.abs(0.95))
X_test.loc[id, 'ABS_99Q'] = np.percentile(x, np.abs(0.99))
X_test.loc[id, 'KURTOSIS'] = x.kurtosis()
X_test.loc[id, 'SKEW'] = x.skew()
X_test.loc[id, 'MEDIAN'] = x.median()
X_test.loc[id, 'HILBERT_MEAN'] = np.abs(hilbert(x)).mean()
X_test.loc[id, 'HANN_WINDOW_MEAN'] =(convolve(x, hann(150), mode = 'same')/ sum(hann(150)) ).mean()<normalization>
my_submission = pd.DataFrame({'id': id, 'target': preds})
 my_submission.to_csv('submission.csv', index=False )