Kaggle competition: Natural Language Processing with Disaster Tweets
train_data, val_data = train_test_split(real_train, train_size=0.9, random_state=77 )<train_model>
data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv',header=0)
 data.head()
<train_model>
import matplotlib.pyplot as plt
import seaborn as sns
<train_model>
fast_data = data[['text','target']]
<compute_train_metric>
from pathlib import Path
from fastai.text import *
def gini(actual, pred, cmpcol = 0, sortcol = 1):
assert(len(actual)== len(pred))
all = np.asarray(np.c_[ actual, pred, np.arange(len(actual)) ], dtype=np.float)
 all = all[ np.lexsort(( all[:,2], -1*all[:,1])) ]
totalLosses = all[:,0].sum()
giniSum = all[:,0].cumsum().sum() / totalLosses
giniSum -=(len(actual)+ 1)/ 2.
 return giniSum / len(actual)
 def gini_normalized(a, p):
return gini(a, p)/ gini(a, a)
 def gini_xgb(preds, d_train):
targets = d_train.get_label()
gini_score = gini_normalized(targets, preds)
 return [('gini', gini_score)] 
<prepare_x_and_y>
train_df, valid_df = fast_data.loc[:5330,:],fast_data.loc[5330:,:]
d_train = xgb.DMatrix(real_train.drop(['id', 'target'], axis=1), real_train[['target']])
 d_test = xgb.DMatrix(real_test.drop(['id'], axis=1))
n_splits = 5
n_estimators = 7
folds = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=77)
 params = {
'objective': 'binary:logistic',
'eta': 0.015,
'eval_metric': 'auc',
'max_depth': 6,
'min_child_weight': 10,
'gamma': 1,
'reg_lambda': 0.3,
'reg_alpha': 0.07,
'subsample': 0.8,
'colsample_bytree': 0.8,
'silent': False,
}
xgbCV = xgboost.cv(params=params, dtrain=d_train, stratified=True, num_boost_round=10000, feval=gini_xgb,
early_stopping_rounds=100, maximize=True, verbose_eval=5, nfold=3 )<init_hyperparams>
path =Path("." )
@jit
def eval_gini(y_true, y_prob):

 y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = eval_gini(labels, preds)
 return [('gini', gini_score)]
trn_df = real_train
sub_df = real_test
target = trn_df.target
del trn_df["target"]
n_splits = 5
n_estimators = 7
folds = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=77)
 imp_df = np.zeros(( len(trn_df.columns), n_splits))
xgb_evals = np.zeros(( n_estimators, n_splits))
oof = np.empty(len(trn_df))
sub_preds = np.zeros(len(sub_df))
increase = True
np.random.seed(0)
 print('Start fitting: ')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(target, target)) :
trn_dat, trn_tgt = trn_df.iloc[trn_idx], target.iloc[trn_idx]
val_dat, val_tgt = trn_df.iloc[val_idx], target.iloc[val_idx]
clf = XGBClassifier(n_estimators=n_estimators,
max_depth=4,
objective="binary:logistic",
learning_rate=0.1,
subsample=.8,
colsample_bytree=.8,
gamma=1,
reg_alpha=0,
reg_lambda=1,
nthread=2)
 if increase:
pos = pd.Series(trn_tgt == 1)
 trn_dat = pd.concat([trn_dat, trn_dat.loc[pos]], axis=0)
 trn_tgt = pd.concat([trn_tgt, trn_tgt.loc[pos]], axis=0)
 idx = np.arange(len(trn_dat))
np.random.shuffle(idx)
 trn_dat = trn_dat.iloc[idx]
trn_tgt = trn_tgt.iloc[idx]
print('{}th fold'.format(fold_))
clf.fit(trn_dat, trn_tgt,
eval_set=[(trn_dat, trn_tgt),(val_dat, val_tgt)],
eval_metric=gini_xgb,
early_stopping_rounds=None,
verbose=False)
 imp_df[:, fold_] = clf.feature_importances_
oof[val_idx] = clf.predict_proba(val_dat)[:, 1]
xgb_evals[:, fold_] = clf.evals_result_["validation_1"]["gini"]
best_round = np.argsort(xgb_evals[:, fold_])[::-1][0]
print("Fold %2d : %.6f @%4d / best score is %.6f @%4d"
%(fold_ + 1,
eval_gini(val_tgt, oof[val_idx]),
n_estimators,
xgb_evals[best_round, fold_],
best_round))
sub_preds += clf.predict_proba(sub_df)[:, 1] / n_splits
print("Full OOF score : %.6f" % eval_gini(target, oof))
mean_eval = np.mean(xgb_evals, axis=1)
 std_eval = np.std(xgb_evals, axis=1)
 best_round = np.argsort(mean_eval)[::-1][0]
print("Best mean score : %.6f + %.6f @%4d"
%(mean_eval[best_round], std_eval[best_round], best_round))
importances = imp_df.mean(axis=1)
 for i, imp in enumerate(importances):
print("%-20s : %10.4f" %(trn_df.columns[i], imp))
sub_df["target"] = sub_preds
sub_df[["target"]].to_csv("submission_20fold.csv", index=True, float_format="%.9f" )<prepare_x_and_y>
data_lm = TextLMDataBunch.from_df(path, train_df, valid_df, text_cols=['text'], bs=64)
 data_clas = TextClasDataBunch.from_df(path, train_df, valid_df, text_cols=['text'], label_cols=['target'], bs=64 )
d_train = xgb.DMatrix(train_data.drop(['id', 'target'], axis=1), train_data[['target']])
 d_val = xgb.DMatrix(val_data.drop(['id', 'target'], axis=1), val_data[['target']])
 d_test = xgb.DMatrix(real_test.drop(['id'], axis=1))
params = {
'objective': 'binary:logistic',
'eta': 0.015,
'eval_metric': 'auc',
'max_depth': 6,
'min_child_weight': 10,
'gamma': 1,
'reg_lambda': 0.3,
'reg_alpha': 0.07,
'subsample': 0.8,
'colsample_bytree': 0.8,
'silent': True,
'n_estimators': 400,
}
watchlist = [(d_train, 'train'),(d_val, 'valid')]
mdl_xgb = xgb.train(params, d_train, num_boost_round=10000, evals=watchlist, early_stopping_rounds=100,
feval=gini_xgb, maximize=True, verbose_eval=10)
 filename = 'xgb_model.joblib.pkl'
_ = joblib.dump(mdl_xgb, filename, compress=9 )<predict_on_test>
data_lm.show_batch()
test_pred = mdl_xgb.predict(data=d_test )<save_to_csv>
data_clas.show_batch()
submissions = pd.DataFrame()
submissions['id'] = real_test.iloc[:, 0]
submissions['target'] = test_pred
submissions.to_csv('xgb_model.csv', index=False )<set_options>
learn = language_model_learner(data_lm, arch = AWD_LSTM, pretrained = True, drop_mult=0.4)
 learn.lr_find()
learn.recorder.plot()
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
<load_from_csv>
learn.fit_one_cycle(10, 1e-2 )
train_X_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_X_features_865.csv")
 train_X_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_X_features_865.csv")
 y_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_y.csv", index_col=False, header=None)
 y_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_y.csv", index_col=False, header=None )<concatenate>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
train_X = pd.concat([train_X_0, train_X_1], axis=0)
 train_X = train_X.reset_index(drop=True)
 print(train_X.shape)
 train_X.head()<concatenate>
learn.fit_one_cycle(10, 1e-3 )
train_y = pd.concat([y_0, y_1], axis=0)
 train_y = train_y.reset_index(drop=True)
 train_y.columns = ['time_to_failure']
train_y['time_to_failure'].shape<load_from_csv>
learn.save_encoder('word-enc' )
test_X = pd.read_csv(".. /input/lanl-master-s-features-creating-0/test_X_features_10.csv" )<normalization>
learn = text_classifier_learner(data_clas, arch = AWD_LSTM, pretrained = True, drop_mult=0.3)
 learn.load_encoder('word-enc')
 learn.lr_find()
learn.recorder.plot()
scaler = StandardScaler()
train_columns = train_X.columns
train_X[train_columns] = scaler.fit_transform(train_X[train_columns])
 test_X[train_columns] = scaler.transform(test_X[train_columns] )<set_options>
learn.fit_one_cycle(1, 1e-2)
 learn.freeze_to(-2)
 learn.fit_one_cycle(10, slice(5e-3/2., 5e-3))
learn.freeze_to(-3)
 learn.fit_one_cycle(10, slice(5e-3/2., 5e-3))
learn.unfreeze()
learn.fit_one_cycle(10, slice(2e-3/100, 2e-3))
h2o.init(max_mem_size='16G' )<create_dataframe>
test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv',header=0 )
train = h2o.H2OFrame(train_X)
 test = h2o.H2OFrame(test_X)
 y_train = h2o.H2OFrame(train_y )<feature_engineering>
result = pd.DataFrame()
train['time_to_failure'] = y_train['time_to_failure']
x = list(train_X.columns)
 y = train_y.columns[0]<train_model>
for i,row in test.iterrows() :
result.loc[i,"target"] = learn.predict(row["text"])[0]
%%time
nfolds=0
aml = H2OAutoML(max_models=500, seed=555, max_runtime_secs=6000,nfolds=nfolds,sort_metric="MAE")
 aml.train(x=x, y=y, training_frame=train )<set_options>
result["id"] = test.id
<save_to_csv><EOS> 
result[["id","target"]].to_csv("submission.csv",index=False )