Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<categorify>
%matplotlib inline
max_len = 150
train_input = bert_encode(train.Phrase.values, tokenizer, max_len=max_len)
 test_input = bert_encode(test.Phrase.values, tokenizer, max_len=max_len)
 train_labels = tf.keras.utils.to_categorical(train.Sentiment.values, num_classes=5 )<train_model>
%%time
data = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 target = np.log1p(data['target'])
 data.drop(['ID', 'target'], axis=1, inplace=True )
%%time
checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True, verbose=1)
 earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, verbose=1)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
callbacks=[checkpoint, earlystopping],
batch_size=32,
verbose=1
)<save_to_csv>
%%time
leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/train_leak.csv')
 data['leak'] = leak['compiled_leak'].values
data['log_leak'] = np.log1p(leak['compiled_leak'].values )
%%time
model.load_weights('model.h5')
 test_pred = model.predict(test_input)
 sub = pd.read_csv('.. /input/sentiment-analysis-on-movie-reviews/sampleSubmission.csv')
 sub['Sentiment'] = np.argmax(test_pred, axis=-1)
 sub.to_csv('submission.csv', index=False)
<import_modules>
%%time
def rmse(y_true, y_pred):
return mean_squared_error(y_true, y_pred)**.5
reg = XGBRegressor(n_estimators=1000)
 folds = KFold(4, True, 134259)
 fold_idx = [(trn_, val_)for trn_, val_ in folds.split(data)]
scores = []
nb_values = data.nunique(dropna=False)
 nb_zeros =(data == 0 ).astype(np.uint8 ).sum(axis=0)
 features = [f for f in data.columns if f not in ['log_leak', 'leak', 'target', 'ID']]
for _f in features:
score = 0
for trn_, val_ in fold_idx:
reg.fit(
 data[['log_leak', _f]].iloc[trn_], target.iloc[trn_],
eval_set=[(data[['log_leak', _f]].iloc[val_], target.iloc[val_])],
eval_metric='rmse',
early_stopping_rounds=50,
verbose=False 
)
 score += rmse(target.iloc[val_], reg.predict(data[['log_leak', _f]].iloc[val_], ntree_limit=reg.best_ntree_limit)) / folds.n_splits
scores.append(( _f, score))
from fastai.text import * 
<load_from_csv>
report = pd.DataFrame(scores, columns=['feature', 'rmse'] ).set_index('feature')
 report['nb_zeros'] = nb_zeros
report['nunique'] = nb_values
report.sort_values(by='rmse', ascending=True, inplace=True )
train = pd.read_csv(".. /input/train.tsv", sep='\t')
 test = pd.read_csv(".. /input/test.tsv", sep='\t')
 train['Sentiment'] = train['Sentiment'].apply(str)
<define_variables>
report.sort_values('rmse', ascending=False, inplace=True)
 radii = 1000 *(report['rmse'].max() - report['rmse'] ).values
source = ColumnDataSource(data=dict(
 x=report['nunique'].tolist() ,
y=report['nb_zeros'].tolist() ,
desc=report.index.tolist() ,
radius=radii,
fill_color=[
"
],
rmse=report['rmse'].tolist() 
))
TOOLTIPS = [ 
("rmse", "@rmse"), 
("(nunique, nb_zeros)", "(@x, @y)"), 
("feature", "@desc"),
]
TOOLS = "hover, crosshair, pan, wheel_zoom, zoom_in, zoom_out, box_zoom, undo, redo, reset, tap, save, box_select, poly_select, lasso_select"
p = figure(plot_width=600, plot_height=600, tooltips=TOOLTIPS, tools=TOOLS,
title="Number of unique values vs Number of zeros")
 p.xaxis.axis_label = 'Number of unique values in feature'
p.yaxis.axis_label = 'Number of zeros in feature'
p.xaxis.axis_label_text_font_style ='bold'
p.yaxis.axis_label_text_font_style ='bold'
p.title.text_color = '
p.title.text_font_size = '16pt'
p.scatter(
 'x', 'y', source=source,
radius='radius',
fill_color='fill_color',
line_color=None,
fill_alpha=0.8 
)
 output_notebook()
show(p )
test_id = test['PhraseId']<load_pretrained>
report.to_csv('feature_report.csv', index=True )
learn = language_model_learner(data,AWD_LSTM, drop_mult=0.3)
<find_best_params>
good_features = report.loc[report['rmse'] <= 0.7925].index
rmses = report.loc[report['rmse'] <= 0.7925, 'rmse'].values
good_features
learn.lr_find()<train_model>
test = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv' )
learn.fit_one_cycle(5, 1e-2, moms=(0.8,0.7)) 
<train_model>
%%time
tst_leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/test_leak.csv')
 test['leak'] = tst_leak['compiled_leak']
test['log_leak'] = np.log1p(tst_leak['compiled_leak'] )
learn.unfreeze()
learn.fit_one_cycle(5, 1e-3, moms=(0.8,0.7))<categorify>
folds = KFold(n_splits=5, shuffle=True, random_state=1)
 features = [f for f in data if f not in ['ID', 'leak', 'log_leak', 'target']]
data.replace(0, np.nan, inplace=True)
 data['log_of_mean'] = np.log1p(data[features].replace(0, np.nan ).mean(axis=1))
data['mean_of_log'] = np.log1p(data[features] ).replace(0, np.nan ).mean(axis=1)
 data['log_of_median'] = np.log1p(data[features].replace(0, np.nan ).median(axis=1))
data['nb_nans'] = data[features].isnull().sum(axis=1)
 data['the_sum'] = np.log1p(data[features].sum(axis=1))
data['the_std'] = data[features].std(axis=1)
 data['the_kur'] = data[features].kurtosis(axis=1)
 test.replace(0, np.nan, inplace=True)
 test['log_of_mean'] = np.log1p(test[features].replace(0, np.nan ).mean(axis=1))
test['mean_of_log'] = np.log1p(test[features] ).replace(0, np.nan ).mean(axis=1)
 test['log_of_median'] = np.log1p(test[features].replace(0, np.nan ).median(axis=1))
test['nb_nans'] = test[features].isnull().sum(axis=1)
 test['the_sum'] = np.log1p(test[features].sum(axis=1))
test['the_std'] = test[features].std(axis=1)
 test['the_kur'] = test[features].kurtosis(axis=1)
 features = good_features.tolist()
features = features + ['log_leak', 'log_of_mean', 'mean_of_log', 'log_of_median', 'nb_nans', 'the_sum', 'the_std', 'the_kur']
dtrain = lgb.Dataset(data=data[features],
label=target, free_raw_data=False)
 test['target'] = 0
dtrain.construct()
oof_preds = np.zeros(data.shape[0])
 for trn_idx, val_idx in folds.split(data):
lgb_params = {
'objective': 'regression',
'num_leaves': 58,
'subsample': 0.6143,
'colsample_bytree': 0.6453,
'min_split_gain': np.power(10, -2.5988),
'reg_alpha': np.power(10, -2.2887),
'reg_lambda': np.power(10, 1.7570),
'min_child_weight': np.power(10, -0.1477),
'verbose': -1,
'seed': 3,
'boosting_type': 'gbdt',
'max_depth': -1,
'learning_rate': 0.05,
'metric': 'l2',
}
clf = lgb.train(
 params=lgb_params,
train_set=dtrain.subset(trn_idx),
valid_sets=dtrain.subset(val_idx),
num_boost_round=10000,
early_stopping_rounds=100,
verbose_eval=0 
)
 oof_preds[val_idx] = clf.predict(dtrain.data.iloc[val_idx])
 test['target'] += clf.predict(test[features])/ folds.n_splits
print(mean_squared_error(target.iloc[val_idx],
oof_preds[val_idx])**.5)
 data['predictions'] = oof_preds
data.loc[data['leak'].notnull() , 'predictions'] = np.log1p(data.loc[data['leak'].notnull() , 'leak'])
 print('OOF SCORE : %9.6f'
%(mean_squared_error(target, oof_preds)**.5))
print('OOF SCORE with LEAK : %9.6f'
%(mean_squared_error(target, data['predictions'])**.5))
<categorify><EOS> 
test['target'] = np.expm1(test['target'])
 test.loc[test['leak'].notnull() , 'target'] = test.loc[test['leak'].notnull() , 'leak']
test[['ID', 'target']].to_csv('leaky_submission.csv', index=False, float_format='%.2f' )