Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<save_to_csv>
import numpy as np
import pandas as pd
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error, mean_squared_log_error
from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler
from sklearn.model_selection import KFold
print('fastText Model')
 seed_everything(1234 + 0)
 fastText_model = NeuralNet(embedding_matrix,dropout_rate=DROPOUT_RATE,lstm_units=LSTM_UNITS,dense_hidden_units=DENSE_HIDDEN_UNITS)
 fastText_learn = Learner(databunch, fastText_model, loss_func=custom_loss)
 fastText_test_preds = train_model(fastText_learn,test_dataset,output_dim=1,batch_size = batch_size,n_epochs = EPOCHS)
 fastText_model_submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': np.squeeze(fastText_test_preds[:,0])
 })
 fastText_model_submission.to_csv('submission.csv', index=False)
<choose_model_class>
%%time
data = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 target = np.log1p(data['target'])
 data.drop(['ID', 'target'], axis=1, inplace=True )
<save_to_csv>
%%time
leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/train_leak.csv')
 data['leak'] = leak['compiled_leak'].values
data['log_leak'] = np.log1p(leak['compiled_leak'].values )
<define_variables>
%%time
def rmse(y_true, y_pred):
return mean_squared_error(y_true, y_pred)**.5
reg = XGBRegressor(n_estimators=1000)
 folds = KFold(4, True, 134259)
 fold_idx = [(trn_, val_)for trn_, val_ in folds.split(data)]
scores = []
nb_values = data.nunique(dropna=False)
 nb_zeros =(data == 0 ).astype(np.uint8 ).sum(axis=0)
 features = [f for f in data.columns if f not in ['log_leak', 'leak', 'target', 'ID']]
for _f in features:
score = 0
for trn_, val_ in fold_idx:
reg.fit(
 data[['log_leak', _f]].iloc[trn_], target.iloc[trn_],
eval_set=[(data[['log_leak', _f]].iloc[val_], target.iloc[val_])],
eval_metric='rmse',
early_stopping_rounds=50,
verbose=False 
)
 score += rmse(target.iloc[val_], reg.predict(data[['log_leak', _f]].iloc[val_], ntree_limit=reg.best_ntree_limit)) / folds.n_splits
scores.append(( _f, score))
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]<define_variables>
report = pd.DataFrame(scores, columns=['feature', 'rmse'] ).set_index('feature')
 report['nb_zeros'] = nb_zeros
report['nunique'] = nb_values
report.sort_values(by='rmse', ascending=True, inplace=True )
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—' 
<categorify>
report.to_csv('feature_report.csv', index=True )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<load_from_csv>
good_features = report.loc[report['rmse'] <= 0.7955].index
rmses = report.loc[report['rmse'] <= 0.7955, 'rmse'].values
good_features
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
<prepare_x_and_y>
test = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv' )
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str )<feature_engineering>
%%time
tst_leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/test_leak.csv')
 test['leak'] = tst_leak['compiled_leak']
test['log_leak'] = np.log1p(tst_leak['compiled_leak'] )
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False )<string_transform>
folds = KFold(n_splits=5, shuffle=True, random_state=1)
 features = [f for f in data if f not in ['ID', 'leak', 'log_leak', 'target']]
data.replace(0, np.nan, inplace=True)
 data['log_of_mean'] = np.log1p(data[features].replace(0, np.nan ).mean(axis=1))
data['mean_of_log'] = np.log1p(data[features] ).replace(0, np.nan ).mean(axis=1)
 data['log_of_median'] = np.log1p(data[features].replace(0, np.nan ).median(axis=1))
data['nb_nans'] = data[features].isnull().sum(axis=1)
 data['the_sum'] = np.log1p(data[features].sum(axis=1))
data['the_std'] = data[features].std(axis=1)
 data['the_kur'] = data[features].kurtosis(axis=1)
 test.replace(0, np.nan, inplace=True)
 test['log_of_mean'] = np.log1p(test[features].replace(0, np.nan ).mean(axis=1))
test['mean_of_log'] = np.log1p(test[features] ).replace(0, np.nan ).mean(axis=1)
 test['log_of_median'] = np.log1p(test[features].replace(0, np.nan ).median(axis=1))
test['nb_nans'] = test[features].isnull().sum(axis=1)
 test['the_sum'] = np.log1p(test[features].sum(axis=1))
test['the_std'] = test[features].std(axis=1)
 test['the_kur'] = test[features].kurtosis(axis=1)
 features = good_features.tolist()
features = features + ['log_leak', 'log_of_mean', 'mean_of_log', 'log_of_median', 'nb_nans', 'the_sum', 'the_std', 'the_kur']
dtrain = lgb.Dataset(data=data[features],
label=target, free_raw_data=False)
 test['target'] = 0
dtrain.construct()
oof_preds = np.zeros(data.shape[0])
 for trn_idx, val_idx in folds.split(data):
lgb_params = {
'objective': 'regression',
'num_leaves': 58,
'subsample': 0.6143,
'colsample_bytree': 0.6453,
'min_split_gain': np.power(10, -2.5988),
'reg_alpha': np.power(10, -2.2887),
'reg_lambda': np.power(10, 1.7570),
'min_child_weight': np.power(10, -0.1477),
'verbose': -1,
'seed': 3,
'boosting_type': 'gbdt',
'max_depth': -1,
'learning_rate': 0.05,
'metric': 'l2',
}
clf = lgb.train(
 params=lgb_params,
train_set=dtrain.subset(trn_idx),
valid_sets=dtrain.subset(val_idx),
num_boost_round=10000,
early_stopping_rounds=100,
verbose_eval=0 
)
 oof_preds[val_idx] = clf.predict(dtrain.data.iloc[val_idx])
 test['target'] += clf.predict(test[features])/ folds.n_splits
print(mean_squared_error(target.iloc[val_idx],
oof_preds[val_idx])**.5)
 data['predictions'] = oof_preds
data.loc[data['leak'].notnull() , 'predictions'] = np.log1p(data.loc[data['leak'].notnull() , 'leak'])
 print('OOF SCORE : %9.6f'
%(mean_squared_error(target, oof_preds)**.5))
print('OOF SCORE with LEAK : %9.6f'
%(mean_squared_error(target, data['predictions'])**.5))
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)
 sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])* train_df[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()<concatenate>
test['target'] = np.expm1(test['target'])
 test.loc[test['leak'].notnull() , 'target'] = test.loc[test['leak'].notnull() , 'leak']
test[['ID', 'target']].to_csv('leaky_submission.csv', index=False, float_format='%.2f' )