Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<train_model>
%matplotlib inline
history=model.fit(X_train, y_train, validation_data=(X_val, y_val),epochs=6, batch_size=256, verbose=1, callbacks=callback )<save_to_csv>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 t = train_df.iloc[:,2:].duplicated(keep=False)
 duplicated_indices = t[t].index.values
print("Removed {} duplicated rows: {}".format(len(duplicated_indices), duplicated_indices))
train_df.iat[duplicated_indices[0], 1] = np.expm1(np.log1p(train_df.target.loc[duplicated_indices] ).mean())
 train_df.drop(duplicated_indices[1:], inplace=True)
 total_df = pd.concat([train_df.drop('target', axis=1), test_df], axis=0 ).drop('ID', axis=1)
 y = np.log1p(train_df.target)
 train_idx = range(0, len(train_df))
test_idx = range(len(train_df), len(total_df))
zero_std_cols = train_df.drop("ID", axis=1 ).columns[train_df.std() == 0]
total_df.drop(zero_std_cols, axis=1, inplace=True)
 print("Removed {} constant columns".format(len(zero_std_cols)))
 _, unique_indices = np.unique(total_df.iloc[train_idx], axis=1, return_index=True)
 colsToRemove = [i for i in range(total_df.shape[1])if i not in unique_indices]
total_df = total_df.iloc[:, unique_indices]
print("Dropped {} duplicate columns: {}".format(len(colsToRemove), colsToRemove))
total_df = np.log1p(total_df)
 for col in total_df.columns:
nonzero_rows = total_df[col] != 0
total_df.loc[nonzero_rows, col] = scale(total_df.loc[nonzero_rows, col].values)
y_pred=model.predict_classes(X_test)
 sub_file = pd.read_csv('.. /input/sampleSubmission.csv',sep=',')
 sub_file.Sentiment=y_pred
sub_file.to_csv('Submission.csv',index=False )<set_options>
aggregate_df = pd.DataFrame()
n0_df = total_df[total_df>0]
aggregate_df['n0_count'] = total_df.astype(bool ).sum(axis=1)
 aggregate_df['n0_mean'] = n0_df.mean(axis=1)
 aggregate_df['n0_median'] = n0_df.median(axis=1)
 aggregate_df['n0_kurt'] = n0_df.kurt(axis=1)
 aggregate_df['n0_min'] = n0_df.min(axis=1)
 aggregate_df['n0_std'] = n0_df.std(axis=1)
 aggregate_df['n0_skew'] = n0_df.skew(axis=1)
 aggregate_df['mean'] = total_df.mean(axis=1)
 aggregate_df['std'] = total_df.std(axis=1)
 aggregate_df['max'] = total_df.max(axis=1)
 aggregate_df['nunique'] = total_df.nunique(axis=1)
 aggregate_df['sum_zeros'] =(total_df == 0 ).astype(int ).sum(axis=1)
 aggregate_df['geometric_mean'] = n0_df.apply(
 lambda x: np.exp(np.log(x ).mean()), axis=1 
)
 del n0_df
aggregate_df.reset_index(drop=True, inplace=True)
 print("Created features for: {}".format(aggregate_df.columns.values))
warnings.filterwarnings(action='once')
 device = torch.device('cuda' )<string_transform>
COMPONENTS = 10
sparse_matrix = scipy.sparse.csr_matrix(total_df.values)
 methods = [
{'method': TSNE(n_components=3, init='pca'), 'data': 'train'},
{'method': TruncatedSVD(n_components=COMPONENTS), 'data': 'sparse'},
{'method': PCA(n_components=COMPONENTS), 'data': 'total'},
{'method': FastICA(n_components=COMPONENTS), 'data': 'total'},
{'method': GaussianRandomProjection(n_components=COMPONENTS, eps=0.1), 'data': 'total'},
{'method': SparseRandomProjection(n_components=COMPONENTS, dense_output=True), 'data': 'total'}
]
embeddings = []
for run in methods:
name = run['method'].__class__.__name__
if run['data'] == 'sparse':
embedding = run['method'].fit_transform(sparse_matrix)
 elif run['data'] == 'train':
embedding = run['method'].fit_transform(total_df.iloc[train_idx])
 else:
embedding = run['method'].fit_transform(total_df)
 embeddings.append(
 pd.DataFrame(embedding, columns=[f"{name}_{i}" for i in range(embedding.shape[1])])
)
 print(f">> Ran {name}")
 components_df = pd.concat(embeddings, axis=1 ).reset_index(drop=True )
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens )<define_variables>
encoding_dim = 5
enc_input = Input(( total_df.shape[1],))
enc_output = Dense(128, activation='relu' )(enc_input)
 enc_output = Dropout(0.5 )(enc_output)
 enc_output = Dense(encoding_dim, activation='relu' )(enc_output)
 dec_input = Dense(128, activation='relu' )(enc_output)
 dec_output = Dropout(0.5 )(dec_input)
 dec_output = Dense(total_df.shape[1], activation='relu' )(dec_output)
 vanilla_encoder = Model(enc_input, enc_output)
 vanilla_autoencoder = Model(enc_input, dec_output)
 vanilla_autoencoder.compile(optimizer=Adam(0.0001), loss='mean_squared_error')
 vanilla_autoencoder.summary()
vanilla_autoencoder.fit(
 total_df.values, total_df.values,
epochs=6, batch_size=64,
shuffle=True 
)
 dense_ae_df = pd.DataFrame(
 vanilla_encoder.predict(total_df.values, batch_size=64),
columns=['dense_AE_{}'.format(i)for i in range(encoding_dim)]
).reset_index(drop=True )
MAX_SEQUENCE_LENGTH = 220
SEED = 1234
BATCH_SIZE = 32
BERT_MODEL_PATH = '.. /input/bertpretrainedmodels/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'<set_options>
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1',
'15ace8c9f', 'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9',
'd6bb78916', 'b43a7cfd5', '58232a6fb', '1702b5bf0', '324921c7b',
'62e59a501', '2ec5b290f', '241f0f867', 'fb49e4212', '66ace2992',
'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7', '1931ccfdd',
'703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535', 'fc99f9426', '91f701ba2', '0572565c2',
'190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98']
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True<load_pretrained>
def moving_avg(df, prefix, win_size):
print('Creating rolling average on {} columns'.format(win_size))
ts = df.T.rolling(win_size*2, min_periods=1, center=True, win_type='gaussian' ).mean(std=win_size/2 ).iloc[ win_size//2 : : win_size ].T
ts.columns = [prefix+str(n)for n in np.arange(ts.columns.size)]
dts = pd.DataFrame(ts.iloc[:,:-1].values - ts.iloc[:,1:].values)
 dts.columns = ['d_'+prefix+str(n)for n in np.arange(dts.columns.size)]
return ts.join(dts)
 def moving_max(df, prefix, win_size):
print('Creating rolling max on {} columns'.format(win_size))
ts = df.T.rolling(win_size*2, min_periods=1, center=True ).max().iloc[ win_size//2 : : win_size ].T
ts.columns = [prefix+str(n)for n in np.arange(ts.columns.size)]
dts = pd.DataFrame(ts.iloc[:,:-1].values - ts.iloc[:,1:].values)
 dts.columns = ['d_'+prefix+str(n)for n in np.arange(dts.columns.size)]
return ts.join(dts)
 rolling_df = pd.DataFrame()
rolling_df = pd.concat([rolling_df, moving_avg(total_df[cols], 'm_', 30)], axis=1)
 rolling_df = pd.concat([rolling_df, moving_avg(total_df[cols], 'f_', 15)], axis=1)
 rolling_df = pd.concat([rolling_df, moving_avg(total_df[cols], 'd_', 7)], axis=1)
 rolling_df = pd.concat([rolling_df, moving_max(total_df[cols], 'mm_', 30)], axis=1)
 rolling_df = pd.concat([rolling_df, moving_max(total_df[cols], 'fm_', 15)], axis=1)
 rolling_df = pd.concat([rolling_df, moving_max(total_df[cols], 'dm_', 7)], axis=1)
 rolling_df.reset_index(inplace=True, drop=True )
bert_config = BertConfig('.. /input/bertjson/bert_config_new.json')
 tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True )<load_from_csv>
feature_df = pd.concat([components_df, aggregate_df, dense_ae_df, rolling_df], axis=1 ).fillna(0 )
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
 test_df['comment_text'] = test_df['comment_text'].astype(str)
 X_test = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), MAX_SEQUENCE_LENGTH, tokenizer )<load_pretrained>
dev_X, val_X, dev_y, val_y = train_test_split(feature_df.iloc[train_idx], y, test_size = 0.15, random_state = 42)
 def run_lgb(train_X, train_y, val_X, val_y, test_X=None):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 140,
"max_depth" : 13,
"max_bin" : 55,
"learning_rate" : 0.005,
"feature_fraction" : 0.9,
"bagging_fraction" : 0.8,
"bagging_frequency" : 5,
"bagging_seed" : 42,
"verbosity" : -1,
"seed": 42
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 5000,
valid_sets=[lgval],
early_stopping_rounds=100,
verbose_eval=200,
evals_result=evals_result)
 if test_X is not None:
pred_test_y = np.expm1(model.predict(test_X, num_iteration=model.best_iteration))
return pred_test_y, model, evals_result
else:
return model, evals_result
model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y)
 print("LightGBM Training Completed..." )
model = BertForSequenceClassification(bert_config, num_labels=16)
 model.load_state_dict(torch.load(".. /input/bert2epaux01wwd1e4decay95weachrebasev1/bert_pytorch.bin"))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()<categorify>
percentile = 20
skb = SelectPercentile(score_func=lambda X,y: np.mean(np.abs(shap_values[:, :dev_X.shape[1]]), axis=0), percentile=percentile ).fit(dev_X.values, dev_y.values)
 print("Using {} features".format(skb.transform(val_X.values ).shape[1]))
pred_test, s_model, s_evals_result = run_lgb(skb.transform(dev_X.values), dev_y,
skb.transform(val_X.values), val_y,
skb.transform(feature_df.iloc[test_idx].values))
<create_dataframe><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub["target"] = pred_test
print(sub.head())
 sub.to_csv('feat_lgb{:.3f}.csv'.format(evals_result['valid_0']['rmse'][-1]), index=False )