Kaggle competition: Santander Value Prediction Challenge
def load_and_proc() :
train['comment_text'] = train['comment_text'].apply(lambda x : clean_text(x))
test['comment_text'] = test['comment_text'].apply(lambda x : clean_text(x))
train['comment_text'] = train['comment_text'].apply(lambda x : clean_numbers(x))
test['comment_text'] = test['comment_text'].apply(lambda x : clean_numbers(x))
train['comment_text'] = train['comment_text'].apply(lambda x : clean_contractions(x,contraction_mapping))
test['comment_text'] = test['comment_text'].apply(lambda x : clean_contractions(x,contraction_mapping))
df_train, df_valid = train_test_split(train, test_size=0.33)
 df_test = test
df_train.loc[:,'set_'] = 'train'
df_valid.loc[:,'set_'] = 'valid'
df_test.loc[:,'set_'] = 'test'
set_indices = df_train.loc[:,'set_']
set_indices.append(df_valid.loc[:,'set_'])
 set_indices.append(df_test.loc[:,'set_'])
 y_train = np.asarray(df_train['target'])
 y_valid = np.asarray(df_valid['target'])
 set_indices_label = df_train.loc[:,'set_']
set_indices_label = set_indices_label.append(df_valid.loc[:,'set_'])
 X_train = df_train['comment_text'].fillna('_
X_val = df_valid['comment_text'].fillna('_
X_test = df_test['comment_text'].fillna('_
all_text = list(X_train)
 all_text.append(list(X_val))
all_text.append(list(X_test))
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(all_text)
 X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(X_test)
 X_train = pad_sequences(X_train, maxlen=maxlen)
 X_val = pad_sequences(X_val, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 np.random.seed(2019)
 train_idx = np.random.permutation(len(X_train))
valid_idx = np.random.permutation(len(X_val))
print(type(X_train))
print(type(y_train))
X_train = X_train[train_idx]
X_val = X_val[valid_idx]
Y_train = y_train[train_idx]
Y_val = y_valid[valid_idx]
return X_train, X_val, X_test, Y_train, Y_val, tokenizer.word_index<split>
class SklearnWrapper(object):
def __init__(self, clf, seed=0, params=None, seed_bool = True):
if(seed_bool == True):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def get_oof(clf, x_train, y, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(kf):
print('
Fold {}'.format(i))
x_tr = x_train.iloc[train_index]
y_tr = y[train_index]
x_te = x_train.iloc[test_index]
clf.train(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)
X_train, X_val,X_test, Y_train, Y_val, word_index = load_and_proc()<set_options>
train_df = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 test_df = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv' )
gc.collect()
del train
del test<split>
X_train = train_df.drop(["ID", "target"], axis=1)
 y_train = np.log1p(train_df["target"].values)
 X_test = test_df.drop(["ID"], axis=1)
 ntrain = len(X_train)
 ntest = len(X_test )
def load_glove() :
EMBEDDING_FILE = '.. /input/emb-model/glove.840B.300d.txt'
def get_coef(word, *arr): return word, np.asarray(arr, dtype=np.float32)[:300]
embeddings_index = dict(get_coef(*o.split(" ")) for o in open(EMBEDDING_FILE))
return embeddings_index<split>
colsToRemove = []
for col in X_train.columns:
if X_train[col].std() == 0:
colsToRemove.append(col)
 X_train.drop(colsToRemove, axis=1, inplace=True)
 X_test.drop(colsToRemove, axis=1, inplace=True)
def load_paragram() :
EMBEDDING_FILE = '.. /input/paragram-300-sl999/paragram_300_sl999.txt'
def get_coef(word, *arr): return word, np.asarray(arr, dtype=np.float32)[:300]
embeddings_index = dict(get_coef(*o.split(" ")) for o in open(EMBEDDING_FILE, errors='ignore', encoding='utf-8'))
return embeddings_index<split>
colsToRemove = []
colsScaned = []
dupList = {}
columns = X_train.columns
for i in range(len(columns)-1):
v = X_train[columns[i]].values
dupCols = []
for j in range(i+1,len(columns)) :
if np.array_equal(v, X_train[columns[j]].values):
colsToRemove.append(columns[j])
 if columns[j] not in colsScaned:
dupCols.append(columns[j])
 colsScaned.append(columns[j])
 dupList[columns[i]] = dupCols
X_train.drop(colsToRemove, axis=1, inplace=True)
 X_test.drop(colsToRemove, axis=1, inplace=True )
def load_fasttext() :
EMBEDDING_FILE = '.. /input/fasttext-wiki-news-300d-1m/wiki-news-300d-1M.vec'
def get_coef(word, *arr): return word, np.asarray(arr, dtype=np.float32)
 embeddings_index = dict(get_coef(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)> 100)
 return embeddings_index<init_hyperparams>
weight =(( X_train != 0 ).sum() /len(X_train)).values
tmp_train = X_train[X_train!=0].fillna(0)
 tmp_test = X_test[X_test!=0].fillna(0)
 X_train["weight_count"] =(tmp_train*weight ).sum(axis=1)
 X_test["weight_count"] =(tmp_test*weight ).sum(axis=1)
 X_train["count_not0"] =(X_train != 0 ).sum(axis=1)
 X_test["count_not0"] =(X_test != 0 ).sum(axis=1)
 X_train["sum"] = X_train.sum(axis=1)
 X_test["sum"] = X_test.sum(axis=1)
 X_train["var"] = X_train.var(axis=1)
 X_test["var"] = X_test.var(axis=1)
 X_train["mean"] = X_train.mean(axis=1)
 X_test["mean"] = X_test.mean(axis=1)
 X_train["std"] = X_train.std(axis=1)
 X_test["std"] = X_test.std(axis=1)
 X_train["max"] = X_train.max(axis=1)
 X_test["max"] = X_test.max(axis=1)
 X_train["min"] = X_train.min(axis=1)
 X_test["min"] = X_test.min(axis=1)
def create_emb(word_index, embeddings_index):
emb_mean,emb_std = -0.005838499,0.48782197
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
count_found = nb_words
for word, i in tqdm(word_index.items()):
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 word_sent = TextBlob(word ).sentiment
extra_embed = [word_sent.polarity, word_sent.subjectivity]
if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = word.lower()
embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = word.upper()
embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = word.capitalize()
embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = ps.stem(word)
 embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = ls.stem(word)
 embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
key = ss.stem(word)
 embedding_vector = embeddings_index.get(key)
 if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, extra_embed)
 continue
embedding_matrix[i,300:] = extra_embed
count_found -= 1
print('Total words ', nb_words)
 print("Got embedding for ",count_found," words.")
 return embedding_matrix<define_variables>
SEED = 5
NFOLDS = 5
kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)
 ridge_params = {'alpha':30.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,
'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}
ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)
 ridge_oof_train, ridge_oof_test = get_oof(ridge, X_train, y_train, X_test)
final_matrix = glove_embedding<set_options>
rms = sqrt(mean_squared_error(y_train, ridge_oof_train))
print('Ridge OOF RMSE: {}'.format(rms))
print("Modeling Stage")
 X_train["ridge_preds"] = ridge_oof_train
X_test["rigde_preds"] = ridge_oof_test
gc.collect()<concatenate>
dev_X, val_X, dev_y, val_y = train_test_split(X_train, y_train, test_size = 0.2, random_state = 42 )
def dot_product(x, kernel):

 if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)) , axis=-1)
 else:
return K.dot(x, kernel )<choose_model_class>
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 40,
"learning_rate" : 0.005,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.5,
"bagging_frequency" : 5,
"bagging_seed" : 42,
"verbosity" : -1,
"seed": 42
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 5000,
valid_sets=[lgval],
early_stopping_rounds=100,
verbose_eval=50,
evals_result=evals_result)
 pred_test_y = np.expm1(model.predict(test_X, num_iteration=model.best_iteration))
return pred_test_y, model, evals_result
class Attention(Layer):
def __init__(self,
W_regulariser=None,b_regulariser=None,u_regulariser=None,
W_constraint=None,b_constraint=None,u_constraint=None,
biase=True, **kwargs):
self.support_masking = True
self.initializer = initializers.get('glorot_uniform')
 self.W_regulariser = regularizers.get(W_regulariser)
 self.b_regulariser = regularizers.get(b_regulariser)
 self.u_regulariser = regularizers.get(u_regulariser)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.u_constraint = constraints.get(u_constraint)
 self.biase = biase
super(Attention,self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[-1],input_shape[-1],),
initializer=self.initializer,
name='{}_W'.format(self.name),
regularizer=self.W_regulariser,
constraint=self.W_constraint)
 if self.biase:
self.b = self.add_weight(shape=(input_shape[-1],),
initializer = 'zero',
name='{}_b'.format(self.name),
regularizer=self.b_regulariser,
constraint=self.b_constraint 
)
 self.u = self.add_weight(shape=(input_shape[-1],),
initializer=self.initializer,
name='{}_u'.format(self.name),
regularizer=self.u_regulariser,
constraint=self.u_constraint)
 super(Attention, self ).build(input_shape)
 def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
uit = dot_product(x, self.W)
 if self.biase:
uit += self.b
uit = K.tanh(uit)
 ait = dot_product(uit, self.u)
 a = K.exp(ait)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], input_shape[-1] 
<choose_model_class>
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, X_test)
 print("LightGBM Training Completed..." )
def model_lstm_du(final_embedding):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[final_embedding], trainable = False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x1)
 x1 = Conv1D(64, kernel_size=3, padding = "valid", kernel_initializer = "he_uniform" )(x1)
 x2 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x2)
 x2 = Conv1D(64, kernel_size=4, padding = "valid", kernel_initializer = "he_uniform" )(x2)
 avg_pool1 = GlobalAveragePooling1D()(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 avg_pool2 = GlobalAveragePooling1D()(x2)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = concatenate([avg_pool1, max_pool1,avg_pool2, max_pool2])
 conc = Dense(128, activation='relu' )(conc)
 conc = Dense(64, activation='relu' )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs = inp, outputs = outp)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr = 1e-3, decay=0.0), metrics=['accuracy'])
 return model<set_options>
sub = pd.read_csv('.. /input/santander-value-prediction-challenge/sample_submission.csv')
 sub["target"] = pred_test
print(sub.head())
gc.collect()<train_model>
sub.to_csv('sub_lgbm_ridge_agg.csv', index=False )