Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<string_transform>
from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
from catboost import CatBoostRegressor
from skopt import gp_minimize, forest_minimize
from skopt.space import Real, Categorical, Integer
from skopt.plots import plot_convergence
from skopt.plots import plot_objective, plot_evaluations
from skopt.utils import use_named_args
from sklearn.preprocessing import MinMaxScaler, StandardScaler
import matplotlib.pyplot as plt
from xgboost import XGBRegressor
from vecstack import stacking
import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error
import os
spaces = ['\u200b', '\u200e', '\u202a', '\u202c', '\ufeff', '\uf0d8', '\u2061', '\x10', '\x7f', '\x9d', '\xad', '\xa0']
def remove_space(text):

 for space in spaces:
text = text.replace(space, ' ')
 text = text.strip()
text = re.sub('\s+', ' ', text)
 return text
def remove_diacritics(s):
return ''.join(c for c in normalize('NFKD', s.replace('ø', 'o' ).replace('Ø', 'O' ).replace('⁻', '-' ).replace('₋', '-'))
if category(c)!= 'Mn')
 special_punc_mappings = {"—": "-", "–": "-", "_": "-", '”': '"', "″": '"', '“': '"', '•': '.', '−': '-',
"’": "'", "‘": "'", "´": "'", "`": "'", '\u200b': ' ', '\xa0': ' ','،':'','„':'',
'…': '...', '\ufeff': ''}
def clean_special_punctuations(text):
for punc in special_punc_mappings:
if punc in text:
text = text.replace(punc, special_punc_mappings[punc])
 text = remove_diacritics(text)
 return text
def clean_number(text):
text = re.sub(r'(\d+ )([a-zA-Z])', '\g<1> \g<2>', text)
 text = re.sub(r'(\d+ )(th|st|nd|rd)', '\g<1>\g<2> ', text)
 text = re.sub(r'(\d+),(\d+)', '\g<1>\g<2>', text)
 return text
regular_punct = list(string.punctuation)
 extra_punct = [
',', '.', '"', ':', ')', '(', '!', '?', '|', ';', "'", '$', '&',
'/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›',
'♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”',
'–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾',
'═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼',
'▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',
'，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',
'¹', '≤', '‡', '√', '«', '»', '´', 'º', '¾', '¡', '§', '£', '₤',
':)', ':)', ':-)', '(:', '(:', '(-:', ':')',
':D', ': D', ':-D', 'xD', 'x-D', 'XD', 'X-D',
'<3', ':*',
';-)', ';)', ';-D', ';D', '(;', '(-;',
':-(', ':(', ':(', ''):', ')-:',
'-- :','(', ':'(', ':"('',]
def handle_emojis(text):
text = re.sub(r'(:\s?\)|:-\)|\(\s?:|\(-:|:'\)) ', ' EMO_POS ', text)
 text = re.sub(r'(:\s?D|:-D|x-?D|X-?D)', ' EMO_POS ', text)
 text = re.sub(r'(<3|:\*)', ' EMO_POS ', text)
 text = re.sub(r'(;-?\)|;-?D|\(-?;)', ' EMO_POS ', text)
 text = re.sub(r'(:\s?\(|:-\(|\)\s?:|\)-:)', ' EMO_NEG ', text)
 text = re.sub(r'(:,\(|:'\(|:"\() ', ' EMO_NEG ', text)
 return text
def stop(text):
text = " ".join([w.lower() for w in text.split() ])
 stop_words = stopwords.words('english')
 words = [w for w in text.split() if not w in stop_words]
return " ".join(words)
 all_punct = list(set(regular_punct + extra_punct))
all_punct.remove('-')
 all_punct.remove('.')
 def clean_repeat_words(text):
text = re.sub(r"(I|i )(I|i)+ng", "ing", text)
 text = re.sub(r"(L|l )(L|l )(L|l)+y", "lly", text)
 text = re.sub(r"(A|a )(A|a )(A|a)+", "a", text)
 text = re.sub(r"(C|c )(C|c )(C|c)+", "cc", text)
 text = re.sub(r"(D|d )(D|d )(D|d)+", "dd", text)
 text = re.sub(r"(E|e )(E|e )(E|e)+", "ee", text)
 text = re.sub(r"(F|f )(F|f )(F|f)+", "ff", text)
 text = re.sub(r"(G|g )(G|g )(G|g)+", "gg", text)
 text = re.sub(r"(I|i )(I|i )(I|i)+", "i", text)
 text = re.sub(r"(K|k )(K|k )(K|k)+", "k", text)
 text = re.sub(r"(L|l )(L|l )(L|l)+", "ll", text)
 text = re.sub(r"(M|m )(M|m )(M|m)+", "mm", text)
 text = re.sub(r"(N|n )(N|n )(N|n)+", "nn", text)
 text = re.sub(r"(O|o )(O|o )(O|o)+", "oo", text)
 text = re.sub(r"(P|p )(P|p )(P|p)+", "pp", text)
 text = re.sub(r"(Q|q )(Q|q)+", "q", text)
 text = re.sub(r"(R|r )(R|r )(R|r)+", "rr", text)
 text = re.sub(r"(S|s )(S|s )(S|s)+", "ss", text)
 text = re.sub(r"(T|t )(T|t )(T|t)+", "tt", text)
 text = re.sub(r"(V|v )(V|v)+", "v", text)
 text = re.sub(r"(Y|y )(Y|y )(Y|y)+", "y", text)
 text = re.sub(r"plzz+", "please", text)
 text = re.sub(r"(Z|z )(Z|z )(Z|z)+", "zz", text)
 text = re.sub(r"(-+|\.+)", " ", text)
 return text
def spacing_punctuation(text):

 for punc in all_punct:
if punc in text:
text = text.replace(punc, f' {punc} ')
 return text
def preprocess(text):

 text = remove_space(text)
 text = clean_special_punctuations(text)
 text = handle_emojis(text)
 text = clean_number(text)
 text = spacing_punctuation(text)
 text = clean_repeat_words(text)
 text = remove_space(text)
 return text
mispell_dict = {'😉':'wink','😂':'joy','😀':'stuck out tongue', 'theguardian':'the guardian','deplorables':'deplorable', 'theglobeandmail':'the globe and mail', 'justiciaries': 'justiciary','creditdation': 'Accreditation','doctrne':'doctrine','fentayal': 'fentanyl','designation-': 'designation','CONartist' : 'con-artist','Mutilitated' : 'Mutilated','Obumblers': 'bumblers','negotiatiations': 'negotiations','dood-': 'dood','irakis' : 'iraki','cooerate': 'cooperate','COx':'cox','racistcomments':'racist comments','envirnmetalists': 'environmentalists',}
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
def correct_contraction(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x<categorify>
temp = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv')
 test_ID = temp['ID'].values
del temp
tqdm.pandas()
def text_clean_wrapper(df):
df["comment_text"] = df["comment_text"].astype('str' ).transform(preprocess)
 df['comment_text'] = df['comment_text'].transform(lambda x: correct_spelling(x, mispell_dict))
df['comment_text'] = df['comment_text'].transform(lambda x: correct_contraction(x, contraction_mapping))
return df
train = df_parallelize_run(train, text_clean_wrapper)
 test = df_parallelize_run(test, text_clean_wrapper)
 gc.enable()
del mispell_dict, all_punct, special_punc_mappings, regular_punct, extra_punct
gc.collect()<categorify>
train = pd.read_csv('.. /input/46-features-lgb/46feat_trainData.csv.gz',compression='gzip')
 test = pd.read_csv('.. /input/46-features-lgb/46feat_testData.csv.gz',compression='gzip' )
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def load_embeddings(embed_dir=EMB_PATH):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in tqdm(open(embed_dir)))
 return embedding_index
def build_embedding_matrix(word_index, embeddings_index, max_features, lower = True, verbose = True):
embedding_matrix = np.zeros(( max_features, 300))
for word, i in tqdm(word_index.items() ,disable = not verbose):
if lower:
word = word.lower()
if i >= max_features: continue
try:
embedding_vector = embeddings_index[word]
except:
embedding_vector = embeddings_index["unknown"]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_matrix(word_index, embeddings_index):
embedding_matrix = np.zeros(( len(word_index)+ 1,300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embeddings_index[word]
except:
embedding_matrix[i] = embeddings_index["unknown"]
return embedding_matrix<prepare_x_and_y>
X_train, X_test = train_test_split(train, test_size=0.2, random_state=5 )
maxlen = 220
max_features = 100000
embed_size = 300
tokenizer = Tokenizer(num_words=max_features, lower=True)
 print('fitting tokenizer')
 tokenizer.fit_on_texts(list(train[TEXT_COL])+ list(test[TEXT_COL]))
word_index = tokenizer.word_index
X_train = tokenizer.texts_to_sequences(list(train[TEXT_COL]))
y_train = train['target'].values
X_test = tokenizer.texts_to_sequences(list(test[TEXT_COL]))
X_train = pad_sequences(X_train, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 del tokenizer
gc.collect()<prepare_x_and_y>
y_train = X_train['log_target']
y_test = X_test['log_target']
t2 = 'log_target'
foo = train.columns
foo = list(foo)
 foo.remove(t2)
 colNames = foo
embeddings_index1 = load_embeddings()
embedding_matrix1 = build_matrix(word_index, embeddings_index1)
 del embeddings_index1
gc.collect()<load_pretrained>
models = [
CatBoostRegressor(iterations=200,
learning_rate=0.03,
depth=4,
loss_function='RMSE',
eval_metric='RMSE',
random_seed=99,
od_type='Iter',
od_wait=50,
logging_level='Silent'),
CatBoostRegressor(iterations=500,
learning_rate=0.06,
depth=3,
loss_function='RMSE',
eval_metric='RMSE',
random_seed=99,
od_type='Iter',
od_wait=50,
logging_level='Silent'),
XGBRegressor(eta=0.1,reg_lambda=1,reg_alpha=10),
XGBRegressor(eta=0.02,reg_lambda=1,reg_alpha=10,n_estimators=300),
XGBRegressor(eta=0.002,max_depth=15,n_estimators=200),
]
embeddings_index2 = load_embeddings(glove_path)
 embedding_matrix2 = build_matrix(word_index, embeddings_index2)
 del embeddings_index2
gc.collect()<define_variables>
S_train, S_test = stacking(models, X_train[colNames], y_train, X_test[colNames],
regression = True, metric = mean_absolute_error, n_folds = 5,
shuffle = True,random_state = 0, verbose = 2 )
embedding_matrix = embedding_matrix1*0.6 + embedding_matrix2*0.4<import_modules>
dim_learning_rate = Real(low=1e-6, high=1e-1, prior='log-uniform',name='learning_rate')
 dim_estimators = Integer(low=50, high=500,name='n_estimators')
 dim_max_depth = Integer(low=1, high=6,name='max_depth' )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<set_options>
dimensions = [dim_learning_rate,
dim_estimators,
dim_max_depth]
gc.collect()<import_modules>
default_parameters = [1e-2,300,4]
import keras.layers as L
from keras.models import Model
from keras.optimizers import Adam<choose_model_class>
def createModel(learning_rate,n_estimators,max_depth):
model = XGBRegressor(n_estimators=n_estimators,
learning_rate=learning_rate,
max_depth=max_depth,
random_state=0)
 model = model.fit(S_train, y_train)
 y_pred = model.predict(S_test)
 lv = mean_absolute_error(y_test, y_pred)
 return lv
def LSTM_ATT_BN(verbose = False, compile = True):
sequence_input = L.Input(shape=(maxlen,), dtype='int32')
 embedding_layer = L.Embedding(len(word_index)+ 1,
300,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)
 x = embedding_layer(sequence_input)
 x = L.SpatialDropout1D(0.2 )(x)
 x = L.Bidirectional(L.CuDNNGRU(64, return_sequences=True))(x)
 att = Attention(maxlen )(x)
 avg_pool1 = L.GlobalAveragePooling1D()(x)
 max_pool1 = L.GlobalMaxPooling1D()(x)
 x = L.concatenate([att,avg_pool1, max_pool1])
 x = L.Dense(128,activation='relu' )(x)
 x = L.BatchNormalization()(x)
 preds = L.Dense(1, activation='sigmoid' )(x)
 model = Model(sequence_input, preds)
 if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
 return model 
<choose_model_class>
@use_named_args(dimensions=dimensions)
 def fitness(learning_rate,n_estimators,max_depth):

 print('learning rate: {0:.1e}'.format(learning_rate))
print('estimators:', n_estimators)
 print('max depth:', max_depth)
 lv= createModel(learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth = max_depth)
 print()
print("Error: {}".format(lv))
print()
return lv
<choose_model_class>
error = fitness(default_parameters )
<choose_model_class>
search_result = gp_minimize(func=fitness,
dimensions=dimensions,
acq_func='EI',
n_calls=20,
x0=default_parameters )
<choose_model_class>
search_result.x
<choose_model_class>
model = XGBRegressor(n_estimators=search_result.x[1],
learning_rate=search_result.x[0],
max_depth=search_result.x[2],
random_state=0)
 model = model.fit(S_train, y_train )
def model_gru_atten_3() :
inp = L.Input(shape=(maxlen,))
x = L.Embedding(len(word_index)+ 1, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = L.Bidirectional(L.CuDNNGRU(128, return_sequences=True))(x)
 x = L.Bidirectional(L.CuDNNGRU(100, return_sequences=True))(x)
 x = L.Bidirectional(L.CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = L.Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<split>
y_pred = model.predict(S_test )
<define_variables><EOS> 
result = pd.DataFrame({'ID':test_ID
,'target':np.expm1(y_pred)})
 result.to_csv('46feat_stacked_ensemble_regr_models.csv', index=False )