Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<categorify>
tqdm.pandas()
n_folds = 5
def rmsle_cv(model,df):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(df.values)
 rmse= np.sqrt(-cross_val_score(model, df.values, y_reg, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 frequency_encoding_all = df_all.copy()
def frequency_encoding(frame, col):
freq_encoding = frame.groupby([col] ).size() /frame.shape[0]
freq_encoding = freq_encoding.reset_index().rename(columns={0:'{}_Frequency'.format(col)})
 return frame.merge(freq_encoding, on=col, how='left')
 for col in categorical_features:
frequency_encoding_all = frequency_encoding(frequency_encoding_all, col)
 frequency_encoding_all = frequency_encoding_all.drop(categorical_features,axis=1, inplace=False)
 del frequency_encoding_all['SalePrice']
frequency_encoding_train = frequency_encoding_all[:len_train]
frequency_encoding_test = frequency_encoding_all[len_train:]
del frequency_encoding_all
model = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model,frequency_encoding_train)
 print("Frequency encoding(5-folds)LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
kfold_lgb_model = model.fit(frequency_encoding_train, y_reg)
 train_prediction = model.predict(frequency_encoding_train)
 prediction = np.expm1(model.predict(frequency_encoding_test.values))
submission = pd.DataFrame({'Id': Id, 'SalePrice': prediction})
 submission.to_csv('Frequency_5FOLD.csv', index=False )<compute_train_metric>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 seed_everything()
n_folds = frequency_encoding_train.shape[0]
def rmsle_cv(model,df):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(df.values)
 rmse= np.sqrt(-cross_val_score(model, df.values, y_reg, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 model = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model,frequency_encoding_train)
 print("Label encoding(LOOCV)LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
loovc_lgb_model = model.fit(frequency_encoding_train, y_reg)
 train_prediction = model.predict(frequency_encoding_train)
 prediction = np.expm1(loovc_lgb_model.predict(frequency_encoding_test.values))
submission = pd.DataFrame({'Id': Id, 'SalePrice': prediction})
 submission.to_csv('Frequency_LOOCV.csv', index=False )<categorify>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
def clean_tag(text):
if '[math]' in text:
text = re.sub('\[math\].*?math\]', '[formula]', text)
 if 'http' in text or 'www' in text:
text = re.sub('(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-?=%.]+', '[url]', text)
 return text
train["question_text"] = train["question_text"].apply(lambda x: clean_tag(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_tag(x))
one_hot_encoding = pd.concat([one_hot_encoding_train,one_hot_encoding_test])
 one_hot_encoding['Total_sqr_footage'] = one_hot_encoding['BsmtFinSF1'] + one_hot_encoding['BsmtFinSF2'] + one_hot_encoding['1stFlrSF'] + one_hot_encoding['2ndFlrSF']
one_hot_encoding['Total_Bathrooms'] = one_hot_encoding['FullBath'] + one_hot_encoding['HalfBath'] + one_hot_encoding['BsmtFullBath'] + one_hot_encoding['BsmtHalfBath']
one_hot_encoding['Total_porch_sf'] = one_hot_encoding['OpenPorchSF'] + one_hot_encoding['3SsnPorch'] + one_hot_encoding['EnclosedPorch'] + one_hot_encoding['ScreenPorch'] + one_hot_encoding['WoodDeckSF']
one_hot_encoding['TotalHouse'] = one_hot_encoding['TotalBsmtSF'] + one_hot_encoding['1stFlrSF'] + one_hot_encoding['2ndFlrSF']
one_hot_encoding['TotalArea'] = one_hot_encoding['TotalBsmtSF'] + one_hot_encoding['1stFlrSF'] + one_hot_encoding['2ndFlrSF'] + one_hot_encoding["GarageArea"]
one_hot_encoding['GrLivArea_OverallQual'] = one_hot_encoding['GrLivArea'] * one_hot_encoding['OverallQual']
one_hot_encoding['LotArea_OverallQual'] = one_hot_encoding['LotArea'] * one_hot_encoding['OverallQual']<categorify>
puncts=[',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′',
'█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓',
'—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━',
'⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○',
'◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐',
'├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\x97', '↺',
'∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊',
'┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´',
'↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\x93', '≧', '］',
'\x94', '∀', '♛', '\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％',
'╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭',
'┊', '╯', '☾', '➔', '∴', '\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐',
'▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\x91', '➚', '◡', '╰', '\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒',
'☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜',
'‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢',
'≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥',
'⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ',
'✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦',
'◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ',
'⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋',
'､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞',
'◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ',
'✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱',
'☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹',
'⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚',
'☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛',
'⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙',
'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘',
'˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛',
'☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑',
'✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ',
'⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘',
'⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊',
'⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷',
'◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝',
'⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢',
'≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']
<categorify>
%%time
def clean_punct(x):
for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_punct(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_punct(x))
one_hot_encoding['haspool'] = one_hot_encoding['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 one_hot_encoding['hasgarage'] = one_hot_encoding['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 one_hot_encoding['hasbsmt'] = one_hot_encoding['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 one_hot_encoding['hasfireplace'] = one_hot_encoding['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<sort_values>
embed_size = 300
max_features = 200000
maxlen = 72
X = train["question_text"].fillna("_
X_test = test["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features, filters='')
 tokenizer.fit_on_texts(list(X)+list(X_test))
X = tokenizer.texts_to_sequences(X)
 X_test = tokenizer.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 Y = train['target'].values
sub = test[['qid']]
del train, test
gc.collect()
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in one_hot_encoding.columns:
if one_hot_encoding[i].dtype in numeric_dtypes:
numerics2.append(i)
 skew_features = one_hot_encoding[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)
 skews = pd.DataFrame({'skew':skew_features})
 skews.head()<choose_model_class>
lem = WordNetLemmatizer()
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" "))
for o in open(EMBEDDING_FILE)
 if o.split(" ")[0] in word_index or o.split(" ")[0].lower() in word_index)
 emb_mean, emb_std = -0.005838499, 0.48782197
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
elif embeddings_index.get(word.capitalize())is not None:
embedding_matrix[i] = embeddings_index.get(word.capitalize())
 elif embeddings_index.get(word.upper())is not None:
embedding_matrix[i] = embeddings_index.get(word.upper())
 del embeddings_index
gc.collect()
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" "))
for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')
 if len(o)>100 and(o.split(" ")[0] in word_index or o.split(" ")[0].lower() in word_index))
emb_mean, emb_std = -0.005838499, 0.48782197
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
elif embeddings_index.get(word.capitalize())is not None:
embedding_matrix[i] = embeddings_index.get(word.capitalize())
 elif embeddings_index.get(word.upper())is not None:
embedding_matrix[i] = embeddings_index.get(word.upper())
 del embeddings_index
gc.collect()
return embedding_matrix
seed_everything()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean(( 1.28*embedding_matrix_1, 0.72*embedding_matrix_3), axis=0)
 del embedding_matrix_1, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )
one_hot_encoding_train = one_hot_encoding[:len_train]
one_hot_encoding_test = one_hot_encoding[len_train:]
del one_hot_encoding
kfolds = KFold(n_splits=5, shuffle=True, random_state=42)
 def cv_rmse(model):
rmse = np.sqrt(-cross_val_score(model, one_hot_encoding_train, y_reg,
scoring="neg_mean_squared_error",
cv = kfolds))
return(rmse)
 def ridge_selector(k):
ridge_model = make_pipeline(RobustScaler() ,
RidgeCV(alphas = [k],
cv=kfolds)).fit(one_hot_encoding_train, y_reg)
 ridge_rmse = cv_rmse(ridge_model ).mean()
return(ridge_rmse)
 r_alphas = [.0001,.0003,.0005,.0007,.0009, 
.01, 0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 20, 30, 50, 60, 70, 80]
ridge_scores = []
for alpha in r_alphas:
score = ridge_selector(alpha)
 ridge_scores.append(score)
 plt.plot(r_alphas, ridge_scores, label='Ridge')
 plt.legend('center')
 plt.xlabel('alpha')
 plt.ylabel('score' )<train_on_grid>
class AttentionWeightedAverage(Layer):

 def __init__(self, return_attention=False, **kwargs):
self.init = initializers.RandomUniform(seed=10000)
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None
class AdamW(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,
epsilon=1e-8, decay=0., **kwargs):
super(AdamW, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.wd = K.variable(weight_decay, name='weight_decay')
 self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
wd = self.wd
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)- lr * wd * p
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.wd)) ,
'epsilon': self.epsilon}
base_config = super(AdamW, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
alphas_alt = [9,9.1,9.2,9.3,9.4,9.5,9.6,9.7,9.8,9.9,10,10.1,10.2]
ridge_model2 = make_pipeline(RobustScaler() ,
RidgeCV(alphas = alphas_alt,
cv=kfolds)).fit(one_hot_encoding_train, y_reg)
 print("Ridge rmse : ",cv_rmse(ridge_model2 ).mean() )<train_model>
def LSTM_GRU(spatialdropout=0.20, rnn_units=64, weight_decay=0.07):
K.clear_session()
x_input = Input(shape=(maxlen,))
emb = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False, name='Embedding' )(x_input)
 emb = SpatialDropout1D(spatialdropout, seed=1024 )(emb)
 rnn1 = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True, kernel_initializer=glorot_uniform(seed=111100),
recurrent_initializer=Orthogonal(gain=1.0, seed=123000)) )(emb)
 rnn2 = Bidirectional(CuDNNGRU(rnn_units, return_sequences=True, kernel_initializer=glorot_uniform(seed=111000),
recurrent_initializer=Orthogonal(gain=1.0, seed=1203000)) )(rnn1)
 x = concatenate([rnn1, rnn2])
 x = GlobalMaxPooling1D()(x)
 x_output = Dense(1, activation='sigmoid', kernel_initializer=glorot_uniform(seed=111100))(x)
 model = Model(inputs=x_input, outputs=x_output)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay),)
 return model
print("Best of alpha in ridge model :" ,ridge_model2.steps[1][1].alpha_ )<train_model>
def poolRNN(spatialdropout=0.2, gru_units=128, weight_decay=0.04):
K.clear_session()
inp = Input(shape=(maxlen,))
embedding_layer = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False )(inp)
 embedding_layer = SpatialDropout1D(spatialdropout, seed=1024 )(embedding_layer)
 rnn_1 = Bidirectional(CuDNNGRU(gru_units, return_sequences=True,
kernel_initializer=glorot_uniform(seed=10000),
recurrent_initializer=Orthogonal(gain=1.0, seed=123000)) )(embedding_layer)
 last = Lambda(lambda t: t[:, -1], name='last' )(rnn_1)
 maxpool = GlobalMaxPooling1D()(rnn_1)
 attn = AttentionWeightedAverage()(rnn_1)
 average = GlobalAveragePooling1D()(rnn_1)
 c = concatenate([last, maxpool, attn], axis=1)
 c = Reshape(( 3, -1))(c)
 c = Lambda(lambda x:K.sum(x, axis=1))(c)
 x = BatchNormalization()(c)
 x = Dense(200, activation='relu', kernel_initializer=glorot_uniform(seed=111000))(x)
 x = Dropout(0.2, seed=1024 )(x)
 x = BatchNormalization()(x)
 output_layer = Dense(1, activation="sigmoid", kernel_initializer=glorot_uniform(seed=111000))(x)
 model = Model(inputs=inp, outputs=output_layer)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model
alphas2 = [0.00005, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005,
0.0006, 0.0007, 0.0008]
lasso_model2 = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7,
alphas = alphas2,
random_state = 42)).fit(one_hot_encoding_train, y_reg)
 scores = lasso_model2.steps[1][1].mse_path_
plt.plot(alphas2, scores, label='Lasso')
 plt.legend(loc='center')
 plt.xlabel('alpha')
 plt.ylabel('RMSE')
 plt.tight_layout()
plt.show()<compute_test_metric>
def BiLSTM_CNN(spatialdropout=0.2, rnn_units=128, filters=[100, 80, 30, 12], weight_decay=0.10):
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=spatialdropout, seed=10000 )(x)
 x = Bidirectional(CuDNNLSTM(rnn_units, return_sequences=True,
kernel_initializer=glorot_uniform(seed=111000),
recurrent_initializer=Orthogonal(gain=1.0, seed=123000)) )(x)
 x1 = Conv1D(filters=filters[0], activation='relu', kernel_size=1,
padding='same', kernel_initializer=glorot_uniform(seed=110000))(x)
 x2 = Conv1D(filters=filters[1], activation='relu', kernel_size=2,
padding='same', kernel_initializer=glorot_uniform(seed=120000))(x)
 x3 = Conv1D(filters=filters[2], activation='relu', kernel_size=3,
padding='same', kernel_initializer=glorot_uniform(seed=130000))(x)
 x4 = Conv1D(filters=filters[3], activation='relu', kernel_size=5,
padding='same', kernel_initializer=glorot_uniform(seed=140000))(x)
 x1 = GlobalMaxPool1D()(x1)
 x2 = GlobalMaxPool1D()(x2)
 x3 = GlobalMaxPool1D()(x3)
 x4 = GlobalMaxPool1D()(x4)
 c = concatenate([x1, x2, x3, x4])
 x = Dense(200, activation='relu', kernel_initializer=glorot_uniform(seed=111000))(c)
 x = Dropout(0.2, seed=10000 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid", kernel_initializer=glorot_uniform(seed=110000))(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model
print("Best of alpha in lasso model :",lasso_model2.steps[1][1].alpha_)
 print("lasso rmse : ",cv_rmse(lasso_model2 ).mean() )<train_on_grid>
def singleRNN(spatialdropout=0.20, rnn_units=120, weight_decay=0.08):
K.clear_session()
inp = Input(shape=(maxlen,))
embedding_layer = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False )(inp)
 embedding_layer = SpatialDropout1D(spatialdropout, seed=1024 )(embedding_layer)
 x = Bidirectional(CuDNNGRU(rnn_units, return_sequences=True,
kernel_initializer=glorot_uniform(seed=111000),
recurrent_initializer=Orthogonal(gain=1.0, seed=123000)) )(embedding_layer)
 x = AttentionWeightedAverage()(x)
 x = Dense(100, kernel_initializer=glorot_uniform(seed=111000))(x)
 x = Dropout(0.12, seed=1024 )(x)
 x = BatchNormalization()(x)
 x = ReLU()(x)
 output_layer = Dense(1, activation="sigmoid", kernel_initializer=glorot_uniform(seed=111000))(x)
 model = Model(inputs=inp, outputs=output_layer)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=weight_decay))
return model
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
elastic_cv = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
elastic_model3 = elastic_cv.fit(one_hot_encoding_train, y_reg)
 print("elastic model rmse : ",cv_rmse(elastic_model3 ).mean() )<train_model>
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2
xgb3 = XGBRegressor(learning_rate =0.01, n_estimators=5200, max_depth=3,
min_child_weight=0 ,gamma=0, subsample=0.7,
colsample_bytree=0.7,objective= 'reg:linear',
nthread=4,scale_pos_weight=1,seed=27, reg_alpha=0.00006)
 xgb_fit = xgb3.fit(one_hot_encoding_train, y_reg)
 print("Xgboost model rmse : ",cv_rmse(xgb_fit ).mean() )<choose_model_class>
f1, threshold = f1_smart(np.squeeze(Y[val_list]), np.squeeze(oof[val_list]))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))
<create_dataframe><EOS> 
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>threshold ).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )