Kaggle competition: Quora Insincere Questions Classification
def rmsle(y_true, y_pred):
return np.sqrt(mean_squared_log_error(np.expm1(y_true), np.expm1(y_pred)))
 neg_rmsle = make_scorer(rmsle, greater_is_better=False)
 params = { 'alpha':(0.001, 1), 'fit_intercept':(0,1.99)}<train_on_grid>
Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU,
Conv1D ,GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D,
Conv2D, MaxPool2D, concatenate,
Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, Bidirectional, 
)
 %matplotlib inline
def target(**params):
fit_intercept = int(params['fit_intercept'])
 fit_intercept_dict = { 0: False, 1: True }
model = Lasso(alpha = params['alpha'],
fit_intercept = fit_intercept_dict[fit_intercept],
copy_X = True)
 scores = cross_val_score(model, train_X, train_y, scoring=neg_rmsle, cv=3)
 return scores.mean()
lasso_alpha = BayesianOptimization(target, params, random_state=101)
 lasso_alpha.maximize(init_points=5, n_iter=15, acq='ucb', kappa=2 )<save_to_csv>
DEBUG = False
EMBED_SIZE = 300
MAX_FEATURES = 10000 if DEBUG else 90000
SEQUENCE_LENGTH = 50
GLOVE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
WIKI = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
PARAGRAM = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
lasso = Lasso(alpha=0.3077 ).fit(train_X, train_y)
 pd.DataFrame(np.e ** lasso.predict(test_X), columns=['Prediction'] ).to_csv('lasso_bayesian.csv', index=True, index_label='Id' )<train_on_grid>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 if DEBUG:
train_df = train_df[:100000]
train_df['question_text'] = train_df['question_text'].str.lower()
test_df['question_text'] = test_df['question_text'].str.lower()
DEBUG, train_df.shape, test_df.shape
def target(**params):
fit_intercept = int(params['fit_intercept'])
 fit_intercept_dict = { 0: False, 1: True }
model = ElasticNet(alpha = params['alpha'],
fit_intercept = fit_intercept_dict[fit_intercept],
copy_X = True)
 scores = cross_val_score(model, train_X, train_y, scoring=neg_rmsle, cv=3)
 return scores.mean()
elastic_alpha = BayesianOptimization(target, params, random_state=101)
 elastic_alpha.maximize(init_points=5, n_iter=15, acq='ucb', kappa=2 )<save_to_csv>
def load_embedding(file, tokenizer):
def split(line):
word, *arr = line.split(' ')
 return word, np.asarray(arr, dtype='float32')
 with open(file, encoding='utf8', errors='ignore')as f:
if DEBUG:
embeddings = dict(split(line)for line in tqdm(itertools.islice(f, 10000)) if len(line)> 100)
 else:
embeddings = dict(split(line)for line in tqdm(f)if len(line)> 100)
 values = np.stack(embeddings.values())
 mean = values.mean()
std = values.std()
n_words = min(MAX_FEATURES, len(tokenizer.word_index))
embedding_matrix = np.random.normal(mean, std,(n_words, EMBED_SIZE))
for word, i in tokenizer.word_index.items() :
if i < MAX_FEATURES and word in embeddings:
embedding_matrix[i] = embeddings[word]
return embedding_matrix
el1 = ElasticNet(alpha=0.3077 ).fit(train_X, train_y)
 pd.DataFrame(np.e ** el1.predict(test_X), columns=['Prediction'] ).to_csv('el1.csv', index=True, index_label='Id')
 el2 = ElasticNet(alpha=0.2426 ).fit(train_X, train_y)
 pd.DataFrame(np.e ** el2.predict(test_X), columns=['Prediction'] ).to_csv('el2.csv', index=True, index_label='Id')
 el3 = ElasticNet(alpha=0.04738 ).fit(train_X, train_y)
 pd.DataFrame(np.e ** el3.predict(test_X), columns=['Prediction'] ).to_csv('el3.csv', index=True, index_label='Id' )<save_to_csv>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
forest = RandomForestRegressor().fit(train_X, train_y)
 forest_res = np.e ** forest.predict(test_X)
 pd.DataFrame(forest_res, columns=['Prediction'] ).to_csv('random_forest.csv', index=True, index_label='Id' )<load_from_csv>
def model_gru_atten_3(embedding):
input_ = Input(shape=(SEQUENCE_LENGTH,))
x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding] )(input_)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(96, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(SEQUENCE_LENGTH )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
train_path = '/kaggle/input/restaurantrevenue/train.csv'
test_path = '/kaggle/input/restaurantrevenue/test.csv'
data_train = pd.read_csv(train_path)
 data_test = pd.read_csv(test_path)
 IDtest = data_test['Id']<sort_values>
class AveragedModel:
def __init__(self, tokenizer):
self.embeddings = [
load_embedding(GLOVE, tokenizer),
load_embedding(WIKI, tokenizer),
load_embedding(PARAGRAM, tokenizer),
]
self.models = [model_gru_atten_3(embedding)for embedding in self.embeddings]
def fit(self, *args, **kwargs):
for model in self.models:
model.fit(*args, **kwargs)
 def predict(self, *args, **kwargs):
outputs = [model.predict(*args, **kwargs)for model in self.models]
return np.array(outputs ).mean(axis=0 )
correlation['revenue'].sort_values(ascending=False )<count_missing_values>
X = train_df["question_text"]
y = train_df['target'].values
predicted_y = np.zeros_like(y, dtype='float32')
 indices = []
splitted_Xy = []
models = []
splitter = StratifiedKFold(2, random_state=0)
 for train_index, test_index in splitter.split(X, y):
indices.append(( train_index, test_index))
tokenizer = Tokenizer(num_words=MAX_FEATURES)
 tokenizer.fit_on_texts(X[train_index])
 train_X = tokenizer.texts_to_sequences(X[train_index])
 train_X = pad_sequences(train_X, maxlen=SEQUENCE_LENGTH)
 test_X = tokenizer.texts_to_sequences(X[test_index])
 test_X = pad_sequences(test_X, maxlen=SEQUENCE_LENGTH)
 train_y = y[train_index]
test_y = y[test_index]
splitted_Xy.append(( train_X, test_X, train_y, test_y))
model = AveragedModel(tokenizer)
 models.append(model )
data_train.isna().sum()<feature_engineering>
for model,(train_X, test_X, train_y, test_y),(train_index, test_index)in zip(models, splitted_Xy, indices):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(test_X, test_y), verbose=0)
 yp = model.predict([test_X], batch_size=512, verbose=0 ).flatten()
predicted_y[test_index] = yp
def count_years(open_date):
date_parse = datetime.strptime(open_date, '%m/%d/%Y')
 date_now = datetime.now()
return date_now.year - date_parse.year<create_dataframe>
threshold = 0.5
confusion_matrix(y,(predicted_y > threshold ).astype(int))
open_years = []
for i in data_train['Open Date']:
open_years.append(count_years(i))
df_open_years = pd.DataFrame({ 'open_years' : open_years })
 group_years = df_open_years['open_years'].value_counts()<count_values>
f1_score(y, predicted_y > threshold )
city_most = data_train['City'].value_counts() [data_train['City'].value_counts() > 2].index<groupby>
def find_best_threshold(y, predicted_y):
best_f1, best_threshold = 0, 0
for threshold in np.arange(0.01, 0.99, 0.01):
f1 = f1_score(y, predicted_y > threshold)
 if f1 > best_f1:
best_f1, best_threshold = f1, threshold
return best_f1, best_threshold
best_f1, best_threshold = find_best_threshold(y, predicted_y)
 best_f1, best_threshold
city_transform = []
for i in data_train['City']:
if i in city_most:
city_transform.append(i)
 else:
city_transform.append('other')
 df_city_transform = pd.DataFrame({ 'city_transform' : city_transform })
 group_city = df_city_transform['city_transform'].value_counts()<prepare_output>
rocs = [(fpr, tpr, thresholds, roc_auc, best_f1, best_threshold)]
for i in range(3):
for model,(train_X, test_X, train_y, test_y),(train_index, test_index)in zip(models, splitted_Xy, indices):
model.fit(train_X, train_y, batch_size=1024, epochs=1, validation_data=(test_X, test_y), verbose=0)
 yp = model.predict([test_X], batch_size=1024, verbose=0 ).flatten()
predicted_y[test_index] = yp
fpr_, tpr_, thresholds_ = roc_curve(y, predicted_y, pos_label=1)
 roc_auc_ = auc(fpr_, tpr_)
 best_f1, best_threshold = find_best_threshold(y, predicted_y)
 rocs.append(( fpr_, tpr_, thresholds_, roc_auc_, best_f1, best_threshold))
print(f'Epoch {i + 2}: ROC_AUC={roc_auc_}')
 print('Confusion matrix', confusion_matrix(y,(predicted_y > best_threshold ).astype(int)))
 rocs
<define_variables><EOS> 
aucs = [i[3] for i in rocs]
argmax = np.argmax(aucs)
 best_epochs = argmax + 1
best_threshold = rocs[argmax][5]
aucs, best_epochs, aucs[argmax], best_threshold