Kaggle competition: NMLO Contest 3 - Regression
filepath1="weights1_best.h5"
checkpoint = ModelCheckpoint(filepath1, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = create_capsule(embedding_matrix)
 print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=6, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks,)
 model.load_weights(filepath1)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 val_pred['model1'] = y_pred
y_test = model.predict([X_test], batch_size=1024, verbose=2)
 test_pred['model1'] = y_test
f1_score, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1_score, threshold))<train_model>
train_data = pd.read_csv('/kaggle/input/nmlo-contest-3/train.csv')
 test_data = pd.read_csv('/kaggle/input/nmlo-contest-3/test.csv')
 train = train_data.copy()
test = test_data.copy()
clr = CyclicLR(base_lr=0.001, max_lr=0.002,
step_size=300., mode='exp_range',
gamma=0.99994)
 filepath2="weights2_best.h5"
checkpoint = ModelCheckpoint(filepath2, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 callbacks = [checkpoint, clr]
model = create_lstm_atten(embedding_matrix)
 print(model.summary())
 model.fit(X_train, Y_train, batch_size=1024, epochs=8, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks,)
 model.load_weights(filepath2)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 val_pred['model2'] = y_pred
y_test = model.predict([X_test], batch_size=1024, verbose=2)
 test_pred['model2'] = y_test
f1_score, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1_score, threshold))<train_model>
train.isnull().sum()
filepath3="weights3_best.h5"
checkpoint = ModelCheckpoint(filepath3, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = create_cnn(embedding_matrix)
 print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=10, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks,)
 model.load_weights(filepath3)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 val_pred['model3'] = y_pred
y_test = model.predict([X_test], batch_size=1024, verbose=2)
 test_pred['model3'] = y_test
f1_score, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1_score, threshold))<train_on_grid>
train.isnull().sum()
filepath4="weights4_best.h5"
checkpoint = ModelCheckpoint(filepath4, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = create_capsule(embedding_matrix_1)
 print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=6, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks,)
 model.load_weights(filepath4)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 val_pred['model4'] = y_pred
y_test = model.predict([X_test], batch_size=1024, verbose=2)
 test_pred['model4'] = y_test
f1_score, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1_score, threshold))<import_modules>
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import f1_score as f1<concatenate>
cases_scaled = StandardScaler().fit_transform(train['cases'][:,np.newaxis]);
low_range = cases_scaled[cases_scaled[:,0].argsort() ][:10]
high_range= cases_scaled[cases_scaled[:,0].argsort() ][-10:]
print('outer range(low)of the distribution:')
 print(low_range)
 print('
outer range(high)of the distribution:')
 print(high_range )
stack_x = np.concatenate([i for i in val_pred.values() ], axis = 1)
 stack_test = np.concatenate([i for i in test_pred.values() ], axis = 1 )<prepare_x_and_y>
from scipy.stats import norm, skew
from scipy import stats
y_test = np.zeros(( 56370,))<split>
train['cases'].replace([0], 1, inplace = True)
 train['ed'].replace([0], 1, inplace = True)
 train['inc'].replace([0], 1, inplace = True)
 train['pop'].replace([0], 1, inplace = True)
 print(( train['cases']==0 ).value_counts())
 test['ed'].replace([0], 1, inplace = True)
 test['inc'].replace([0], 1, inplace = True)
 test['pop'].replace([0], 1, inplace = True )
stack_train, oof_train, stack_train_target, oof_target = train_test_split(stack_x, Y_val, random_state = 2018, test_size = 0.2 )<define_variables>
train['cases'] = np.log(train['cases'])
oof_test = np.zeros(oof_train.shape[0] )<init_hyperparams>
res = stats.probplot(train['inc'], plot=plt)
 train['inc'] = np.log(train['inc'])
 test['inc'] = np.log(test['inc'])
params = {}
params['learning_rate'] = 0.05
params['boosting_type'] = 'gbdt'
params['objective'] = 'binary'
params['metric'] = 'binary_logloss'
params['sub_feature'] = 0.5
params['num_leaves'] = 10
params['min_data'] = 50
params['max_depth'] = 6<choose_model_class>
res = stats.probplot(train['pop'], plot=plt)
 train['pop'] = np.log(train['pop'])
 test['pop'] = np.log(test['pop'])
kfold = StratifiedKFold(n_splits=10, random_state=2018, shuffle=True )<predict_on_test>
train = pd.get_dummies(train)
 print(train.shape )
for i,(train_index, valid_index)in enumerate(kfold.split(stack_train, stack_train_target)) :
stack_x_train, stack_x_val, stack_y_train, stack_y_val = stack_train[train_index], stack_train[valid_index], stack_train_target[train_index], stack_train_target[valid_index]
d_train = lgb.Dataset(stack_x_train, label=stack_y_train)
 clf = lgb.train(params, d_train, 100)
 stack_y_pred =(clf.predict(stack_x_val)> 0.36 ).astype(int)
 f1_score = f1(stack_y_val, stack_y_pred)
 print('f1 score for fold ', i, ': ', f1_score)
 y_test +=(clf.predict(stack_test)) /10
oof_test +=(clf.predict(oof_train)) /10<categorify>
from sklearn.model_selection import train_test_split
for i in [0.30, 0.31, 0.32, 0.33, 0.34, 0.35, 0.36, 0.37, 0.38]:
print('threshold: ', i, "f1: ", f1(oof_target,(oof_test > i ).astype(int)) )<save_to_csv>
train_y = train['cases']
train.drop('id', axis = 1, inplace = True)
 train.drop('cases', axis = 1, inplace = True )
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>0.36 ).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )<split>
X_train, X_test, y_train, y_test = train_test_split(train, train_y, test_size=0.2, shuffle=True )
def tune_seed(seed):
stack_train, oof_train, stack_train_target, oof_target = train_test_split(stack_x, Y_val, random_state = seed, test_size = 0.2)
 oof_test = np.zeros(oof_target.size)
 for i,(train_index, valid_index)in enumerate(kfold.split(stack_train, stack_train_target)) :
stack_x_train, stack_x_val, stack_y_train, stack_y_val = stack_train[train_index], stack_train[valid_index], stack_train_target[train_index], stack_train_target[valid_index]
d_train = lgb.Dataset(stack_x_train, label=stack_y_train)
 clf = lgb.train(params, d_train, 100)
 oof_test +=(clf.predict(oof_train)) /10
score = f1(oof_target,(oof_test > 0.34 ).astype(int))
return score<define_variables>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
holdout_scores = []
for s in range(1999, 2020):
holdout_score = tune_seed(s)
 print('f1 score at seed ', s, ' is ', holdout_score )<count_values>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X_train)
 rmse= np.sqrt(-cross_val_score(model, X_train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
np.sum(oof_target)/(len(oof_target))<groupby>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
np.sum(pred_test_y)/len(pred_test_y )<import_modules>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU,
Conv1D ,GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D,
Conv2D, MaxPool2D, concatenate,
Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, Bidirectional, 
)
 %matplotlib inline<define_variables>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
DEBUG = False
EMBED_SIZE = 300
MAX_FEATURES = 10000 if DEBUG else 90000
SEQUENCE_LENGTH = 50
GLOVE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
WIKI = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
PARAGRAM = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'<load_from_csv>
GBoost = GradientBoostingRegressor(n_estimators=500, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 if DEBUG:
train_df = train_df[:100000]
train_df['question_text'] = train_df['question_text'].str.lower()
test_df['question_text'] = test_df['question_text'].str.lower()
DEBUG, train_df.shape, test_df.shape<string_transform>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )
def load_embedding(file, tokenizer):
def split(line):
word, *arr = line.split(' ')
 return word, np.asarray(arr, dtype='float32')
 with open(file, encoding='utf8', errors='ignore')as f:
if DEBUG:
embeddings = dict(split(line)for line in tqdm(itertools.islice(f, 10000)) if len(line)> 100)
 else:
embeddings = dict(split(line)for line in tqdm(f)if len(line)> 100)
 values = np.stack(embeddings.values())
 mean = values.mean()
std = values.std()
n_words = min(MAX_FEATURES, len(tokenizer.word_index))
embedding_matrix = np.random.normal(mean, std,(n_words, EMBED_SIZE))
for word, i in tokenizer.word_index.items() :
if i < MAX_FEATURES and word in embeddings:
embedding_matrix[i] = embeddings[word]
return embedding_matrix<choose_model_class>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
model_xgb.fit(X_train, y_train )
def model_gru_atten_3(embedding):
input_ = Input(shape=(SEQUENCE_LENGTH,))
x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding] )(input_)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(96, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(SEQUENCE_LENGTH )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<train_model>
xg = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.4603, gamma=0.0468,
gpu_id=-1, importance_type='gain', interaction_constraints='',
learning_rate=0.05, max_delta_step=0, max_depth=3,
min_child_weight=1.7817, missing=np.nan, monotone_constraints='() ',
n_estimators=2200, n_jobs=-1, nthread=-1, num_parallel_tree=1,
random_state=7, reg_alpha=0.464, reg_lambda=0.8571,
scale_pos_weight=1, silent=1, subsample=0.5213,
tree_method='exact', validate_parameters=1, verbosity=None)
class AveragedModel:
def __init__(self, tokenizer):
self.embeddings = [
load_embedding(GLOVE, tokenizer),
load_embedding(WIKI, tokenizer),
load_embedding(PARAGRAM, tokenizer),
]
self.models = [model_gru_atten_3(embedding)for embedding in self.embeddings]
def fit(self, *args, **kwargs):
for model in self.models:
model.fit(*args, **kwargs)
 def predict(self, *args, **kwargs):
outputs = [model.predict(*args, **kwargs)for model in self.models]
return np.array(outputs ).mean(axis=0 )<predict_on_test>
score_lasso = rmsle_cv(lasso)
 score_lasso.mean()
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
<save_to_csv>
score = rmsle_cv(model_xgb)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
<train_model>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
best_epochs = 2
best_threshold = 0.35
X = train_df["question_text"]
y = train_df['target'].values
tokenizer = Tokenizer(num_words=MAX_FEATURES)
 tokenizer.fit_on_texts(X)
 train_X = tokenizer.texts_to_sequences(X)
 train_X = pad_sequences(train_X, maxlen=SEQUENCE_LENGTH)
 train_y = y
model = AveragedModel(tokenizer)
 model.fit(train_X, train_y, batch_size=512, epochs=best_epochs, verbose=0)
 test_X = tokenizer.texts_to_sequences(test_df["question_text"])
 test_X = pad_sequences(test_X, maxlen=SEQUENCE_LENGTH)
 print('Tokenized test dataset')
 prediction = model.predict([test_X], batch_size=1024, verbose=0 ).flatten()
predicted_label =(prediction > best_threshold ).astype(int)
 positive = predicted_label.mean()
print(f'Tested on {model}: {positive:.3%} positive')
 out_df = pd.read_csv('.. /input/test.csv', usecols=['qid'])
 out_df = pd.DataFrame({'qid': out_df['qid'].values})
 out_df['prediction'] = predicted_label
out_df.to_csv("submission.csv", index=False )<import_modules>
score = rmsle_cv(model_lgb)
 print(" score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print(os.listdir(".. /input")) 
<compute_test_metric>
score = rmsle_cv(lasso)
 print(" score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
def calc_ratio(p):
return p/(1-p)
 def pretoken(X):
X = X.map(lambda x : re.sub(r'[^\w\s]','',x))
X = X.map(lambda x : x.lower())
 X = X.map(lambda x : x.split())
 return X
def tokenize_padding(X, max_length, token):
X = token.texts_to_sequences(X)
 X = pad_sequences(X, maxlen=max_length, padding='post')
 return X
def train_preproc(X, max_length):
X = pretoken(X)
 token = Tokenizer()
token.fit_on_texts(X_train)
 X = tokenize_padding(X,max_length,token)
 return X, token, len(token.word_index)+ 1
def test_preproc(X, max_length, token):
X = pretoken(X)
 X = tokenize_padding(X,max_length,token)
 return X
def cutoff(x, frac):
if x > frac:
return 1
else:
return 0<load_from_csv>
score = rmsle_cv(GBoost)
 GBoost.fit(X_train, y_train)
 sample = GBoost.predict(test)
 sample = np.exp(sample)
 print(" score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
trainData = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<split>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
X_train , X_dev, y_train, y_dev = train_test_split(trainData['question_text'], trainData['target'], test_size=0.2, random_state=93)
 max_sen_length = 30
X_train, token, vocab_size = train_preproc(X_train, max_sen_length)
 X_dev = test_preproc(X_dev, max_sen_length, token)
 X_test = test_preproc(test['question_text'], max_sen_length,token )<feature_engineering>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
averaged_models.fit(X_train, y_train)
 submission = averaged_models.predict(test)
 submission = np.exp(submission)
embedding_matrix = np.random.normal(glove_mean, glove_std, [vocab_size,300])
 for word, i in token.word_index.items() :
try:
embedding_matrix[i] = embeddings_index[word]
except:
continue<choose_model_class>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
model2 = Sequential()
model2.add(Embedding(vocab_size, 300, weights = [embedding_matrix], input_length = max_sen_length, trainable = False))
model2.add(SpatialDropout1D(0.2))
model2.add(Bidirectional(CuDNNLSTM(100, return_sequences = True)))
 model2.add(Bidirectional(CuDNNLSTM(100, return_sequences = True)))
 model2.add(Flatten())
 model2.add(Dense(16, activation = 'relu'))
model2.add(Dropout(0.5))
model2.add(Dense(8, activation = 'relu'))
model2.add(Dropout(0.5))
model2.add(Dense(1, activation = 'sigmoid'))
opt = keras.optimizers.Adam(lr = 1e-3, decay = 1e-5)
 model2.compile(loss = 'binary_crossentropy', optimizer = opt, metrics = [f1,'accuracy'])
 model2.fit(X_train, y_train, batch_size = 256, epochs = 5, validation_data =(X_dev,y_dev), verbose = 2 )<choose_model_class>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
filter_size = [1,2,3,4]
conv_lst = []
pool_lst = []
flat_lst = []
model_input = Input(shape=(max_sen_length,), dtype='int32', name = 'model_input')
 emb_layer = Embedding(vocab_size, 300, weights = [embedding_matrix], input_length = max_sen_length, trainable = False )(model_input)
 x = SpatialDropout1D(0.4 )(emb_layer)
 for i in range(len(filter_size)) :
conv_lst += [Conv1D(filters = 64, kernel_size = filter_size[i], padding = 'same', name = 'conv'+str(filter_size[i]))(x)]
pool_lst += [MaxPooling1D(pool_size = max_sen_length+1-filter_size[i] )(conv_lst[i])]
flat_lst += [Flatten()(pool_lst[i])]
x = concatenate(flat_lst)
 x = Dense(16, activation = 'relu' )(x)
 x = Dropout(0.5 )(x)
 x = Dense(8, activation = 'relu' )(flat_lst[i])
 model_output = Dense(1, activation = 'sigmoid' )(x)
 opt = keras.optimizers.Adam(lr = 1e-3, decay = 1e-5)
 model = Model(inputs=model_input, outputs = model_output)
 model.compile(loss = 'binary_crossentropy', optimizer = opt, metrics = [f1,'accuracy'])
 model.fit(X_train, y_train, epochs = 3, batch_size=256, validation_data =(X_dev,y_dev), verbose = 2 )<find_best_params>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
ensemble_p = 0.3
y_dev_predict = pd.DataFrame(ensemble_p*model.predict(X_dev)+(1-ensemble_p)*model2.predict(X_dev))
f1_lst = []
for i in range(1,7):
f1_lst += [f1_score(y_dev, y_dev_predict[0].map(lambda x : cutoff(x, i/10)))]
ensemble_cutoff =(np.argmax(f1_lst)+ 1)/10<save_to_csv>
I decided to use the stock lasso regression.Xgboost and Lgb were both banned for this competition .
<import_modules><EOS> 
sub = pd.DataFrame()
sub['id'] = range(0,993)
 sub['cases'] = sample
sub.to_csv('submission.csv',index=False)
 print(sub )