Kaggle competition: Quora Question Pairs
Add, AveragePooling1D, Multiply, GRU, GRUCell, LSTMCell, SimpleRNNCell, SimpleRNN, TimeDistributed, RNN,
RepeatVector, Conv1D, MaxPooling1D, Concatenate, GlobalAveragePooling1D, UpSampling1D)
 warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 %matplotlib inline<define_variables>
print(check_output(["ls", ".. /input"] ).decode("utf8"))
print("This is just a test")
 quora_train = pd.read_csv(".. /input/train.csv")
 quora_test = pd.read_csv(".. /input/test.csv" )
EPOCHS=120
NNBATCHSIZE=20
BATCHSIZE = 4000
SEED = 321
SELECT = True
SPLITS = 5
LR = 0.001
fe_config = [ 
(True, 4000),
]<define_variables>
print('Total number of question pairs for training: {}'.format(len(quora_train)))
 print('Duplicate pairs: {}%'.format(round(quora_train['is_duplicate'].mean() *100, 2)))
 qids = pd.Series(quora_train['qid1'].tolist() + quora_train['qid2'].tolist())
 print('Total number of questions in the training data: {}'.format(len(
 np.unique(qids))))
print('Number of questions that appear multiple times: {}'.format(np.sum(qids.value_counts() > 1)) )
def seed_everything(seed : int)-> NoReturn :
rn.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED)
<load_from_csv>
p = quora_train['is_duplicate'].mean()
print('Predicted score:', lgl(quora_train['is_duplicate'],
np.zeros_like(quora_train['is_duplicate'])+ p))
sub = pd.DataFrame({'test_id': quora_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index = False)
 print(sub.head() )
def read_data(base : os.path.abspath)-> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
train = pd.read_csv('/kaggle/input/clean-kalman/train_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/clean-kalman/test_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub 
<create_dataframe>
p = quora_train['is_duplicate'].mean()
print('Predicted score:', lgl(quora_train['is_duplicate'],
np.zeros_like(quora_train['is_duplicate'])+ p))
sub = pd.DataFrame({'test_id': quora_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index = False)
 print(sub.head() )
def batching(df : pd.DataFrame,
batch_size : int)-> pd.DataFrame :
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df 
<create_dataframe>
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
print(stops)
 plt.figure(figsize=(15, 5))
train_word_match = quora_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[quora_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[quora_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate' )
def lag_with_pct_change(df : pd.DataFrame,
shift_sizes : Optional[List]=[1, 2],
add_pct_change : Optional[bool]=False,
add_pct_change_lag : Optional[bool]=False)-> pd.DataFrame:
for shift_size in shift_sizes:
df['signal_shift_pos_'+str(shift_size)] = df.groupby('group')['signal'].shift(shift_size ).fillna(0)
 df['signal_shift_neg_'+str(shift_size)] = df.groupby('group')['signal'].shift(-1*shift_size ).fillna(0)
 if add_pct_change:
df['pct_change'] = df['signal'].pct_change()
if add_pct_change_lag:
for shift_size in shift_sizes:
df['pct_change_shift_pos_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(shift_size ).fillna(0)
 df['pct_change_shift_neg_'+str(shift_size)] = df.groupby('group')['pct_change'].shift(-1*shift_size ).fillna(0)
 return df 
<create_dataframe>
train_qs = pd.Series(quora_train['question1'].tolist() + quora_train['question2'].tolist() ).astype(str)
 test_qs = pd.Series(quora_test['question1'].tolist() + quora_test['question2'].tolist() ).astype(str)
 dist_train = train_qs.apply(len)
 dist_test = test_qs.apply(len)
 print('ok' )
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
batch_size : int)-> pd.DataFrame:
df = batching(df, batch_size=batch_size)
 if create_all_data_feats:
df = lag_with_pct_change(df, [1, 2, 3], add_pct_change=False, add_pct_change_lag=False)
 df['signal_2'] = df['signal'] ** 2
return df 
<feature_engineering>
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
train_qs.head(5)
 words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame)-> Tuple[pd.DataFrame , pd.DataFrame, List]:
use_cols = [col for col in df.columns if col not in ['index','group', 'open_channels', 'time']]
df = df.replace([np.inf, -np.inf], np.nan)
 df_test = df_test.replace([np.inf, -np.inf], np.nan)
 for col in use_cols:
col_mean = pd.concat([df[col], df_test[col]], axis=0 ).mean()
df[col] = df[col].fillna(col_mean)
 df_test[col] = df_test[col].fillna(col_mean)
 gc.collect()
return df, df_test, use_cols 
<concatenate>
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R
def augment(X: np.array, y:np.array)-> Tuple[np.array, np.array]:
X = np.vstack(( X, np.flip(X, axis=1)))
 y = np.vstack(( y, np.flip(y, axis=1)))
 return X, y<create_dataframe>
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = quora_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = quora_test.apply(tfidf_word_match_share, axis=1, raw=True)
 x_train['gen2'] = np.sqrt(train_word_match)
 x_train['gen2_tfidf'] = np.sqrt(tfidf_train_word_match)
 x_test['gen2'] = np.sqrt(x_test['tfidf_word_match'])
 x_test['gen2_tfidf'] = np.sqrt(x_test['tfidf_word_match'])
 x_train['sin_w'] = np.sin(train_word_match)
 x_train['sin_tf'] = np.sin(tfidf_train_word_match)
 x_test['sin_w'] = np.sin(x_test['word_match'])
 x_test['sin_tf'] = np.sin(x_test['tfidf_word_match'])
 x_train['2w'] = train_word_match * train_word_match
x_train['2tf'] = tfidf_train_word_match * tfidf_train_word_match
x_test['2w'] = x_test['word_match'] * x_test['word_match']
x_test['2tf'] = x_test['tfidf_word_match'] * x_test['tfidf_word_match']
y_train = quora_train['is_duplicate'].values
print('OK' )
def run_cv_model_by_batch(train : pd.DataFrame,
test : pd.DataFrame,
splits : int,
batch_col : Text,
feats : List,
sample_submission: pd.DataFrame,
nn_epochs : int,
nn_batch_size : int)-> NoReturn:
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
if n_fold < 2:
train_x, train_y = augment(train_x, train_y)
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 cb_prg = tfa.callbacks.TQDMProgressBar(leave_epoch_progress=False,leave_overall_progress=False, show_epoch_progress=False,show_overall_progress=True)
 model.fit(train_x,train_y,
epochs=nn_epochs,
callbacks=[cb_prg, cb_lr_schedule, MacroF1(model, valid_x,valid_y)],
batch_size=nn_batch_size,verbose=0,
validation_data=(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 logger.info(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ =f1_score(np.argmax(train_tr, axis=2 ).reshape(-1), np.argmax(oof_, axis=1), average = 'macro')
 logger.info(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis=1 ).astype(int)
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 display(sample_submission.head())
 np.save('oof.npy', oof_)
 np.save('preds.npy', preds_)
 return 
<set_options>
train_orig = pd.read_csv('.. /input/train.csv', header=0)
 test_orig = pd.read_csv('.. /input/test.csv', header=0)
 tic0=timeit.default_timer()
df1 = train_orig[['question1']].copy()
df2 = train_orig[['question2']].copy()
df1_test = test_orig[['question1']].copy()
df2_test = test_orig[['question2']].copy()
df2.rename(columns = {'question2':'question1'},inplace=True)
 df2_test.rename(columns = {'question2':'question1'},inplace=True)
 train_questions = df1.append(df2)
 train_questions = train_questions.append(df1_test)
 train_questions = train_questions.append(df2_test)
 train_questions.info()
train_questions.drop_duplicates(subset = ['question1'],inplace=True)
 train_questions.reset_index(inplace=True,drop=True)
 questions_dict = pd.Series(train_questions.index.values,index=train_questions.question1.values ).to_dict()
train_cp = train_orig.copy()
test_cp = test_orig.copy()
train_cp.drop(['qid1','qid2'],axis=1,inplace=True)
 test_cp['is_duplicate'] = -1
test_cp.rename(columns={'test_id':'id'},inplace=True)
 comb = pd.concat([train_cp,test_cp])
 comb['q1_hash'] = comb['question1'].map(questions_dict)
 comb['q2_hash'] = comb['question2'].map(questions_dict)
 print('ok' )
def lr_schedule(epoch):
if epoch < 40:
lr = LR
elif epoch < 50:
lr = LR / 3
elif epoch < 60:
lr = LR / 6
elif epoch < 75:
lr = LR / 9
elif epoch < 85:
lr = LR / 12
elif epoch < 100:
lr = LR / 15
else:
lr = LR / 50
return lr<normalization>
q1_vc = comb.q1_hash.value_counts().to_dict()
q2_vc = comb.q2_hash.value_counts().to_dict()
def try_apply_dict(x,dict_to_apply):
try:
return dict_to_apply[x]
except KeyError:
return 0
comb['q1_freq'] = comb['q1_hash'].map(lambda x: try_apply_dict(x,q1_vc)+ try_apply_dict(x,q2_vc))
comb['q2_freq'] = comb['q2_hash'].map(lambda x: try_apply_dict(x,q1_vc)+ try_apply_dict(x,q2_vc))
print('test')
 train_comb = comb[comb['is_duplicate'] >= 0][['id','q1_hash','q2_hash','q1_freq','q2_freq','is_duplicate']]
test_comb = comb[comb['is_duplicate'] < 0][['id','q1_hash','q2_hash','q1_freq','q2_freq']]
train_comb.drop(['is_duplicate'],axis=1,inplace=True)
 print('ok' )
class Mish(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 self.supports_masking = True
def call(self, inputs):
return inputs * K.tanh(K.softplus(inputs))
def get_config(self):
base_config = super(Mish, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def mish(x):
	return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 get_custom_objects().update({'mish': Activation(mish)} )<set_options>
print('ok' )
class Attention(Layer):

 def __init__(self, hidden_size,
num_heads = 8,
attention_dropout=.1,
trainable=True,
name='Attention'):
if hidden_size % num_heads != 0:
raise ValueError("Hidden size must be evenly divisible by the number of heads.")
 self.hidden_size = hidden_size
self.num_heads = num_heads
self.trainable = trainable
self.attention_dropout = attention_dropout
self.dense = tf.keras.layers.Dense(self.hidden_size, use_bias=False)
 super(Attention, self ).__init__(name=name)
 def split_heads(self, x):

 with tf.name_scope("split_heads"):
batch_size = tf.shape(x)[0]
length = tf.shape(x)[1]
depth =(self.hidden_size // self.num_heads)
 x = tf.reshape(x, [batch_size, length, self.num_heads, depth])
 return tf.transpose(x, [0, 2, 1, 3])
 def combine_heads(self, x):

 with tf.name_scope("combine_heads"):
batch_size = tf.shape(x)[0]
length = tf.shape(x)[2]
x = tf.transpose(x, [0, 2, 1, 3])
 return tf.reshape(x, [batch_size, length, self.hidden_size])
 def call(self, inputs):

 q = self.dense(inputs)
 k = self.dense(inputs)
 v = self.dense(inputs)
 q = self.split_heads(q)
 k = self.split_heads(k)
 v = self.split_heads(v)
 depth =(self.hidden_size // self.num_heads)
 q *= depth ** -0.5
logits = tf.matmul(q, k, transpose_b=True)
 weights = tf.nn.softmax(logits, name="attention_weights")
 if self.trainable:
weights = tf.nn.dropout(weights, 1.0 - self.attention_dropout)
 attention_output = tf.matmul(weights, v)
 attention_output = self.combine_heads(attention_output)
 attention_output = self.dense(attention_output)
 return attention_output
def compute_output_shape(self, input_shape):
return tf.TensorShape(input_shape )<compute_train_metric>
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242)
 print('ok' )
def categorical_focal_loss(gamma=2.0, alpha=0.25):

 def focal_loss(y_true, y_pred):
epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1.0-epsilon)
 cross_entropy = -y_true*K.log(y_pred)
 weight = alpha * y_true * K.pow(( 1-y_pred), gamma)
 loss = weight * cross_entropy
loss = K.sum(loss, axis=1)
 return loss
return focal_loss<choose_model_class>
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.05
params['max_depth'] = 4
params['min_child_weight'] = 3
params['gamma'] = 0
params['subsample'] = 0.9
params['colsample_bytree'] = 0.9
params['nthread'] = 4
params['seed'] = 50
params['scale_pos_weight=1'] = 1
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 450, watchlist, early_stopping_rounds=50, verbose_eval=10)
 print('ok' )
def WaveNetResidualConv1D(num_filters, kernel_size, stacked_layer):
def build_residual_block(l_input):
resid_input = l_input
for dilation_rate in [2**i for i in range(stacked_layer)]:
l_sigmoid_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='sigmoid' )(l_input)
 l_tanh_conv1d = Conv1D(
 num_filters, kernel_size, dilation_rate=dilation_rate,
padding='same', activation='mish' )(l_input)
 l_input = Multiply()([l_sigmoid_conv1d, l_tanh_conv1d])
 l_input = Conv1D(num_filters, 1, padding='same' )(l_input)
 resid_input = Add()([resid_input ,l_input])
 return resid_input
return build_residual_block
def Classifier(shape_):
num_filters_ = 16
kernel_size_ = 3
stacked_layers_ = [12, 8, 4, 1]
l_input = Input(shape=(shape_))
x = Conv1D(num_filters_, 1, padding='same' )(l_input)
 x = WaveNetResidualConv1D(num_filters_, kernel_size_, stacked_layers_[0] )(x)
 x = Conv1D(num_filters_*2, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*2, kernel_size_, stacked_layers_[1] )(x)
 x = Conv1D(num_filters_*4, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*4, kernel_size_, stacked_layers_[2] )(x)
 x = Conv1D(num_filters_*8, 1, padding='same' )(x)
 x = WaveNetResidualConv1D(num_filters_*8, kernel_size_, stacked_layers_[3] )(x)
 l_output = Dense(11, activation='softmax' )(x)
 model = models.Model(inputs=[l_input], outputs=[l_output])
 opt = Adam(lr=LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss=losses.CategoricalCrossentropy() , optimizer=opt, metrics=['accuracy'])
 return model<choose_model_class>
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = quora_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False)
 print('ok' )