Kaggle competition: Quora Question Pairs
<SOS> metric: LogLoss Kaggle data source: quora-question-pairs<define_variables>
tr = pd.read_csv('.. /input/train.csv')
 te = pd.read_csv('.. /input/test.csv')
 SCALE = 0.3627
EPOCHS = 180
NNBATCHSIZE = 16
GROUP_BATCH_SIZE = 4000
SEED = 321
LR = 0.0015
SPLITS = 6
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<load_from_csv>
def word_match_share(x):

 stops = set(stopwords.words('english'))
q1 = x.question1.fillna(' ' ).str.lower().str.split()
q2 = x.question2.fillna(' ' ).str.lower().str.split()
q1 = q1.map(lambda l : set(l)- stops)
 q2 = q2.map(lambda l : set(l)- stops)
 q = pd.DataFrame({'q1':q1, 'q2':q2})
 q['len_inter'] = q.apply(lambda row : len(row['q1'] & row['q2']), axis=1)
 q['len_tot'] = q.q1.map(len)+ q.q2.map(len)
 return(2 * q.len_inter / q.len_tot ).fillna(0 )
def read_data() :
train = pd.read_csv('/kaggle/input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def cbr(x, out_layer, kernel, stride, dilation):
x = Conv1D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = cbr(inp, 64, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 16, 3, 12)
 x = BatchNormalization()(x)
 x = wave_block(x, 32, 3, 8)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 4)
 x = BatchNormalization()(x)
 x = wave_block(x, 128, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 64, 3, 1)
 x = cbr(x, 32, 7, 1, 1)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 30:
lr = LR
elif epoch < 40:
lr = LR / 3
elif epoch < 50:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=5)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 model.save("model-wavenet.h5")
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis = 1 ).astype(int)
 sample_submission.to_csv('submission_wavenet.csv', index=False, float_format='%.4f')
 def run_everything() :
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training completed...')
 run_everything()<install_modules>
def bin_model(tr, te, bins=100, vpos=1, vss=3):

 tr['word_share'] = word_match_share(tr)
 tr['binned_share'] =(bins * tr.word_share ).round()
pos = tr.groupby('binned_share' ).is_duplicate.sum()
cts = tr.binned_share.value_counts()
te['word_share'] = word_match_share(te)
 te['binned_share'] =(bins * te.word_share ).round()
te_pos = te.binned_share.map(pos, na_action='ignore' ).fillna(0)
 te_cts = te.binned_share.map(cts, na_action='ignore' ).fillna(0)
 prob =(te_pos + vpos)/(te_cts + vss)
 odds = prob /(1 - prob)
 scaled_odds = SCALE * odds
scaled_prob = scaled_odds /(1 + scaled_odds)
 sub = te[['test_id']].copy()
sub['is_duplicate'] = scaled_prob
return sub
!pip install --no-warn-conflicts -q tensorflow-addons<import_modules>
sub = bin_model(tr, te)
 sub.to_csv('no_ml_model.csv', index=False, float_format='%.6f')
 sub.head(10 )