Kaggle competition: Quora Question Pairs
def Classifierx(shape_):
inp = Input(shape=(shape_))
x = Bidirectional(GRU(256, return_sequences=True))(inp)
 x = Attention(512 )(x)
 x = TimeDistributed(Dense(256, activation='mish'))(x)
 x = TimeDistributed(Dense(128, activation='mish'))(x)
 out = TimeDistributed(Dense(11, activation='softmax', name='out'))(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = Adam(lr=LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss=losses.CategoricalCrossentropy() , optimizer=opt, metrics=['accuracy'])
 return model<predict_on_test>
df_train = pd.read_csv('.. /input/train.csv')
 df_train.head()
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis=2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis=2 ).reshape(-1)
 score = f1_score(self.targets, pred, average="macro")
 print(f' F1Macro: {score:.5f}')
<normalization>
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal-train_input_mean)/train_input_sigma
test['signal'] =(test.signal-train_input_mean)/train_input_sigma
return train, test<feature_engineering>
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()
def run_everything(fe_config : List)-> NoReturn:
not_feats_cols = ['time']
target_col = ['open_channels']
init_logger()
with timer(f'Reading Data'):
logger.info('Reading Data Started...')
 base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train, test, sample_submission = read_data(base)
 train, test = normalize(train, test)
 logger.info('Reading and Normalizing Data Completed...')
 with timer(f'Creating Features'):
logger.info('Feature Enginnering Started...')
 for config in fe_config:
train = run_feat_enginnering(train, create_all_data_feats=config[0], batch_size=config[1])
 test = run_feat_enginnering(test, create_all_data_feats=config[0], batch_size=config[1])
 train, test, feats = feature_selection(train, test)
 logger.info('Feature Enginnering Completed...')
 with timer(f'Running Wavenet model'):
logger.info(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, splits=SPLITS, batch_col='group', feats=feats, sample_submission=sample_submission, nn_epochs=EPOCHS, nn_batch_size=NNBATCHSIZE)
 logger.info(f'Training completed...')
<set_options>
print('Total number of question pairs for testing: {}'.format(len(df_test)) )
myfavouritenumber = 0
seed = myfavouritenumber
random.seed(seed)
 warnings.filterwarnings('ignore' )<load_from_csv>
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))
%%time
df_train = pd.read_feather('.. /input/ashrae-feather/train.ft')
 df_test = pd.read_feather('.. /input/ashrae-feather/test.ft')
 building = pd.read_feather('.. /input/ashrae-feather/building.ft')
 le = LabelEncoder()
building.primary_use = le.fit_transform(building.primary_use)
 DATA_PATH = ".. /input/ashrae-energy-prediction/"
weather_train = pd.read_csv(DATA_PATH + 'weather_train.csv')
 weather_test = weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv' )<feature_engineering>
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
 plt.legend()
plt.title('Label distribution over word_match_share', fontsize=15)
 plt.xlabel('word_match_share', fontsize=15 )
%%time
df_train = df_train.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20 18")')
 df_train = df_train.query('not(building_id == 681 & meter == 0 & timestamp <= "2016-04-27")')
 df_train = df_train.query('not(building_id == 761 & meter == 0 & timestamp <= "2016-09-02")')
 df_train = df_train.query('not(building_id == 799 & meter == 0 & timestamp <= "2016-09-02")')
 df_train = df_train.query('not(building_id == 802 & meter == 0 & timestamp <= "2016-08-24")')
 df_train = df_train.query('not(building_id == 1073 & meter == 0 & timestamp <= "2016-10-26")')
 df_train = df_train.query('not(building_id == 1094 & meter == 0 & timestamp <= "2016-09-08")')
 df_train = df_train.query('not(building_id == 29 & meter == 0 & timestamp <= "2016-08-10")')
 df_train = df_train.query('not(building_id == 40 & meter == 0 & timestamp <= "2016-06-04")')
 df_train = df_train.query('not(building_id == 45 & meter == 0 & timestamp <= "2016-07")')
 df_train = df_train.query('not(building_id == 106 & meter == 0 & timestamp <= "2016-11")')
 df_train = df_train.query('not(building_id == 107 & meter == 0 & timestamp >= "2016-11-10")')
 df_train = df_train.query('not(building_id == 112 & meter == 0 & timestamp < "2016-10-31 15")')
 df_train = df_train.query('not(building_id == 144 & meter == 0 & timestamp > "2016-05-14" & timestamp < "2016-10-31")')
 df_train = df_train.query('not(building_id == 147 & meter == 0 & timestamp > "2016-06-05 19" & timestamp < "2016-07-18 15")')
 df_train = df_train.query('not(building_id == 171 & meter == 0 & timestamp <= "2016-07-05")')
 df_train = df_train.query('not(building_id == 177 & meter == 0 & timestamp > "2016-06-04" & timestamp < "2016-06-25")')
 df_train = df_train.query('not(building_id == 258 & meter == 0 & timestamp > "2016-09-26" & timestamp < "2016-12-12")')
 df_train = df_train.query('not(building_id == 258 & meter == 0 & timestamp > "2016-08-30" & timestamp < "2016-09-08")')
 df_train = df_train.query('not(building_id == 258 & meter == 0 & timestamp > "2016-09-18" & timestamp < "2016-09-25")')
 df_train = df_train.query('not(building_id == 260 & meter == 0 & timestamp <= "2016-05-11")')
 df_train = df_train.query('not(building_id == 269 & meter == 0 & timestamp > "2016-06-04" & timestamp < "2016-06-25")')
 df_train = df_train.query('not(building_id == 304 & meter == 0 & timestamp >= "2016-11-20")')
 df_train = df_train.query('not(building_id == 545 & meter == 0 & timestamp > "2016-01-17" & timestamp < "2016-02-10")')
 df_train = df_train.query('not(building_id == 604 & meter == 0 & timestamp < "2016-11-21")')
 df_train = df_train.query('not(building_id == 693 & meter == 0 & timestamp > "2016-09-07" & timestamp < "2016-11-23")')
 df_train = df_train.query('not(building_id == 693 & meter == 0 & timestamp > "2016-07-12" & timestamp < "2016-05-29")')
 df_train = df_train.query('not(building_id == 723 & meter == 0 & timestamp > "2016-10-06" & timestamp < "2016-11-22")')
 df_train = df_train.query('not(building_id == 733 & meter == 0 & timestamp > "2016-05-29" & timestamp < "2016-06-22")')
 df_train = df_train.query('not(building_id == 733 & meter == 0 & timestamp > "2016-05-19" & timestamp < "2016-05-20")')
 df_train = df_train.query('not(building_id == 803 & meter == 0 & timestamp > "2016-9-25")')
 df_train = df_train.query('not(building_id == 815 & meter == 0 & timestamp > "2016-05-17" & timestamp < "2016-11-17")')
 df_train = df_train.query('not(building_id == 848 & meter == 0 & timestamp > "2016-01-15" & timestamp < "2016-03-20")')
 df_train = df_train.query('not(building_id == 857 & meter == 0 & timestamp > "2016-04-13")')
 df_train = df_train.query('not(building_id == 909 & meter == 0 & timestamp < "2016-02-02")')
 df_train = df_train.query('not(building_id == 909 & meter == 0 & timestamp < "2016-06-23")')
 df_train = df_train.query('not(building_id == 1008 & meter == 0 & timestamp > "2016-10-30" & timestamp < "2016-11-21")')
 df_train = df_train.query('not(building_id == 1113 & meter == 0 & timestamp < "2016-07-27")')
 df_train = df_train.query('not(building_id == 1153 & meter == 0 & timestamp < "2016-01-20")')
 df_train = df_train.query('not(building_id == 1169 & meter == 0 & timestamp < "2016-08-03")')
 df_train = df_train.query('not(building_id == 1170 & meter == 0 & timestamp > "2016-06-30" & timestamp < "2016-07-05")')
 df_train = df_train.query('not(building_id == 1221 & meter == 0 & timestamp < "2016-11-04")')
 df_train = df_train.query('not(building_id == 1225 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1234 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 0 & timestamp > "2016-01-13 22" & timestamp < "2016-03-08 12")')
 df_train = df_train.query('not(building_id == 1241 & meter == 0 & timestamp > "2016-07-14" & timestamp < "2016-11-19")')
 df_train = df_train.query('not(building_id == 1250 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1255 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1264 & meter == 0 & timestamp > "2016-08-23")')
 df_train = df_train.query('not(building_id == 1265 & meter == 0 & timestamp > "2016-05-06" & timestamp < "2016-05-26")')
 df_train = df_train.query('not(building_id == 1272 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1275 & building_id <= 1280 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1283 & meter == 0 & timestamp > "2016-07-08" & timestamp < "2016-08-03")')
 df_train = df_train.query('not(building_id >= 1291 & building_id <= 1302 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1303 & meter == 0 & timestamp > "2016-07-25 22" & timestamp < "2016-07-27 16")')
 df_train = df_train.query('not(building_id == 1303 & meter == 0 & timestamp > "2016-01-26" & timestamp < "2016-06-02 12")')
 df_train = df_train.query('not(building_id == 1319 & meter == 0 & timestamp > "2016-05-17 16" & timestamp < "2016-06-07 12")')
 df_train = df_train.query('not(building_id == 1319 & meter == 0 & timestamp > "2016-08-18 14" & timestamp < "2016-09-02 14")')
 df_train = df_train.query('not(building_id == 1322 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-10-14 22" & timestamp < "2016-10-17 08")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-07-01 14" & timestamp < "2016-07-05 06")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-10-14 22" & timestamp < "2016-10-17 08")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-07-01 14" & timestamp < "2016-07-05 06")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-10-14 22" & timestamp < "2016-10-17 08")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-07-01 14" & timestamp < "2016-07-05 06")')
 df_train = df_train.query('not(building_id == 1272 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1291 & building_id <= 1297 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1300 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1302 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1291 & building_id <= 1299 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1221 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1225 & building_id <= 1226 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1241 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1223 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1226 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1225 & building_id <= 1226 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1305 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1307 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1223 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1231 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1272 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1275 & building_id <= 1297 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1300 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1302 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1293 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-25 12")')
 df_train = df_train.query('not(building_id == 1302 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-25 12")')
 df_train = df_train.query('not(building_id == 1223 & meter == 0 & timestamp > "2016-9-28 07" & timestamp < "2016-10-11 18")')
 df_train = df_train.query('not(building_id == 1225 & meter == 1 & timestamp > "2016-8-22 23" & timestamp < "2016-10-11 14")')
 df_train = df_train.query('not(building_id == 1230 & meter == 1 & timestamp > "2016-8-22 08" & timestamp < "2016-10-05 18")')
 df_train = df_train.query('not(building_id == 904 & meter == 0 & timestamp < "2016-02-17 08")')
 df_train = df_train.query('not(building_id == 986 & meter == 0 & timestamp < "2016-02-17 08")')
 df_train = df_train.query('not(building_id == 954 & meter == 0 & timestamp < "2016-08-08 11")')
 df_train = df_train.query('not(building_id == 954 & meter == 0 & timestamp < "2016-06-23 08")')
 df_train = df_train.query('not(building_id >= 745 & building_id <= 770 & meter == 1 & timestamp > "2016-10-05 01" & timestamp < "2016-10-10 09")')
 df_train = df_train.query('not(building_id >= 774 & building_id <= 787 & meter == 1 & timestamp > "2016-10-05 01" & timestamp < "2016-10-10 09")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-05-11 09" & timestamp < "2016-05-12 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-05-11 09" & timestamp < "2016-05-12 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-05-11 09" & timestamp < "2016-05-12 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp == "2016-02-26 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp == "2016-02-26 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp == "2016-02-26 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-03-29 10" & timestamp < "2016-03-30 12")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-03-29 10" & timestamp < "2016-03-30 12")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-03-29 10" & timestamp < "2016-03-30 12")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-01-19 23" & timestamp < "2016-01-28 15")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-01-19 23" & timestamp < "2016-01-28 15")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-01-19 23" & timestamp < "2016-01-28 15")')
 df_train = df_train.query('not(building_id != 1227 & building_id != 1281 & building_id != 1314 & building_id >=1223 & building_id < 1335 & meter==0 & meter_reading==0)')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & meter==1 & timestamp > "2016-07-16 04" & timestamp < "2016-07-19 11")')
 df_train = df_train.query('not(building_id == 107 & meter == 0 & timestamp <= "2016-07-06")')
 df_train = df_train.query('not(building_id == 180 & timestamp >= "2016-02-17 12")')
 df_train = df_train.query('not(building_id == 182 & meter == 0)')
 df_train = df_train.query('not(building_id == 191 & meter == 0 & timestamp >= "2016-12-22 09")')
 df_train = df_train.query('not(building_id == 192 & meter == 1 & timestamp >= "2016-05-09 18")')
 df_train = df_train.query('not(building_id == 192 & meter == 3 & timestamp >= "2016-03-29 05" & timestamp <= "2016-04-04 08")')
 df_train = df_train.query('not(building_id == 207 & meter == 1 & timestamp > "2016-07-02 20" & timestamp < "2016-08-25 12")')
 df_train = df_train.query('not(building_id == 258 & timestamp > "2016-09-18" & timestamp < "2016-12-12 13")')
 df_train = df_train.query('not(building_id == 258 & timestamp > "2016-08-29 08" & timestamp < "2016-09-08 14")')
 df_train = df_train.query('not(building_id == 257 & meter == 1 & timestamp < "2016-03-25 16")')
 df_train = df_train.query('not(building_id == 260 & meter == 1 & timestamp > "2016-05-10 17" & timestamp < "2016-08-17 11")')
 df_train = df_train.query('not(building_id == 260 & meter == 1 & timestamp > "2016-08-28 01" & timestamp < "2016-10-31 13")')
 df_train = df_train.query('not(building_id == 220 & meter == 1 & timestamp > "2016-09-23 01" & timestamp < "2016-09-23 12")')
 df_train = df_train.query('not(building_id == 281 & meter == 1 & timestamp > "2016-10-25 08" & timestamp < "2016-11-04 15")')
 df_train = df_train.query('not(building_id == 273 & meter == 1 & timestamp > "2016-04-03 04" & timestamp < "2016-04-29 15")')
 df_train = df_train.query('not(building_id == 28 & meter == 0 & timestamp < "2016-10-14 20")')
 df_train = df_train.query('not(building_id == 71 & meter == 0 & timestamp < "2016-08-18 20")')
 df_train = df_train.query('not(building_id == 76 & meter == 0 & timestamp > "2016-06-04 09" & timestamp < "2016-06-04 14")')
 df_train = df_train.query('not(building_id == 101 & meter == 0 & timestamp > "2016-10-12 13" & timestamp < "2016-10-12 18")')
 df_train = df_train.query('not(building_id == 7 & meter == 1 & timestamp > "2016-11-03 09" & timestamp < "2016-11-28 14")')
 df_train = df_train.query('not(building_id == 9 & meter == 1 & timestamp > "2016-12-06 08")')
 df_train = df_train.query('not(building_id == 43 & meter == 1 & timestamp > "2016-04-03 08" & timestamp < "2016-06-06 13")')
 df_train = df_train.query('not(building_id == 60 & meter == 1 & timestamp > "2016-05-01 17" & timestamp < "2016-05-01 21")')
 df_train = df_train.query('not(building_id == 75 & meter == 1 & timestamp > "2016-08-05 13" & timestamp < "2016-08-26 12")')
 df_train = df_train.query('not(building_id == 95 & meter == 1 & timestamp > "2016-08-08 10" & timestamp < "2016-08-26 13")')
 df_train = df_train.query('not(building_id == 97 & meter == 1 & timestamp > "2016-08-08 14" & timestamp < "2016-08-25 14")')
 df_train = df_train.query('not(building_id == 1232 & meter == 1 & timestamp > "2016-06-23 16" & timestamp < "2016-08-31 20")')
 df_train = df_train.query('not(building_id == 1236 & meter == 1 & meter_reading >= 3000)')
 df_train = df_train.query('not(building_id == 1239 & meter == 1 & timestamp > "2016-03-11 16" & timestamp < "2016-03-27 17")')
 df_train = df_train.query('not(building_id == 1264 & meter == 1 & timestamp > "2016-08-22 17" & timestamp < "2016-09-22 20")')
 df_train = df_train.query('not(building_id == 1264 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1269 & meter == 1 & meter_reading >= 2000)')
 df_train = df_train.query('not(building_id == 1272 & meter == 1 & timestamp > "2016-08-11 12" & timestamp < "2016-08-30 19")')
 df_train = df_train.query('not(building_id == 1273 & meter == 1 & timestamp > "2016-05-31 14" & timestamp < "2016-06-17")')
 df_train = df_train.query('not(building_id == 1276 & meter == 1 & timestamp < "2016-02-03 23")')
 df_train = df_train.query('not(building_id == 1280 & meter == 1 & timestamp > "2016-05-18" & timestamp < "2016-05-26 09")')
 df_train = df_train.query('not(building_id == 1280 & meter == 1 & timestamp > "2016-02-28 23" & timestamp < "2016-05-02 05")')
 df_train = df_train.query('not(building_id == 1280 & meter == 1 & timestamp > "2016-06-12 01" & timestamp < "2016-7-07 06")')
 df_train = df_train.query('not(building_id == 1288 & meter == 1 & timestamp > "2016-07-07 15" & timestamp < "2016-08-12 17")')
 df_train = df_train.query('not(building_id == 1311 & meter == 1 & timestamp > "2016-04-25 18" & timestamp < "2016-05-13 14")')
 df_train = df_train.query('not(building_id == 1099 & meter == 2)')
 df_train = df_train.query('not(building_id == 1329 & meter == 0 & timestamp > "2016-04-28 00" & timestamp < "2016-04-28 07")')
 df_train = df_train.query('not(building_id == 1331 & meter == 0 & timestamp > "2016-04-28 00" & timestamp < "2016-04-28 07")')
 df_train = df_train.query('not(building_id == 1427 & meter == 0 & timestamp > "2016-04-11 10" & timestamp < "2016-04-11 14")')
 df_train = df_train.query('not(building_id == 1426 & meter == 2 & timestamp > "2016-05-03 09" & timestamp < "2016-05-03 14")')
 df_train = df_train.query('not(building_id == 1345 & meter == 0 & timestamp < "2016-03-01")')
 df_train = df_train.query('not(building_id == 1346 & timestamp < "2016-03-01")')
 df_train = df_train.query('not(building_id == 1359 & meter == 0 & timestamp > "2016-04-25 17" & timestamp < "2016-07-22 14")')
 df_train = df_train.query('not(building_id == 1365 & meter == 0 & timestamp > "2016-08-19 00" & timestamp < "2016-08-19 07")')
 df_train = df_train.query('not(building_id == 1365 & meter == 0 & timestamp > "2016-06-18 22" & timestamp < "2016-06-19 06")')
 df_train = df_train.query('not(building_id == 18 & meter == 0 & timestamp > "2016-06-04 09" & timestamp < "2016-06-04 16")')
 df_train = df_train.query('not(building_id == 18 & meter == 0 & timestamp > "2016-11-05 05" & timestamp < "2016-11-05 15")')
 df_train = df_train.query('not(building_id == 101 & meter == 0 & meter_reading > 800)')
 df_train = df_train.query('not(building_id == 1384 & meter == 0 & meter_reading == 0)')
 df_train = df_train.query('not(building_id >= 1289 & building_id <= 1301 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1243 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1263 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1284 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1286 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1263 & meter == 0 & timestamp > "2016-11-10 11" & timestamp < "2016-11-10 15")')
 df_train = df_train.query('not(building_id == 1238 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1329 & meter == 2 & timestamp > "2016-11-21 12" & timestamp < "2016-11-29 12")')
 df_train = df_train.query('not(building_id == 1249 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1250 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-03-05 18" & timestamp < "2016-03-05 22")')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-03-27 00" & timestamp < "2016-03-27 23")')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-04-11 09" & timestamp < "2016-04-13 03")')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-04-29 00" & timestamp < "2016-04-30 15")')
 df_train = df_train.query('not(building_id == 1303 & meter == 2 & timestamp < "2016-06-06 19")')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & meter == 1 & timestamp > "2016-08-11 17" & timestamp < "2016-08-12 17")')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & building_id != 1296 & building_id != 129 & building_id != 1298 & building_id != 1299 & meter == 2 & timestamp > "2016-08-11 17" & timestamp < "2016-08-12 17")')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & meter == 3 & timestamp > "2016-08-11 17" & timestamp < "2016-08-12 17")' )<feature_engineering>
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
bm_cols = ['bm', 'weekday', 'hour',]
df_train['hour'] = df_train['timestamp'].dt.hour
df_train['weekday'] = df_train['timestamp'].dt.weekday
df_train['bm'] = df_train['building_id'].apply(lambda x: str(x)) + '_' + df_train['meter'].apply(lambda x: str(x))
bm = df_train.groupby(bm_cols)['meter_reading'].mean().rename('bm_week_hour' ).to_frame()<merge>
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )
df_train = df_train.merge(bm, right_index=True, left_on=bm_cols, how='left' )<drop_column>
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R
df_train.drop(['bm'], axis=1, inplace=True)
 df_train.head()<remove_duplicates>
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )
weather = pd.concat([weather_train, weather_test],ignore_index=True)
 weather_key = ['site_id', 'timestamp']
full_weather = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).sort_values(by=weather_key ).copy()<data_type_conversions>
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values
full_weather['timestamp'] = pd.to_datetime(full_weather['timestamp'] )<feature_engineering>
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train
full_weather['temp_rank'] = full_weather.groupby(['site_id', full_weather.timestamp.dt.date])['air_temperature'].rank('average')
 df_2d = full_weather.groupby(['site_id', full_weather.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
def timestamp_align(df):
df['offset'] = df.site_id.map(site_ids_offsets)
 df['timestamp_aligned'] =(df.timestamp - pd.to_timedelta(df.offset, unit='H'))
df['timestamp'] = df['timestamp_aligned']
del df['timestamp_aligned']
return df<set_options>
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df<data_type_conversions>
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 4
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10 )
<create_dataframe><EOS> 
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )