Kaggle competition: Santander Value Prediction Challenge
test_lengths = torch.from_numpy(np.array([len(x)for x in x_test]))
x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))<create_dataframe>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 transact_cols = [f for f in train.columns if f not in ["ID", "target"]]
y = np.log1p(train["target"] ).values
batch_size = 512
test_dataset = data.TensorDataset(x_test_padded, test_lengths)
 train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)
 valid_dataset = data.Subset(train_dataset, indices=[0, 1])
 train_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() ,
sequence_index=0,
length_index=1,
label_index=2)
 test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max() , sequence_index=0, length_index=1)
 train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_collator)
 valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)
 test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)
 databunch = DataBunch(train_dl=train_loader, valid_dl=valid_loader, collate_fn=train_collator )<compute_test_metric>
gc.collect() ;
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1', '15ace8c9f',
'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9', 'd6bb78916', 'b43a7cfd5',
'58232a6fb', '1702b5bf0', '324921c7b', '62e59a501', '2ec5b290f', '241f0f867',
'fb49e4212', '66ace2992', 'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7',
'1931ccfdd', '703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535', 'fc99f9426', '91f701ba2', '0572565c2', '190db8488',
'adb64ff71', 'c47340d97', 'c5a231d81'
]
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2<choose_model_class>
def get_beautiful_test(test):
test_rnd = np.round(test.iloc[:, 1:], 2)
 ugly_indexes = []
non_ugly_indexes = []
for idx in tqdm(range(len(test))):
if not np.all(
 test_rnd.iloc[idx, :].values==test.iloc[idx, 1:].values 
):
ugly_indexes.append(idx)
 else:
non_ugly_indexes.append(idx)
 print(len(ugly_indexes), len(non_ugly_indexes))
np.save('test_ugly_indexes', np.array(ugly_indexes))
np.save('test_non_ugly_indexes', np.array(non_ugly_indexes))
test = test.iloc[non_ugly_indexes].reset_index(drop=True)
 return test, non_ugly_indexes, ugly_indexes
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch, model, loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<save_to_csv>
test, non_ugly_indexes, ugly_indexes = get_beautiful_test(test)
 test["target"] = train["target"].mean()
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<define_variables>
CPU_CORES = 1
def _get_leak(df, cols, lag=0, verbose=False):

 series_str = df[cols[lag+2:]].apply(lambda x: "_".join(x.round(2 ).astype(str)) , axis=1)
 series_shifted_str = df[cols].shift(lag+2, axis=1)[cols[lag+2:]].apply(lambda x: "_".join(x.round(2 ).astype(str)) , axis=1)
 if verbose:
target_rows = series_shifted_str.progress_apply(lambda x: np.where(x == series_str)[0])
 else:
target_rows = series_shifted_str.apply(lambda x: np.where(x == series_str)[0])
 target_vals = target_rows.apply(lambda x: df.loc[x[0], cols[lag]] if len(x)==1 else 0)
 return target_vals
def get_all_leak(df, cols=None, nlags=15):

 df = df.copy()
for i in range(nlags):
if "leaked_target_"+str(i)not in df.columns:
print("Processing lag {}".format(i))
df["leaked_target_"+str(i)] = _get_leak(df, cols, i)
 return df
def compiled_leak_result() :
max_nlags = len(cols)- 2
train_leak = train[["ID", "target"] + cols]
train_leak["compiled_leak"] = 0
train_leak["nonzero_mean"] = train[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_value_corrects = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 train_leak[c] = _get_leak(train_leak, cols, i)
 leaky_cols.append(c)
 train_leak = train.join(
 train_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)
 zeroleak = train_leak["compiled_leak"]==0
train_leak.loc[zeroleak, "compiled_leak"] = train_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(train_leak["compiled_leak"] > 0))
_correct_counts = sum(train_leak["compiled_leak"]==train_leak["target"])
 leaky_value_corrects.append(_correct_counts/leaky_value_counts[-1])
 print("Leak values found in train", leaky_value_counts[-1])
 print(
 "% of correct leaks values in train ",
leaky_value_corrects[-1] 
)
 tmp = train_leak.copy()
tmp.loc[zeroleak, "compiled_leak"] = tmp.loc[zeroleak, "nonzero_mean"]
scores.append(np.sqrt(mean_squared_error(y, np.log1p(tmp["compiled_leak"] ).fillna(14.49))))
print(
 'Score(filled with nonzero mean)',
scores[-1] 
)
 result = dict(
 score=scores,
leaky_count=leaky_value_counts,
leaky_correct=leaky_value_corrects, 
)
 return train_leak, result
EMBEDDING_FILES = ['.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt']
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
IDENTITY_COLUMNS = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<categorify>
result = pd.DataFrame.from_dict(result, orient='columns')
 result
result.to_csv('train_leaky_stat.csv', index=False)
 train_leak.head()
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<choose_model_class>
best_score = np.min(result['score'])
 best_lag = np.argmin(result['score'])
 print('best_score', best_score, '
best_lag', best_lag )
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<prepare_x_and_y>
def rewrite_compiled_leak(leak_df, lag):
leak_df["compiled_leak"] = 0
for i in range(lag):
c = "leaked_target_"+str(i)
 zeroleak = leak_df["compiled_leak"]==0
leak_df.loc[zeroleak, "compiled_leak"] = leak_df.loc[zeroleak, c]
return leak_df
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str)
 for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False )<string_transform>
leaky_cols = [c for c in train_leak.columns if 'leaked_target_' in c]
train_leak = rewrite_compiled_leak(train_leak, best_lag)
 train_leak[['ID']+leaky_cols+['compiled_leak']].head()
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))<string_transform>
train_res = train_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 train_res.to_csv('train_leak.csv', index=False )
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )<concatenate>
def compiled_leak_result_test(max_nlags):
test_leak = test[["ID", "target"] + cols]
test_leak["compiled_leak"] = 0
test_leak["nonzero_mean"] = test[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 test_leak[c] = _get_leak(test_leak, cols, i, verbose=True)
 leaky_cols.append(c)
 test_leak = test.join(
 test_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)[["ID", "target"] + cols + leaky_cols+["compiled_leak", "nonzero_mean"]]
zeroleak = test_leak["compiled_leak"]==0
test_leak.loc[zeroleak, "compiled_leak"] = test_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(test_leak["compiled_leak"] > 0))
print("Leak values found in test", leaky_value_counts[-1])
 result = dict(
 leaky_count=leaky_value_counts, 
)
 return test_leak, result
x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
test_leak, test_result = compiled_leak_result_test(max_nlags=38 )
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])* train_df[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()<concatenate>
test_result = pd.DataFrame.from_dict(test_result, orient='columns')
 test_result
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )<train_model>
test_result.to_csv('test_leaky_stat.csv', index=False )
checkpoint_predictions = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch )<prepare_output>
test_leak = rewrite_compiled_leak(test_leak, best_lag)
 test_leak[['ID']+leaky_cols+["compiled_leak", "nonzero_mean"]].head()
predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )<create_dataframe>
NLAGS = 15
all_df = get_all_leak(all_df, cols=cols, nlags=NLAGS )
submission = pd.DataFrame.from_dict({'id': test_df.id,
'prediction': predictions} )<save_to_csv>
leaky_cols = ["leaked_target_"+str(i)for i in range(NLAGS)]
train = train.join(all_df.set_index("ID")[leaky_cols], on="ID", how="left")
 test = test.join(all_df.set_index("ID")[leaky_cols], on="ID", how="left" )
submission.to_csv('submission.csv', index=False )<save_to_csv>
train["nonzero_mean"] = train[transact_cols].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1)
 test["nonzero_mean"] = test[transact_cols].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 )
submission.to_csv('submission.csv', index=False )<set_options>
train["compiled_leak"] = 0
test["compiled_leak"] = 0
for i in range(NLAGS):
train.loc[train["compiled_leak"] == 0, "compiled_leak"] = train.loc[train["compiled_leak"] == 0, "leaked_target_"+str(i)]
test.loc[test["compiled_leak"] == 0, "compiled_leak"] = test.loc[test["compiled_leak"] == 0, "leaked_target_"+str(i)]
print("Leak values found in train and test ", sum(train["compiled_leak"] > 0), sum(test["compiled_leak"] > 0))
print("% of correct leaks values in train ", sum(train["compiled_leak"] == train["target"])/sum(train["compiled_leak"] > 0))
train.loc[train["compiled_leak"] == 0, "compiled_leak"] = train.loc[train["compiled_leak"] == 0, "nonzero_mean"]
test.loc[test["compiled_leak"] == 0, "compiled_leak"] = test.loc[test["compiled_leak"] == 0, "nonzero_mean"]
tqdm.pandas()<randomize_order>
np.sqrt(mean_squared_error(y, np.log1p(train["compiled_leak"] ).fillna(14.49)) )
<load_from_csv><EOS> 
sub = test[["ID"]]
sub["target"] = test["compiled_leak"]
sub.to_csv("baseline_submission_with_leaks.csv", index=False )