Kaggle competition: Avito Demand Prediction Challenge
BERT_MODEL = 'bert-base-uncased'
CASED = 'uncased' in BERT_MODEL
INPUT = '.. /input/jigsaw-bert-preprocessed-input/'
TEXT_COL = 'comment_text'
MAXLEN = 250<install_modules>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_y_train = df_train['deal_probability']
df_x_train = df_train.drop(['deal_probability'], axis=1 )
os.system('pip install --no-index --find-links=".. /input/pytorchpretrainedbert/" pytorch_pretrained_bert' )<import_modules>
for col in df_x_train.columns:
print(col, len(df_x_train[col].unique()))
from pytorch_pretrained_bert import BertTokenizer
from pytorch_pretrained_bert.modeling import BertModel 
<define_variables>
df_x_train['image_top_1'].fillna(value=3067, inplace=True)
 df_test['image_top_1'].fillna(value=3067, inplace=True )
BERT_FP = '.. /input/torch-bert-weights/bert-base-uncased/bert-base-uncased/'<load_pretrained>
df_x_train['param_1'].fillna(value='_NA_', inplace=True)
 df_test['param_1'].fillna(value='_NA_', inplace=True)
 df_x_train['param_2'].fillna(value='_NA_', inplace=True)
 df_test['param_2'].fillna(value='_NA_', inplace=True)
 df_x_train['param_3'].fillna(value='_NA_', inplace=True)
 df_test['param_3'].fillna(value='_NA_', inplace=True )
def get_bert_embed_matrix() :
bert = BertModel.from_pretrained(BERT_FP)
 bert_embeddings = list(bert.children())[0]
bert_word_embeddings = list(bert_embeddings.children())[0]
mat = bert_word_embeddings.weight.data.numpy()
return mat<feature_engineering>
df_x_train['description'].fillna(value='_NA_', inplace=True)
 df_test['description'].fillna(value='_NA_', inplace=True )
embedding_matrix = get_bert_embed_matrix()<load_from_csv>
config = argparse.Namespace()
train = pd.read_csv(INPUT + 'train_bert-base-uncased_ids.csv' ).sample(frac = 1.0, random_state = 23 )<load_from_csv>
def tknzr_fit(col, df_trn, df_test):
tknzr = Tokenizer(filters='', lower=False, split='뷁', oov_token='oov')
 tknzr.fit_on_texts(df_trn[col])
 return np.array(tknzr.texts_to_sequences(df_trn[col])) , np.array(tknzr.texts_to_sequences(df_test[col])) , tknzr
test = pd.read_csv(INPUT + 'test_bert-base-uncased_ids.csv' )<prepare_x_and_y>
tr_reg, te_reg, tknzr_reg = tknzr_fit('region', df_x_train, df_test)
 tr_pcn, te_pcn, tknzr_pcn = tknzr_fit('parent_category_name', df_x_train, df_test)
 tr_cn, te_cn, tknzr_cn = tknzr_fit('category_name', df_x_train, df_test)
 tr_ut, te_ut, tknzr_ut = tknzr_fit('user_type', df_x_train, df_test)
 tr_city, te_city, tknzr_city = tknzr_fit('city', df_x_train, df_test)
 tr_p1, te_p1, tknzr_p1 = tknzr_fit('param_1', df_x_train, df_test)
 tr_p2, te_p2, tknzr_p2 = tknzr_fit('param_2', df_x_train, df_test)
 tr_p3, te_p3, tknzr_p3 = tknzr_fit('param_3', df_x_train, df_test )
x_train = np.zeros(( train.shape[0],MAXLEN),dtype=np.int)
 for i,ids in tqdm(enumerate(list(train[TEXT_COL]))):
input_ids = [int(i)for i in ids.split() [:MAXLEN]]
inp_len = len(input_ids)
 x_train[i,:inp_len] = np.array(input_ids)
 x_test = np.zeros(( test.shape[0],MAXLEN),dtype=np.int)
 for i,ids in tqdm(enumerate(list(test[TEXT_COL]))):
input_ids = [int(i)for i in ids.split() [:MAXLEN]]
inp_len = len(input_ids)
 x_test[i,:inp_len] = np.array(input_ids)
 with open('temporary.pickle', mode='wb')as f:
pickle.dump(x_test, f)
 del x_test
gc.collect()<data_type_conversions>
tr_week = pd.to_datetime(df_x_train['activation_date'] ).dt.weekday.astype(np.int32 ).values
te_week = pd.to_datetime(df_test['activation_date'] ).dt.weekday.astype(np.int32 ).values
tr_week = np.expand_dims(tr_week, axis=-1)
 te_week = np.expand_dims(te_week, axis=-1 )
identity_columns = ['male','female','homosexual_gay_or_lesbian','christian','jewish','muslim','black','white','psychiatric_or_mental_illness']
y_identities =(train[identity_columns] >= 0.5 ).astype(int ).values
weights = np.ones(( len(train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']].values 
<import_modules>
tr_imgt1 = df_x_train['image_top_1'].astype(np.int32 ).values
te_imgt1 = df_test['image_top_1'].astype(np.int32 ).values
tr_imgt1 = np.expand_dims(tr_imgt1, axis=-1)
 te_imgt1 = np.expand_dims(te_imgt1, axis=-1 )
from keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.losses import binary_crossentropy
from keras import backend as K
import keras.layers as L
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.models import Model
from keras.optimizers import Adam
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import KFold<choose_model_class>
eps = 1e-10
tr_price = np.log(df_x_train['price']+eps)
 te_price = np.log(df_test['price']+eps)
 tr_price[tr_price.isna() ] = -1.
 te_price[te_price.isna() ] = -1.
 tr_price = np.expand_dims(tr_price, axis=-1)
 te_price = np.expand_dims(te_price, axis=-1 )
def build_model(embedding_matrix, num_aux_targets, loss_weight):

 words = Input(shape=(MAXLEN,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.3 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([GlobalMaxPooling1D()(x),GlobalAveragePooling1D()(x),])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss=[custom_loss,'binary_crossentropy'], loss_weights=[loss_weight, 1.0], optimizer='adam')
 return model<compute_test_metric>
tr_itemseq = np.log(df_x_train['item_seq_number'])
 te_itemseq = np.log(df_test['item_seq_number'])
 tr_itemseq = np.expand_dims(tr_itemseq, axis=-1)
 te_itemseq = np.expand_dims(te_itemseq, axis=-1 )
def custom_loss(y_true, y_pred):
return binary_crossentropy(K.reshape(y_true[:,0],(-1,1)) , y_pred)* y_true[:,1] 
<split>
config.len_desc = 100000
tr_ind, val_ind = train_test_split(list(range(len(x_train))),test_size = 0.05, random_state = 23 )<train_model>
from keras.preprocessing.sequence import pad_sequences
NUM_MODELS = 1
BATCH_SIZE = 512
EPOCHS = 5
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 512
checkpoint_predictions = []
checkpoint_val_preds = []
weights = []
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1],loss_weight)
 for global_epoch in range(EPOCHS):
model.fit(x_train[tr_ind],[y_train[tr_ind], y_aux_train[tr_ind]],validation_data =(x_train[val_ind],[y_train[val_ind], y_aux_train[val_ind]]),
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
callbacks=[
LearningRateScheduler(lambda epoch: 1e-3 *(0.6 ** global_epoch))
] 
)
 with open('temporary.pickle', mode='rb')as f:
x_test = pickle.load(f)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 checkpoint_val_preds.append(model.predict(x_train[val_ind], batch_size=2048)[0].flatten())
 del x_test
gc.collect()
weights.append(2 ** global_epoch)
 del model
gc.collect()
predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )<prepare_output>
tknzr_desc = Tokenizer(num_words=config.len_desc, lower='True' )
val_preds = np.average(checkpoint_val_preds, weights=weights, axis=0 )<compute_test_metric>
tknzr_desc.fit_on_texts(df_x_train['description'].values )
def power_mean(x, p=-5):
return np.power(np.mean(np.power(x, p)) ,1/p)
 def get_s_auc(y_true,y_pred,y_identity):
mask = y_identity==1
try:
s_auc = roc_auc_score(y_true[mask],y_pred[mask])
 except:
s_auc = 1
return s_auc
def get_bpsn_auc(y_true,y_pred,y_identity):
mask =(y_identity==1)&(y_true==0)|(y_identity==0)&(y_true==1)
 try:
bpsn_auc = roc_auc_score(y_true[mask],y_pred[mask])
 except:
bpsn_auc = 1
return bpsn_auc
def get_bspn_auc(y_true,y_pred,y_identity):
mask =(y_identity==1)&(y_true==1)|(y_identity==0)&(y_true==0)
 try:
bspn_auc = roc_auc_score(y_true[mask],y_pred[mask])
 except:
bspn_auc = 1
return bspn_auc
def get_total_auc(y_true,y_pred,y_identities):
N = y_identities.shape[1]
saucs = np.array([get_s_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 bpsns = np.array([get_bpsn_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 bspns = np.array([get_bspn_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 M_s_auc = power_mean(saucs)
 M_bpsns_auc = power_mean(bpsns)
 M_bspns_auc = power_mean(bspns)
 rauc = roc_auc_score(y_true,y_pred)
 total_auc = M_s_auc + M_bpsns_auc + M_bspns_auc + rauc
total_auc/= 4
return total_auc<compute_test_metric>
tr_desc_seq = tknzr_desc.texts_to_sequences(df_x_train['description'].values)
 te_desc_seq = tknzr_desc.texts_to_sequences(df_test['description'].values )
get_total_auc(y_train[val_ind][:,0],val_preds,y_identities[val_ind] )<save_to_csv>
config.maxlen= 75
df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = predictions
df_submit.to_csv('submission.csv', index=False )<import_modules>
tr_desc_pad = pad_sequences(tr_desc_seq, maxlen=config.maxlen)
 te_desc_pad = pad_sequences(te_desc_seq, maxlen=config.maxlen )
tqdm.pandas()<load_from_csv>
gc.collect()
TEXT_COL = 'comment_text'
EMB_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id' )<categorify>
config.len_reg = len(tknzr_reg.word_index)
 config.len_pcn = len(tknzr_pcn.word_index)
 config.len_cn = len(tknzr_cn.word_index)
 config.len_ut = len(tknzr_ut.word_index)
 config.len_city = len(tknzr_city.word_index)+1
config.len_week = 7
config.len_imgt1 = int(df_x_train['image_top_1'].max())+1
config.len_p1 = len(tknzr_p1.word_index)+1
config.len_p2 = len(tknzr_p2.word_index)+1
config.len_p3 = len(tknzr_p3.word_index)+1
config.len_price = 1
config.len_itemseq = 1
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
def convert_to_bool(df, col_name):
df[col_name] = np.where(df[col_name] >= 0.5, True, False)
 def convert_dataframe_to_bool(df):
bool_df = df.copy()
for col in ['target'] + identity_columns:
convert_to_bool(bool_df, col)
 return bool_df
train = convert_dataframe_to_bool(train )<categorify>
config.emb_reg = 8
config.emb_pcn = 4
config.emb_cn = 8
config.emb_ut = 2
config.emb_city = 16
config.emb_week = 4
config.emb_imgt1 = 16
config.emb_p1 = 8
config.emb_p2 = 16
config.emb_p3 = 16
config.emb_price = 16
config.emb_itemseq = 16
config.emb_desc = 100
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def load_embeddings(embed_dir=EMB_PATH):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in tqdm(open(embed_dir)))
 return embedding_index
def build_embedding_matrix(word_index, embeddings_index, max_features, lower = True, verbose = True):
embedding_matrix = np.zeros(( max_features, 300))
for word, i in tqdm(word_index.items() ,disable = not verbose):
if lower:
word = word.lower()
if i >= max_features: continue
try:
embedding_vector = embeddings_index[word]
except:
embedding_vector = embeddings_index["unknown"]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_matrix(word_index, embeddings_index):
embedding_matrix = np.zeros(( len(word_index)+ 1,300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embeddings_index[word]
except:
embedding_matrix[i] = embeddings_index["unknown"]
return embedding_matrix<prepare_x_and_y>
X = np.array([tr_reg, tr_pcn, tr_cn, tr_ut, tr_city, tr_week, tr_imgt1, tr_p1, tr_p2, tr_p3,
tr_price, tr_itemseq])
 X_test = np.array([te_reg, te_pcn, te_cn, te_ut, te_city, te_week, te_imgt1, te_p1, te_p2, te_p3,
te_price, te_itemseq])
 Y = df_y_train
maxlen = 220
max_features = 100000
embed_size = 300
tokenizer = Tokenizer(num_words=max_features, lower=True)
 print('fitting tokenizer')
 tokenizer.fit_on_texts(list(train[TEXT_COL])+ list(test[TEXT_COL]))
word_index = tokenizer.word_index
X_train = tokenizer.texts_to_sequences(list(train[TEXT_COL]))
y_train = train['target'].values
X_test = tokenizer.texts_to_sequences(list(test[TEXT_COL]))
X_train = pad_sequences(X_train, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 del tokenizer
gc.collect() 
<load_pretrained>
X_train = [x[train_idx] for x in X]
X_valid = [x[valid_idx] for x in X]
X_test = [x for x in X_test]
Y_train = Y[train_idx]
Y_valid = Y[valid_idx]
embeddings_index = load_embeddings()<prepare_x_and_y>
X_train.append(tr_desc_pad[train_idx])
 X_valid.append(tr_desc_pad[valid_idx])
 X_test.append(te_desc_pad )
embedding_matrix = build_matrix(word_index, embeddings_index )<set_options>
gc.collect()
del embeddings_index
gc.collect()<import_modules>
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)) )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
config.batch_size = 4096
def build_model(verbose = False, compile = True):
sequence_input = L.Input(shape=(maxlen,), dtype='int32')
 embedding_layer = L.Embedding(len(word_index)+ 1,
300,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)
 x = embedding_layer(sequence_input)
 x = L.SpatialDropout1D(0.2 )(x)
 x = L.Bidirectional(L.CuDNNLSTM(64, return_sequences=True))(x)
 att = Attention(maxlen )(x)
 avg_pool1 = L.GlobalAveragePooling1D()(x)
 max_pool1 = L.GlobalMaxPooling1D()(x)
 x = L.concatenate([att,avg_pool1, max_pool1])
 preds = L.Dense(1, activation='sigmoid' )(x)
 model = Model(sequence_input, preds)
 if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
 return model 
<compute_test_metric>
def get_model() :
K.clear_session()
inp_reg = Input(shape=(1,), name='inp_region')
 emb_reg = Embedding(config.len_reg, config.emb_reg, name='emb_region' )(inp_reg)
 inp_pcn = Input(shape=(1,), name='inp_parent_category_name')
 emb_pcn = Embedding(config.len_pcn, config.emb_pcn, name='emb_parent_category_name' )(inp_pcn)
 inp_cn = Input(shape=(1,), name='inp_category_name')
 emb_cn = Embedding(config.len_cn, config.emb_cn, name="emb_category_name" )(inp_cn)
 inp_ut = Input(shape=(1,), name='inp_user_type')
 emb_ut = Embedding(config.len_ut, config.emb_ut, name='emb_user_type' )(inp_ut)
 inp_city = Input(shape=(1,), name='inp_city')
 emb_city = Embedding(config.len_city, config.emb_city, name='emb_city' )(inp_city)
 inp_week = Input(shape=(1,), name='inp_week')
 emb_week = Embedding(config.len_week, config.emb_week, name='emb_week' )(inp_week)
 inp_imgt1 = Input(shape=(1,), name='inp_imgt1')
 emb_imgt1 = Embedding(config.len_imgt1, config.emb_imgt1, name='emb_imgt1' )(inp_imgt1)
 inp_p1 = Input(shape=(1,), name='inp_p1')
 emb_p1 = Embedding(config.len_p1, config.emb_p1, name='emb_p1' )(inp_p1)
 inp_p2 = Input(shape=(1,), name='inp_p2')
 emb_p2 = Embedding(config.len_p2, config.emb_p2, name='emb_p2' )(inp_p2)
 inp_p3 = Input(shape=(1,), name='inp_p3')
 emb_p3 = Embedding(config.len_p3, config.emb_p3, name='emb_p3' )(inp_p3)
 conc_cate = concatenate([emb_reg, emb_pcn, emb_cn, emb_ut, emb_city, emb_week, emb_imgt1, emb_p1, emb_p2, emb_p3], axis=-1, name='concat_categorcal_vars')
 conc_cate = GlobalMaxPool1D()(conc_cate)
 inp_price = Input(shape=(1,), name='inp_price')
 emb_price = Dense(config.emb_price, activation='tanh', name='emb_price' )(inp_price)
 inp_itemseq = Input(shape=(1,), name='inp_itemseq')
 emb_itemseq = Dense(config.emb_itemseq, activation='tanh', name='emb_itemseq' )(inp_itemseq)
 conc_cont = concatenate([conc_cate, emb_price, emb_itemseq], axis=-1)
 x = Dense(200, activation='relu' )(conc_cont)
 x = Dense(50, activation='relu' )(x)
 inp_desc = Input(shape=(config.maxlen,), name='inp_desc')
 emb_desc = Embedding(config.len_desc, config.emb_desc, name='emb_desc' )(inp_desc)
 desc_layer = GRU(40, return_sequences=False )(emb_desc)
 conc_desc = concatenate([x, desc_layer], axis=-1)
 outp = Dense(1, activation='sigmoid', name='output' )(conc_desc)
 model = Model(inputs = [inp_reg, inp_pcn, inp_cn, inp_ut, inp_city, inp_week, inp_imgt1, inp_p1, inp_p2, inp_p3,
inp_price, inp_itemseq, inp_desc], outputs = outp)
 return model
SUBGROUP_AUC = 'subgroup_auc'
BPSN_AUC = 'bpsn_auc'
BNSP_AUC = 'bnsp_auc'
TOXICITY_COLUMN = 'target'
def compute_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except ValueError:
return np.nan
def compute_subgroup_auc(df, subgroup, label, model_name):
subgroup_examples = df[df[subgroup]]
return compute_auc(subgroup_examples[label], subgroup_examples[model_name])
 def compute_bpsn_auc(df, subgroup, label, model_name):

 subgroup_negative_examples = df[df[subgroup] & ~df[label]]
non_subgroup_positive_examples = df[~df[subgroup] & df[label]]
examples = subgroup_negative_examples.append(non_subgroup_positive_examples)
 return compute_auc(examples[label], examples[model_name])
 def compute_bnsp_auc(df, subgroup, label, model_name):

 subgroup_positive_examples = df[df[subgroup] & df[label]]
non_subgroup_negative_examples = df[~df[subgroup] & ~df[label]]
examples = subgroup_positive_examples.append(non_subgroup_negative_examples)
 return compute_auc(examples[label], examples[model_name])
 def compute_bias_metrics_for_model(dataset,
subgroups,
model,
label_col,
include_asegs=False):

 records = []
for subgroup in subgroups:
record = {
'subgroup': subgroup,
'subgroup_size': len(dataset[dataset[subgroup]])
 }
record[SUBGROUP_AUC] = compute_subgroup_auc(dataset, subgroup, label_col, model)
 record[BPSN_AUC] = compute_bpsn_auc(dataset, subgroup, label_col, model)
 record[BNSP_AUC] = compute_bnsp_auc(dataset, subgroup, label_col, model)
 records.append(record)
 return pd.DataFrame(records ).sort_values('subgroup_auc', ascending=True )<compute_train_metric>
model = get_model()
model.compile(optimizer='adam', loss = root_mean_squared_error, metrics=[root_mean_squared_error])
 model.summary()
def calculate_overall_auc(df, model_name):
true_labels = df[TOXICITY_COLUMN]
predicted_labels = df[model_name]
return metrics.roc_auc_score(true_labels, predicted_labels)
 def power_mean(series, p):
total = sum(np.power(series, p))
return np.power(total / len(series), 1 / p)
 def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):
bias_score = np.average([
power_mean(bias_df[SUBGROUP_AUC], POWER),
power_mean(bias_df[BPSN_AUC], POWER),
power_mean(bias_df[BNSP_AUC], POWER)
 ])
 return(OVERALL_MODEL_WEIGHT * overall_auc)+(( 1 - OVERALL_MODEL_WEIGHT)* bias_score )<set_options>
checkpoint = ModelCheckpoint('best.hdf5', monitor='val_loss', verbose=1, save_best_only=True)
 early = EarlyStopping(patience=3, mode='min' )
pd.options.mode.chained_assignment = None
class CompetitionMetricEarlyStopping(EarlyStopping):
def __init__(self, model, df, X_valid, *args, **kwargs):
kwargs['monitor'] = 'val_competition_metric'
self.model = model
self.df = df
self.X_valid = X_valid
self.bias_metrics_df = None
super().__init__(*args, **kwargs)
 def get_monitor_value(self, logs):
preds = self.model.predict(self.X_valid, batch_size=2048)[:, 0]
self.df['predictions'] = preds
bias_metrics_df = compute_bias_metrics_for_model(self.df, identity_columns, 'predictions', TOXICITY_COLUMN)
 self.bias_metrics_df = bias_metrics_df
val_competition_metric = get_final_metric(bias_metrics_df, calculate_overall_auc(self.df, 'predictions'))
logs['val_competition_metric'] = val_competition_metric
return val_competition_metric<choose_model_class>
model.fit(x=X_train, y=np.array(Y_train), validation_data=(X_valid, Y_valid), batch_size=config.batch_size, epochs=6, callbacks=[checkpoint,early], verbose=1 )
<compute_train_metric><EOS> 
model.load_weights('best.hdf5')
 pred = model.predict(X_test)
 subm = pd.read_csv(".. /input/sample_submission.csv")
 subm['deal_probability'] = pred
subm.to_csv('submit_{}_{:.4f}.csv'.format('nn_p3', 0.226), index=False )