Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<feature_engineering>
!pip install.. /input/iter-strat
test['BadName'] = 0
test.loc[test['Name'].apply(lambda x: len(str(x)))< 3,'BadName']=1<feature_engineering>
ModelCheckpoint, EarlyStopping, LearningRateScheduler, ReduceLROnPlateau)
 %matplotlib inline
%config InlineBackend.figure_format = 'svg'
train_id = train['PetID']
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score<feature_engineering>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 SEED = 1984
seed_everything(SEED )
test_id = test['PetID']
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score<prepare_x_and_y>
x_develop = pd.read_csv('.. /input/lish-moa/train_features.csv')
 y_develop = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 x_test= pd.read_csv('.. /input/lish-moa/test_features.csv')
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 c_cols = x_develop.columns[x_develop.columns.str.startswith('c-')]
g_cols = x_develop.columns[x_develop.columns.str.startswith('g-')]
cont_cols = g_cols.to_list() + c_cols.to_list()
target_cols = y_develop.columns[1:]
N_TARGETS = len(target_cols )
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores<prepare_x_and_y>
def preprocess_df(df):
if 'sig_id' in df.columns:
df.set_index('sig_id', inplace=True)
 df['cp_type'] =(df['cp_type'] == 'trt_cp' ).astype(int)
 df['cp_dose'] =(df['cp_dose'] == 'D2' ).astype(int)
 df = df.join(pd.get_dummies(df['cp_time'], drop_first=False, prefix='cp_time'))
df = df.drop('cp_time', axis=1)
 return df
target=train['AdoptionSpeed']
train=train.drop(['AdoptionSpeed'],axis=1 )<drop_column>
x_develop = preprocess_df(x_develop)
 y_develop = y_develop.set_index('sig_id')
 x_test = preprocess_df(x_test )
df_train=train.drop(['Name','RescuerID','Description','PetID','label_description'],axis=1)
 df_test=test.drop(['Name','RescuerID','Description','PetID','label_description'],axis=1 )<choose_model_class>
n_comp = 28
pca_g = PCA(n_components=n_comp, random_state=SEED)
 x_develop_pca = pca_g.fit_transform(x_develop[g_cols])
 x_develop_pca = pd.DataFrame(x_develop_pca, columns=[f'pca_g-{i}' for i in range(n_comp)], index=x_develop.index)
 x_develop = pd.concat(( x_develop, x_develop_pca), axis=1)
 x_test_pca = pca_g.transform(x_test[g_cols])
 x_test_pca = pd.DataFrame(x_test_pca, columns=[f'pca_g-{i}' for i in range(n_comp)], index=x_test.index)
 x_test = pd.concat(( x_test, x_test_pca), axis=1)
 cont_cols += [f'pca_g-{i}' for i in range(n_comp)]
n_comp = 5
pca_c = PCA(n_components=n_comp, random_state=SEED)
 x_develop_pca = pca_c.fit_transform(x_develop[c_cols])
 x_develop_pca = pd.DataFrame(x_develop_pca, columns=[f'pca_c-{i}' for i in range(n_comp)], index=x_develop.index)
 x_develop = pd.concat(( x_develop, x_develop_pca), axis=1)
 x_test_pca = pca_c.transform(x_test[c_cols])
 x_test_pca = pd.DataFrame(x_test_pca, columns=[f'pca_c-{i}' for i in range(n_comp)], index=x_test.index)
 x_test = pd.concat(( x_test, x_test_pca), axis=1)
 cont_cols += [f'pca_c-{i}' for i in range(n_comp)]
xg=xgb.XGBClassifier()
xg.fit(df_train,target)
 pred_xgb=xg.predict(df_test )<train_model>
class VarianceThreshold:
def __init__(self, threshold):
self.threshold = threshold
def fit(self, df, cont_cols):
self.cont_cols = cont_cols
self.var = x_develop[cont_cols].var()
good_cols = self.var[self.var > self.threshold]
self.index = good_cols.index.to_list()
self.dropcols = [x for x in cont_cols if x not in self.var[self.var > self.threshold].index.to_list() ]
self.validcols = [x for x in cont_cols if x in self.var[self.var > self.threshold].index.to_list() ]
def transform(self, df):
return df.drop(self.dropcols, axis=1)
 def fit_transform(self, df, cont_cols):
self.fit(df, cont_cols)
 return self.transform(df), self.validcols
rf=RandomForestClassifier()
rf.fit(df_train,target)
 pred_rf=rf.predict(df_test )<find_best_model_class>
threshold = 0.6
print('Variance Threshold:', threshold)
 VarThres = VarianceThreshold(threshold)
 x_develop, cont_cols = VarThres.fit_transform(x_develop, cont_cols)
 x_test = VarThres.transform(x_test )
gb=lgb.LGBMClassifier()
gb.fit(df_train,target)
 pred_gb=gb.predict(df_test )<train_model>
qt = QuantileTransformer(output_distribution='normal')
 x_develop[cont_cols] = qt.fit_transform(x_develop[cont_cols])
 x_test[cont_cols] = qt.transform(x_test[cont_cols] )
abc=AdaBoostClassifier()
abc.fit(df_train,target)
 pred_abc=abc.predict(df_test )<choose_model_class>
def create_folds(df, fold_no, fold_type='mls_kfold', save=False):

 if fold_type == 'kfold':
kf = KFold(n_splits=fold_no, shuffle=True, random_state=SEED)
 elif fold_type == 'mls_kfold':
kf = MultilabelStratifiedKFold(n_splits=fold_no, random_state=SEED)
 df['Fold'] = -1
df.reset_index(inplace=True)
 for fold,(t, v)in enumerate(kf.split(df, df)) :
df.loc[v, 'Fold'] = fold
df.set_index('sig_id', inplace=True)
 if save:
df.to_csv('y_develop.csv' )
gbc=GradientBoostingClassifier()
gbc.fit(df_train,target)
 pred_gbc=gbc.predict(df_test )<train_on_grid>
N_FOLDS = 30
fold_type = 'mls_kfold'
create_folds(y_develop, fold_no=N_FOLDS, fold_type=fold_type, save=True )
etc=ExtraTreesClassifier()
etc.fit(df_train,target)
 pred_etc=etc.predict(df_test )<predict_on_test>
class Model() :
def __init__(self, input_shape, output_bias=None):
self.input_shape = input_shape
self.output_bias = output_bias
def create_model1(self):
if self.output_bias is not None:
self.output_bias = tf.keras.initializers.Constant(output_bias)
 inputs = tf.keras.Input(shape=self.input_shape)
 x = L.BatchNormalization()(inputs)
 x = tfa.layers.WeightNormalization(L.Dense(800, activation='swish'))(x)
 x = L.BatchNormalization()(x)
 x = L.Dropout(0.4 )(x)
 x = tfa.layers.WeightNormalization(L.Dense(400, activation='swish'))(x)
 x = L.BatchNormalization()(x)
 x = L.Dropout(0.4 )(x)
 outputs = tfa.layers.WeightNormalization(L.Dense(N_TARGETS,
activation='sigmoid',
bias_initializer=self.output_bias 
)
 )(x)
 model = tf.keras.Model(inputs=inputs, outputs=outputs)
 metrics = [tf.keras.losses.BinaryCrossentropy(name='mean_loss')]
OPTIMIZER = tfa.optimizers.Lookahead(
 tfa.optimizers.AdamW(weight_decay=1e-5),
sync_period=5)
 model.compile(optimizer=OPTIMIZER, loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.0008), metrics=metrics)
 return model
svc=SVC()
svc.fit(df_train,target)
 pred_svc=svc.predict(df_test )<prepare_output>
def calc_pred(x_df, models):
if isinstance(models[0], np.ndarray):
pred_test = np.repeat([models[0]], len(x_df), axis=0)
 else:
for i in models:
if i == 0:
pred_test = models[i].predict(x_df)
 else:
pred_test += models[i].predict(x_df)
 pred_test = pred_test/len(models)
 pred_test = pd.DataFrame(index=x_df.index, columns=target_cols, data=pred_test)
 pred_test.loc[x_df['cp_type'] == 0] = 0
return pred_test
def oof_score(oof: dict):
return np.mean(list(oof.values())) , np.std(list(oof.values()))
def combine_pred(pred):
for i in pred:
if i==0:
r = pred[i]
else:
r = np.append(r, pred[i], axis=0)
 return r
def run_dump_baseline_cv(xtrain=x_develop, ytrain=y_develop, N_FOLDS=N_FOLDS, summary=True, debug=False):
models = {x: '' for x in range(N_FOLDS)}
results = {x: '' for x in range(N_FOLDS)}
oof_bp = {x: [] for x in range(N_FOLDS)}
oof_ap = {x: [] for x in range(N_FOLDS)}
pred_val_fold = {x: [] for x in range(N_FOLDS)}
for foldno in np.sort(y_develop['Fold'].unique()):
x_train_fold = x_develop[y_develop['Fold'] != foldno]
y_train_fold = y_develop[y_develop['Fold'] != foldno].drop('Fold', axis=1)
 x_val_fold = x_develop[y_develop['Fold'] == foldno]
y_val_fold = y_develop[y_develop['Fold'] == foldno].drop('Fold', axis=1)
 train_sample_size = len(y_train_fold)
 val_sample_size = len(y_val_fold)
 print(" ")
 print(f"Fold-%d" %(foldno))
print("Train sample size:", train_sample_size, ", Validation sample size:", val_sample_size)
 y_dev = y_develop.drop('Fold', axis=1)
 models[foldno] = y_dev.mean(axis=0 ).to_numpy()
pred_val_fold[foldno] = np.repeat([models[foldno]], len(y_val_fold), axis=0)
 oof_bp[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score: ', oof_bp[foldno])
 pred_val_fold[foldno][x_val_fold['cp_type'] == 0] == 0
oof_ap[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score with post processing: ', oof_ap[foldno])
 pred_val_fold = combine_pred(pred_val_fold)
 print('
')
 if summary:
print('Summary')
 print('Mean OOF score: %f +/- %f' %(oof_score(oof_bp)))
 print('Mean OOF score after postprocessing: %f +/- %f' %(oof_score(oof_ap)))
 return models, oof_ap, pred_val_fold
def run_cv(xtrain=x_develop, ytrain=y_develop, model=None, N_FOLDS=N_FOLDS, summary=True, debug=False):
histories = {x: '' for x in range(N_FOLDS)}
models = {x: '' for x in range(N_FOLDS)}
results = {x: '' for x in range(N_FOLDS)}
oof_bp = {x: [] for x in range(N_FOLDS)}
oof_ap = {x: [] for x in range(N_FOLDS)}
pred_val_fold = {x: [] for x in range(N_FOLDS)}
for foldno in np.sort(y_develop['Fold'].unique()):
x_train_fold = x_develop[y_develop['Fold'] != foldno]
y_train_fold = y_develop[y_develop['Fold'] != foldno].drop('Fold', axis=1)
 x_val_fold = x_develop[y_develop['Fold'] == foldno]
y_val_fold = y_develop[y_develop['Fold'] == foldno].drop('Fold', axis=1)
 train_sample_size = len(y_train_fold)
 val_sample_size = len(y_val_fold)
 print(" ")
 print(f"Fold-%d" %(foldno))
print("Train sample size:", train_sample_size, ", Validation sample size:", val_sample_size)
 FEATURE_SIZE = x_train_fold.shape[-1]
train_ds = tf.data.Dataset.from_tensor_slices(( x_train_fold, y_train_fold))
train_ds = train_ds.shuffle(1024 ).batch(56)
 val_ds = tf.data.Dataset.from_tensor_slices(( x_val_fold, y_val_fold))
val_ds = val_ds.batch(val_sample_size)
 models[foldno] = model.create_model1()
cb_es = EarlyStopping(monitor='val_mean_loss', patience=5, restore_best_weights=True)
 reduce_lr_loss = ReduceLROnPlateau(monitor='val_mean_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 histories[foldno] = models[foldno].fit(train_ds, validation_data=val_ds, epochs=EPOCHS, callbacks=[cb_es, reduce_lr_loss], verbose=1)
 pred_val_fold[foldno] = models[foldno].predict(x_val_fold)
 oof_bp[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score: ', oof_bp[foldno])
 pred_val_fold[foldno][x_val_fold['cp_type'] == 0] == 0
oof_ap[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score with post processing: ', oof_ap[foldno])
 if SAVE_MODEL:
models[foldno].save(f'weights-fold{foldno}.h5')
 pred_val_fold = combine_pred(pred_val_fold)
 print('
')
 if summary:
print('Summary')
 print('Mean OOF score: %f +/- %f' %(oof_score(oof_bp)))
 print('Mean OOF score after postprocessing: %f +/- %f' %(oof_score(oof_ap)))
 return models, histories, oof_ap, pred_val_fold
def submit(res):
sub = res.reset_index()
sub.to_csv('submission.csv', index=False )
df=pd.DataFrame()
df['abc']=pred_abc
df['etc']=pred_etc
df['gb']=pred_gb
df['gbc']=pred_gbc
df['rf']=pred_rf
df['xgb']=pred_xgb
df['svc']=pred_svc
df.head()<set_options>
EPOCHS = 45
SAVE_MODEL = True
df.mode(axis=1 )<save_to_csv>
if sub.shape[0] != 3982:
output_bias = -np.log(y_develop[y_develop.columns[:-1]].mean(axis=0 ).to_numpy())
 models, histories, oof_ap, pred_val_fold = run_cv(x_develop, y_develop,
model=Model(x_develop.shape[1], output_bias),
N_FOLDS=N_FOLDS,
debug=False)
 pred_test = calc_pred(x_test, models)
 submit(pred_test)
 np.save('LBS.npy', pred_val_fold)
 else:
sub.to_csv('submission.csv', index=False )