Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<define_variables>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.1-py3-none-any.whl pytorch-tabnet
def get_all_img_meta(type, recalc):
if recalc == 1:
if type == "train":
path = ".. /input/train_metadata/"
else:
path = ".. /input/test_metadata/"
all_images = [f for f in sorted(os.listdir(path)) if 
(f.endswith(".json")& os.path.isfile(path + f)) ]
len_images = len(all_images)
 pets = list(set([f[:f.find("-")] for f in sorted(os.listdir(path)) if 
(f.endswith(".json")& os.path.isfile(path + f)) ]))
np_pets = np.asarray(pets ).reshape(len(pets), 1)
 df_cols = Columns.img_num_cols_all.value + ["Lbl_Dsc"]
np_data = np.zeros(( len(pets),len(df_cols)))
 data = np.concatenate(( np_pets,np_data), axis=1)
 df_pet_img_meta = pd.DataFrame(columns=["PetID"]+df_cols, data=data)
 df_pet_img_meta["Lbl_Dsc"] = ""
df_pet_img_meta.set_index("PetID", inplace=True)
 h = 1
for pet in pets:
images = [k for k in all_images if pet in k]
p_rgb = 0
p_dominant_pixel_frac = 0
p_dominant_score = 0
p_vertex_x = 0
p_vertex_y = 0
p_bounding_confidence = 0
p_bounding_importance_frac = 0
p_label_score = 0
p_label_description = ""
num_of_images = len(images)
 for img in images:
imgnum = img.split("-", 1)[1].strip(".json")
 with open(path + img, encoding="utf8")as json_data:
image = json.load(json_data)
 h = h+1
i_rgb = 0
i2_rgb = 0
i_dominant_pixel_frac = 0
i_dominant_score = 0
i_label_score = 0
i_label_description = ""
i_num_colors = 0
i_num_label_annotations = 0
i_vertex_x = image['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2].get('x', 0)
 i_vertex_y = image['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2].get('y', 0)
 i_bounding_confidence = image['cropHintsAnnotation']['cropHints'][0].get('confidence', 0)
 i_bounding_importance_frac = image['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', 0)
 if image.get('imagePropertiesAnnotation' ).get("dominantColors" ).get("colors"):
i_num_colors = 0
t_rgb = 0
for color in image.get('imagePropertiesAnnotation' ).get("dominantColors" ).get("colors"):
if color.get('score', 0)>= 0.00:
r = color.get("color" ).get('red', 255)
 g = color.get("color" ).get('green', 255)
 b = color.get("color" ).get('blue', 255)
 m_rgb =(r**2 + g**2 + b**2)/3
t_rgb = t_rgb + m_rgb
i_dominant_pixel_frac = i_dominant_pixel_frac + color.get('pixelFraction', 0)
 i_dominant_score = i_dominant_score + color.get('score', 0)
 i_num_colors = i_num_colors + 1
if i_num_colors>0:
if t_rgb > 0:
i_rgb = np.sqrt(t_rgb/i_num_colors)
 if i_dominant_score > 0:
i_dominant_score = i_dominant_score/i_num_colors
if image.get('labelAnnotations'):
i_num_label_annotations = 0
for ann in image.get('labelAnnotations'):
if ann.get('score', 0)>= 0.00:
i_label_score = ann.get('score', 0)+ i_label_score
i_num_label_annotations = i_num_label_annotations + 1
if ann.get("description"):
i_label_description = i_label_description + " " + ann.get("description", "")
 if i_num_label_annotations>0:
if i_label_score > 0:
i_label_score = i_label_score/i_num_label_annotations
p_rgb = p_rgb + i_rgb**2
p_dominant_pixel_frac = p_dominant_pixel_frac + i_dominant_pixel_frac
p_dominant_score = p_dominant_score + i_dominant_score
p_vertex_x = p_vertex_x + i_vertex_x
p_vertex_y = p_vertex_y + i_vertex_y
p_bounding_confidence = p_bounding_confidence + i_bounding_confidence
p_bounding_importance_frac = p_bounding_importance_frac + i_bounding_importance_frac
p_label_score = p_label_score + i_label_score
p_label_description = p_label_description + " " + i_label_description
p_rgb = np.sqrt(p_rgb/num_of_images)
 pm_dominant_pixel_frac = p_dominant_pixel_frac/num_of_images
pm_dominant_score = p_dominant_score/num_of_images
p_vertex_x = p_vertex_x/num_of_images
p_vertex_y = p_vertex_y/num_of_images
pm_bounding_confidence = p_bounding_confidence/num_of_images
pm_bounding_importance_frac = p_bounding_importance_frac/num_of_images
pm_label_score = p_label_score/num_of_images
df_pet_img_meta.loc[pet, df_cols] = [p_rgb, p_dominant_pixel_frac, pm_dominant_pixel_frac, p_dominant_score, pm_dominant_score,
p_vertex_x, p_vertex_y, p_bounding_confidence, pm_bounding_confidence,
p_bounding_importance_frac, pm_bounding_importance_frac, p_label_score, pm_label_score,
p_label_description]
elif recalc == 0:
df_pet_img_meta = pd.read_csv(type + "_metadata_all.csv")
 return df_pet_img_meta
rc = 1
if 1 == 1:
plog("Starting all images metadata collection on train")
 train_metadata = get_all_img_meta("train", rc)
 plog("Ended all images metadata collection on train")
 plog("Starting all images metadata collection on test")
 test_metadata = get_all_img_meta("test", rc)
 plog("Ended all images metadata collection on test")
 train_df = train_df.set_index("PetID" ).join(train_metadata ).reset_index()
test_df = test_df.set_index("PetID" ).join(test_metadata ).reset_index()
train_df["Lbl_Dsc"].fillna("", inplace=True)
 test_df["Lbl_Dsc"].fillna("", inplace=True)
 for col in Columns.img_num_cols_all.value:
pass
plog("Done" )<choose_model_class>
sys.path.append(".. /input/moa-scripts")
 sys.path.append(".. /input/moa-models")
 seed_everything()
%matplotlib inline
warnings.filterwarnings('ignore')
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
!ls ".. /input"
def tfidf_2(train, n_comp, out_cols):
stop_words = set(stopwords.words('english'))
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, stop_words = stop_words)
 tfv.fit(list(train))
X = tfv.transform(train)
 svd = TruncatedSVD(n_components=n_comp, random_state=1337)
 svd.fit(X)
 plog("explained_variance_ratio_.sum() "+str(svd.explained_variance_ratio_.sum()))
X = svd.transform(X)
 df_svd = pd.DataFrame(X, columns=out_cols)
 return df_svd
plog("TSVD for train description started")
 for c in Columns.desc_svd_cols.value:
if c in train_df.columns.values:
train_df.drop([c], axis=1, inplace=True)
 svd_train_desc = tfidf_2(train_df.Description, Columns.n_desc_svd_comp.value, Columns.desc_svd_cols.value)
 plog("TSVD for train description ended")
 plog("TSVD for test description started")
 for c in Columns.desc_svd_cols.value:
if c in test_df.columns.values:
test_df.drop([c], axis=1, inplace=True)
 svd_test_desc = tfidf_2(test_df.Description, Columns.n_desc_svd_comp.value, Columns.desc_svd_cols.value)
 plog("TFIDF for test description ended")
 plog("TSVD for train image label started")
 for c in Columns.iann_svd_cols.value:
if c in train_df.columns.values:
train_df.drop([c], axis=1, inplace=True)
 svd_train_lbldsc = tfidf_2(train_df.Lbl_Dsc, Columns.n_iann_svd_comp.value, Columns.iann_svd_cols.value)
 plog("TSVD for train image label ended")
 plog("TSVD for test image label started")
 for c in Columns.iann_svd_cols.value:
if c in test_df.columns.values:
test_df.drop([c], axis=1, inplace=True)
 svd_test_lbldsc = tfidf_2(test_df.Lbl_Dsc, Columns.n_iann_svd_comp.value, Columns.iann_svd_cols.value)
 plog("TSVD for test image label ended")
 plog("TSVD for train desc entities started")
 for c in Columns.entities_svd_cols.value:
if c in train_df.columns.values:
train_df.drop([c], axis=1, inplace=True)
 svd_train_entities = tfidf_2(train_df.Entities, Columns.n_entities_svd_comp.value, Columns.entities_svd_cols.value)
 plog("TSVD for train desc entities ended")
 plog("TSVD for test desc entities started")
 for c in Columns.entities_svd_cols.value:
if c in test_df.columns.values:
test_df.drop([c], axis=1, inplace=True)
 svd_test_entities = tfidf_2(test_df.Entities, Columns.n_entities_svd_comp.value, Columns.entities_svd_cols.value)
 plog("TSVD for test desc entities ended")
 train_df = pd.concat([train_df,svd_train_desc, svd_train_lbldsc, svd_train_entities], axis=1)
 test_df = pd.concat([test_df,svd_test_desc, svd_test_lbldsc, svd_test_entities], axis=1)
 print(train_df.shape)
 print(test_df.shape)
 print("Done" )<normalization>
def load_cat() :
X, y, genes, cells, classnames, features, X_test, test_control, submission = load(onehot=False)
 return X, X_test
X, y, genes, cells, classnames, features, X_test, test_control, submission = load()
Xc, Xc_test = load_cat()
assert np.all(Xc[:, 2:] == X[:, 4:])
 assert np.all(Xc_test[:, 2:] == X_test[:, 4:] )
def nmf_d(train, n_comp, out_cols):
stop_words = set(stopwords.words('english'))
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, stop_words = stop_words)
 tfv.fit(list(train))
X = tfv.transform(train)
 nmf = NMF(n_components=n_comp, random_state=1337)
 nmf.fit(X)
 X = nmf.transform(X)
 df_nmf = pd.DataFrame(X, columns=out_cols)
 return df_nmf
plog("NMF for train description started")
 for c in Columns.desc_nmf_cols.value:
if c in train_df.columns.values:
train_df.drop([c], axis=1, inplace=True)
 nmf_train_desc = nmf_d(train_df.Description, Columns.n_desc_nmf_comp.value, Columns.desc_nmf_cols.value)
 plog("NMF for train description ended")
 plog("NMF for test description started")
 for c in Columns.desc_nmf_cols.value:
if c in test_df.columns.values:
test_df.drop([c], axis=1, inplace=True)
 nmf_test_desc = nmf_d(test_df.Description, Columns.n_desc_nmf_comp.value, Columns.desc_nmf_cols.value)
 plog("NMF for test description ended")
 plog("NMF for train image label started")
 for c in Columns.iann_nmf_cols.value:
if c in train_df.columns.values:
train_df.drop([c], axis=1, inplace=True)
 nmf_train_lbldsc = nmf_d(train_df.Lbl_Dsc, Columns.n_iann_nmf_comp.value, Columns.iann_nmf_cols.value)
 plog("NMF for train image label ended")
 plog("NMF for test image label started")
 for c in Columns.iann_nmf_cols.value:
if c in test_df.columns.values:
test_df.drop([c], axis=1, inplace=True)
 nmf_test_lbldsc = tfidf_2(test_df.Lbl_Dsc, Columns.n_iann_nmf_comp.value, Columns.iann_nmf_cols.value)
 plog("NMF for test image label ended")
 plog("NMF for train desc entities started")
 for c in Columns.entities_nmf_cols.value:
if c in train_df.columns.values:
train_df.drop([c], axis=1, inplace=True)
 nmf_train_entities = nmf_d(train_df.Entities, Columns.n_entities_nmf_comp.value, Columns.entities_nmf_cols.value)
 plog("NMF for train desc entities ended")
 plog("NMF for test desc entities started")
 for c in Columns.entities_nmf_cols.value:
if c in test_df.columns.values:
test_df.drop([c], axis=1, inplace=True)
 nmf_test_entities = nmf_d(test_df.Entities, Columns.n_entities_nmf_comp.value, Columns.entities_nmf_cols.value)
 plog("NMF for test desc entities ended")
 train_df = pd.concat([train_df,nmf_train_desc, nmf_train_lbldsc, nmf_train_entities], axis=1)
 test_df = pd.concat([test_df,nmf_test_desc, nmf_test_lbldsc, nmf_test_entities], axis=1)
 train_df.drop(["Description", "Lbl_Dsc", "Entities", "Name"], axis=1, inplace=True)
 test_df.drop(["Description", "Lbl_Dsc", "Entities", "Name"], axis=1, inplace=True)
 for c in Columns.entities_nmf_cols.value + Columns.iann_nmf_cols.value + Columns.desc_nmf_cols.value:
if c in train_df.columns.values:
pass
if c in test_df.columns.values:
pass
print(train_df.shape)
 print(test_df.shape)
 print("Done" )<data_type_conversions>
splits = sorted(glob.glob(".. /input/moa-pipelines/x*.pkl"))
folds, pipes = list(zip(*[joblib.load(split)for split in splits]))
pprint(splits)
 splitsC = sorted(glob.glob(".. /input/moa-pipelines/c*.pkl"))
foldsC, pipesC = list(zip(*[joblib.load(split)for split in splitsC]))
pprint(splitsC )
if 1 == 0:
for c in train_df.drop(["AdoptionSpeed", "PetID", "RescuerID"], axis=1 ).columns.values.tolist() :
if train_df[c].isna().any() :
train_df[c].fillna(0, inplace=True)
 c_features = [train_df.columns.get_loc(c)for c in train_df.columns if c in Columns.ind_num_cat_columns.value + \
["Age", "Fee", "VideoAmt", "PhotoAmt","Quantity"]]
print(len(train_df))
plog("Generating mock data with smote")
 ros = SMOTENC(random_state=0, sampling_strategy='minority', categorical_features=c_features)
 train_df_c = train_df.copy()
x_train_df_c = train_df_c.drop(["AdoptionSpeed", "PetID", "RescuerID"], axis=1)
 y_train_df_c = train_df_c["AdoptionSpeed"]
x_res, y_res = ros.fit_resample(x_train_df_c, y_train_df_c)
 x_res = pd.DataFrame(data=x_res[len(x_train_df_c)+1:], columns=x_train_df_c.columns.values.tolist())
 y_res = pd.DataFrame(data=y_res[len(y_train_df_c)+1:], columns=["AdoptionSpeed"])
 plog(str(len(x_res)) + " lines of mock data has been generated")
 rnd_resc_arr = []
for i in range(int(len(x_res)/4)) :
rnd_RescuerID = ''.join([random.choice(string.ascii_letters + string.digits)for n in range(32)])
 rnd_resc_arr.append(rnd_RescuerID)
 x_res["RescuerID"] = x_res.apply(lambda x : random.choice(rnd_resc_arr), axis=1)
 rnd_petID_arr = []
for i in range(len(x_res)) :
rnd_petID = ''.join([random.choice(string.ascii_letters + string.digits)for n in range(9)])
 rnd_petID_arr.append(rnd_petID)
 x_res["PetID"] = np.asarray(rnd_petID_arr ).reshape(-1,1)
 train_df_smote = pd.concat([x_res, y_res], axis=1, sort=False)
 print(len(train_df_smote))
train_df = pd.concat([train_df, train_df_smote], axis=0 ,sort=False)
 print(len(train_df))
plog("Done" )<create_dataframe>
inputs = {
'mlp': "moa-pytorch-mlp",
'resnet': "moa-mlp2",
'tabnet': "moa-tabnet",
'mlp_gelu': "moa-mlp-gelu"
}
oof_train, oof_test = [], []
def load_models(pth, folds):
return [sorted(glob.glob(f".. /input/{pth}/split{i}*.pth")) for i in range(len(folds)) ]
plog("Sorting breed1 and breed2 cols before concatenation")
 bm_df_tr = pd.DataFrame(np.sort(train_df[["Breed1", "Breed2"]], axis=1), train_df[["Breed1", "Breed2"]].index, train_df[["Breed1", "Breed2"]].columns)
 bm_df_te = pd.DataFrame(np.sort(test_df[["Breed1", "Breed2"]], axis=1), test_df[["Breed1", "Breed2"]].index, test_df[["Breed1", "Breed2"]].columns)
 plog("Sorted breed1 and breed2 cols before concatenation")
 plog("Merging Breed1 and Breed2 on train")
 train_df["Breed_Merge"] = bm_df_tr.apply(lambda x: "_".join([str(x['Breed1']), str(x['Breed2'])]), axis=1)
 plog("Merged Breed1 and Breed2 on train")
 plog("Merging Breed1 and Breed2 on test")
 test_df["Breed_Merge"] = bm_df_te.apply(lambda x: "_".join([str(x['Breed1']), str(x['Breed2'])]), axis=1)
 plog("Merged Breed1 and Breed2 on test")
 def set_pet_breed(b1, b2):
if(b1 in(0, 307)) &(b2 in(0, 307)) :
return 4
elif(b1 == 307)&(b2 not in(0, 307)) :
return 3
elif(b2 == 307)&(b1 not in(0, 307)) :
return 3
elif(b1 not in(0, 307)) &(b2 not in(0, 307)) &(b1 != b2):
return 2
elif(b1 == 0)&(b2 not in(0, 307)) :
return 1
elif(b2 == 0)&(b1 not in(0, 307)) :
return 1
elif(b1 not in(0, 307)) &(b2 not in(0, 307)) &(b1 == b2):
return 0
else:
return 3
plog("Setting Pet Breed for test dataset")
 test_df["Pet_Breed"] = test_df.apply(lambda x: set_pet_breed(x['Breed1'], x['Breed2']), axis=1)
 plog("Setted Pet Breed for test dataset")
 plog("Setting Pet Breed for train dataset")
 train_df["Pet_Breed"] = train_df.apply(lambda x: set_pet_breed(x['Breed1'], x['Breed2']), axis=1)
 plog("Setted Pet Breed for train dataset")
 def chVal(val, col):
if(col == "Health")&(val == 0):
return 4
else:
return val
plog("Setting Pet Breed for test dataset")
 test_df["Pet_Purity"] = test_df.apply(lambda x: 1 if x["Pet_Breed"] in(0, 1)else 0, axis=1)
 plog("Setted Pet Breed for test dataset")
 plog("Setting Pet Purity for train dataset")
 train_df["Pet_Purity"] = train_df.apply(lambda x: 1 if x["Pet_Breed"] in(0, 1)else 0, axis=1)
 plog("Setted Pet Purity for train dataset")
 plog("Setting Pet Breed for test dataset")
 test_df["Overall_Status"] = test_df.apply(lambda x: math.sqrt(x["Sterilized"])+ \
math.sqrt(x["Dewormed"])+ \
math.sqrt(x["Vaccinated"])+ \
math.sqrt(chVal(x["Health"], "Health")) , axis=1)
 plog("Setted Pet Breed for test dataset")
 plog("Setting Pet Purity for train dataset")
 train_df["Overall_Status"] = train_df.apply(lambda x: math.sqrt(x["Sterilized"])+ \
math.sqrt(x["Dewormed"])+ \
math.sqrt(x["Vaccinated"])+ \
math.sqrt(chVal(x["Health"], "Health")) , axis=1)
 plog("Setted Pet Purity for train dataset")
<categorify>
fitter, train_params = get_params_mlp(len(pipes[0][0].columns_))
models = load_models(inputs['mlp'], folds)
 data = [
X, y, X_test, test_control,
models, folds, pipes
]
oof = mlp_inference(*data, fitter, train_params)
 oof_train.append(oof[0]); oof_test.append(oof[1] )
def label_encoder(train, test, col):
le = LabelEncoder()
val_train = pd.unique(train[col])
 val_test = pd.unique(test[col])
 if np.array_equal(val_train, val_test):
vals = val_train
else :
vals = np.unique(np.concatenate(( val_train, val_test), axis=None))
le.fit(vals)
 train[col] = le.transform(train[col])
 test[col] = le.transform(test[col])
 return train, test
if 1 == 1 :
for c in ["State", "Breed_Merge"]:
plog("Started " + c + " encoding")
 train_df, test_df = label_encoder(train_df, test_df, c)
 plog("Ended " + c + " encoding" )<data_type_conversions>
fitter, train_params = get_params_resnet(pipesC[0][0])
 models = load_models(inputs['resnet'], foldsC)
 data = [
Xc, y, Xc_test, test_control,
models, foldsC, pipesC
]
oof = resnet_inference(*data, fitter, train_params)
 oof_train.append(oof[0]); oof_test.append(oof[1] )
for c in Columns.barplot_cols.value:
if c in train_df.columns.values.tolist() :
train_df[c].astype("int32")
 if c in test_df.columns.values.tolist() :
test_df[c].astype("int32")
 for c in Columns.boxplot_cols.value:
if c in train_df.columns.values.tolist() :
train_df[c].astype("float32")
 if c in test_df.columns.values.tolist() :
test_df[c].astype("float32")
 plog("Done" )<feature_engineering>
tabnet_params, _ = get_params_tabnet(len(pipes[0][0].columns_))
models = load_models(inputs['tabnet'], folds)
 data = [
X, y, X_test, test_control,
models, folds, pipes
]
oof = tabnet_inference(*data, tabnet_params)
 oof_train.append(oof[0]); oof_test.append(oof[1] )
plog("Creating Fee_Per_Pet on train dataset")
 train_df["Fee_Per_Pet"] = train_df["Fee"]/train_df["Quantity"]
plog("Created Fee_Per_Pet on train dataset")
 plog("Creating Fee_Per_Pet on test dataset")
 test_df["Fee_Per_Pet"] = test_df["Fee"]/test_df["Quantity"]
plog("Created Fee_Per_Pet on test dataset" )<set_options>
fitter_params = dict(
 activation = nn.GELU
,dropout=[0.3, 0.3]
,estimator=MLPmem
,memory=16
,smoothing=4e-4, clipping=4e-4
,hidden_size=(128, 128)
 ,early_stopping=8 
)
 fitter, train_params = get_params_pretrain(
 len(pipes[0][0].columns_), y.shape[1],
epochs=100,
optim=partial(LookaheadAdamW, k=10),
optim_params=dict(weight_decay=1e-5, lr=1e-2),
scheduler=optim.lr_scheduler.ReduceLROnPlateau,
sched_params=dict(mode='min', factor=0.1, patience=2),
bn_momentum=0.05,
verbose=100,
**fitter_params 
)
 models = load_models(inputs['mlp_gelu'], folds)
 data = [
X, y, X_test, test_control,
models, folds, pipes
]
oof = mlp_inference(*data, fitter, train_params)
 oof_train.append(oof[0]); oof_test.append(oof[1] )
def set_pet_maturity(typ, age):
if typ == 1:
if age <= 6:
return 0
elif age <= 12:
return 1
elif age <= 3*12:
return 2
elif age <= 6*12:
return 3
elif age <=10*12:
return 4
else:
return 5
if typ == 2 :
if age <= 6:
return 0
elif age <= 12:
return 1
elif age <= 3*12:
return 2
elif age <= 6*12:
return 3
elif age <=10*12:
return 4
else:
return 5
plog("Setting Pet_Maturity for test dataset")
 test_df["Pet_Maturity"] = test_df.apply(lambda x: set_pet_maturity(x['Type'], x['Age']), axis=1)
 plog("Setted Pet_Maturitye for test dataset")
 plog("Setting Pet_Maturity for train dataset")
 train_df["Pet_Maturity"] = train_df.apply(lambda x: set_pet_maturity(x['Type'], x['Age']), axis=1)
 plog("Setted Pet_Maturity for train dataset" )<feature_engineering>
oof_train = np.array(oof_train)
 oof_test = np.array(oof_test )
def set_color_type(c1, c2, c3):
if c1 != 0:
c1 = 1
if c2 != 0:
c2 = 1
if c3 != 0:
c3 = 1
return c1 + c2 + c3
plog("Setting Color Type for test dataset")
 test_df["Color_Type"] = test_df.apply(lambda x: set_color_type(x['Color1'], x['Color2'], x['Color3']), axis=1)
 plog("Setted Color Type for test dataset")
 plog("Setting Color Type for train dataset")
 train_df["Color_Type"] = train_df.apply(lambda x: set_color_type(x['Color1'], x['Color2'], x['Color3']), axis=1)
 plog("Setted Color Type for train dataset" )<drop_column>
w = [0.3, 0.3, 0.3, 0.1]
oof_train_ens = np.array([np.array(w)@ oof_train[:, :, i] for i in range(206)] ).T
oof_test_ens = np.array([np.array(w)@ oof_test[:, :, i] for i in range(206)] ).T
def set_mean_fee(df, col):
dfm = df[[col, "Fee", "Quantity"]].groupby(col ).agg({"Fee":"sum", "Quantity":"sum"} ).reset_index()
dfm["Fee_" + c] = dfm["Fee"] / dfm["Quantity"]
dfm.drop(["Quantity","Fee"], axis = 1, inplace=True)
 df = df.set_index(col ).join(dfm.set_index(col)).reset_index()
return df
for c in Columns.fee_mean_incols.value:
if "Fee_" + c in train_df.columns.values.tolist() :
plog("Dropping "+"Fee_Per_Pet_" + c + " on train dataset")
 train_df.drop("Fee_Per_Pet_" + c, axis=1, inplace=True)
 plog("Creating Fee_Per_Pet_" + c + " on train dataset")
 train_df = set_mean_fee(train_df, c)
 plog("Created Fee_Per_Pet_" + c + " on train dataset")
 if "Fee_" + c in test_df.columns.values.tolist() :
plog("Dropping "+"Fee_Per_Pet_" + c + " on test dataset")
 test_df.drop("Fee_Per_Pet_" + c, axis=1, inplace=True)
 plog("Creating Fee_Per_Pet_" + c + " on test dataset")
 test_df = set_mean_fee(test_df, c)
 plog("Created Fee_Per_Pet_" + c + " on test dataset")
<categorify>
clipping = 1e-5
oof_test_ens = np.clip(oof_test_ens, clipping, 1-clipping)
 oof_train_ens = np.clip(oof_train_ens, clipping, 1-clipping )
for c in Columns.kbin_cols.value:
if c in train_df.columns.values.tolist() :
train_df.drop(c, axis=1, inplace = True)
 if c in test_df.columns.values.tolist() :
test_df.drop(c, axis=1, inplace = True)
 for c in Columns.kbin_incols.value:
if c in train_df.columns.values.tolist() :
kbin_est = KBinsDiscretizer(n_bins=12, encode='ordinal', strategy='quantile')
 kbin_est.fit(train_df[c].values.reshape(-1, 1))
kbin_train = kbin_est.transform(train_df[c].values.reshape(-1, 1))
pd_kbin_train = pd.DataFrame(data=kbin_train, columns=[c+"_Kbin"])
 train_df = pd.concat([train_df,pd_kbin_train], axis=1)
 kbin_test = kbin_est.transform(test_df[c].values.reshape(-1, 1))
pd_kbin_test = pd.DataFrame(data=kbin_test, columns=[c+"_Kbin"])
 test_df = pd.concat([test_df,pd_kbin_test], axis=1)
 plog("Done" )<feature_engineering>
joblib.dump(oof_train_ens, "oof_train_ens.pkl")
 joblib.dump(oof_test_ens, "oof_test_ens.pkl" )
def setItemAdp(val):
if val > 3.5:
return 4
elif val > 2.5:
return 3
elif val > 1.5:
return 2
elif val > 0.5:
return 1
else:
return 0
def itemAdp(df, col):
train_r = df.groupby(col)["AdoptionSpeed"].mean().reset_index()
train_r[col+"_Adp"] = train_r.apply(lambda x: setItemAdp(x['AdoptionSpeed']), axis=1)
 return train_r[[col, col+"_Adp"]]
if 1 == 1:
for c in Columns.item_adp_incols.value:
if c+"_Adp" in train_df.columns.values:
plog("Deleting train "+c+"_Adp")
 train_df.drop([c+"_Adp"], axis=1, inplace=True)
 plog("Creating "+c+"_Adp for train on "+ c)
 df_itr = itemAdp(train_df, c)
 train_df = train_df.set_index(c ).join(df_itr.set_index(c)).reset_index()
plog("Created "+c+"_Adp for train on " + c)
 if c+"_Adp" in test_df.columns.values:
plog("Deleting test "+c+"_Adp")
 test_df.drop([c+"_Adp"], axis=1, inplace=True)
 plog("Creating "+c+"_Adp for test on "+ c)
 test_df = test_df.set_index(c ).join(df_itr.set_index(c)).reset_index()
plog("Created "+c+"_Adp for test on " + c)
 plog("Done" )<drop_column>
corrmatrix = np.ones(( N_MODELS, N_MODELS))
corrmodel = {}
for i in range(N_MODELS):
for j in range(i+1, N_MODELS):
corrmodel[(i,j)] = np.zeros(206, dtype=np.float32)
 for z in range(206):
corrmodel[(i,j)][z] = np.corrcoef(oof_train[i, :, z], oof_train[j, :, z])[0, 1]
corrmodel[(i,j)] = np.where(np.isnan(corrmodel[(i,j)]), 0, corrmodel[(i,j)])
 corrmodel[(i,j)] = corrmodel[(i,j)].mean()
corrmatrix[i][j] = corrmodel[(i,j)]
corrmatrix[j][i] = corrmodel[(i,j)]
[print(k, v)for k, v in corrmodel.items() ]
fig, ax = plt.subplots(figsize=(8, 5))
sns.heatmap(corrmatrix, annot=True);
train_df.drop(["RescuerID"], axis=1, inplace=True)
 test_df.drop(["RescuerID"], axis=1, inplace=True)
 plog("Done" )<create_dataframe>
prediction = join_control_group(oof_test_ens, test_control, submission, classnames)
 submit_preds(prediction, submission, test_control, classnames)
 sub = pd.read_csv("submission.csv");
print(sub.shape)
 sub.iloc[:5, :5]
<create_dataframe><EOS> 
def classwise_logloss(y, yhat):
yhat = np.clip(yhat, 1e-15, 1-1e-15)
 return -(y * np.log(yhat)+(1-y)*(np.log(1-yhat)) ).mean(axis=0)
 errors = np.zeros(( N_MODELS, 206))
for i in range(N_MODELS):
errors[i] = classwise_logloss(y, oof_train[i])
 errors = pd.DataFrame(errors, columns=classnames, index=list(inputs.keys()))
errors.to_csv("errors.csv", index=True )