Kaggle competition: Mechanisms of Action (MoA) Prediction
import numpy as np
import pandas as pd
import gc
import os<prepare_output>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
MJTCP = 32292
def assign_rnd_integer(dataset, number_of_times = 5, seed = MJTCP):
new_dataset = pd.DataFrame()
np.random.seed(seed)
 for c in dataset.columns:
for i in range(number_of_times):
col_name = c+"_"+str(i)
 unique_vals = dataset[c].unique()
labels = np.array(list(range(len(unique_vals))))
np.random.shuffle(labels)
 mapping = pd.DataFrame({c: unique_vals, col_name: labels})
 new_dataset[col_name] =(dataset[[c]] 
.merge(mapping, on = c, how = 'left')[col_name]
).values
return new_dataset
def extract_col_interaction(dataset, col1, col2, tfidf = True):
data = dataset.groupby([col1])[col2].agg(lambda x: " ".join(list([str(y)for y in x])))
 if tfidf:
vectorizer = TfidfVectorizer(tokenizer=lambda x: x.split(" "))
else:
vectorizer = CountVectorizer(tokenizer=lambda x: x.split(" "))
data_X = vectorizer.fit_transform(data)
 dim_red = TruncatedSVD(n_components=1, random_state = 5511)
 data_X = dim_red.fit_transform(data_X)
 result = pd.DataFrame()
result[col1] = data.index.values
result[col1+"_{}_svd".format(col2)] = data_X.ravel()
return result
def get_col_interactions_svd(dataset, tfidf = True):
new_dataset = pd.DataFrame()
for col1,col2 in itertools.permutations(dataset.columns, 2):
data = extract_col_interaction(dataset, col1,col2, tfidf)
 col_name = [x for x in data.columns if "svd" in x][0]
new_dataset[col_name] =(dataset[[col1]] 
.merge(data, on = col1, how = 'left')
)[col_name]
return new_dataset
def get_freq_encoding(dataset):
new_dataset = pd.DataFrame()
for c in dataset.columns:
data = dataset.groupby([c] ).size().reset_index()
new_dataset[c+"_freq"] = dataset[[c]].merge(data, on = c, how = "left")[0]
return new_dataset
def transform_dataset(train, test, func, func_params = {}):
dataset = pd.concat([train, test], ignore_index = True)
 dataset = func(dataset, **func_params)
 if isinstance(dataset, pd.DataFrame):
new_train = dataset.iloc[:train.shape[0],:].reset_index(drop = True)
 new_test = dataset.iloc[train.shape[0]:,:].reset_index(drop = True)
 else:
new_train = dataset[:train.shape[0]]
new_test = dataset[train.shape[0]:]
return new_train, new_test
def get_model(params, random_state = 42):
return lgbm.LGBMClassifier(
 n_estimators=250,
metric='auc',
objective='binary',
n_jobs=3,
random_state = random_state,
**params 
)
 class TargetEncoding() :
def __init__(self, columns_names):
self.columns_names = columns_names
self.learned_values = {}
self.dataset_mean = np.nan
def fit(self, X, y, **fit_params):
X_ = X.copy()
self.learned_values = {}
X_["__target__"] = y
for c in [x for x in X_.columns if x in self.columns_names]:
self.learned_values[c] =(X_[[c,"__target__"]] 
.groupby(c)["__target__"].mean() 
.reset_index())
 self.dataset_mean = np.mean(y)
 return self
def transform(self, X, **fit_params):
transformed_X = X[self.columns_names].copy()
for c in transformed_X.columns:
transformed_X[c] =(transformed_X[[c]] 
.merge(self.learned_values[c], on = c, how = 'left')
)["__target__"]
transformed_X = transformed_X.fillna(self.dataset_mean)
 return transformed_X
def fit_transform(self, X, y, **fit_params):
self.fit(X,y)
 return self.transform(X)
 def get_CV_target_encoding(data, y, encoder, cv = 5):
skfTE = StratifiedKFold(n_splits=cv, random_state = 545167, shuffle = True)
 result = []
for train_indexTE, test_indexTE in skfTE.split(data, y):
encoder.fit(data.iloc[train_indexTE,:].reset_index(drop = True), y[train_indexTE])
 tmp = encoder.transform(data.iloc[test_indexTE,:].reset_index(drop = True))
tmp["index"] = test_indexTE
result.append(tmp)
 result = pd.concat(result, ignore_index = True)
 result = result.sort_values('index' ).reset_index(drop = True ).drop('index', axis = 1)
 return result<data_type_conversions>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
train, test = amazon()
target = "ACTION"
col4train = [x for x in train.columns if x not in [target, "ROLE_TITLE"]]
y = train[target].values
train[col4train] = train[col4train].values.astype(str)
 test[col4train] = test[col4train].values.astype(str)
 new_col4train = col4train
for c1,c2 in combinations(col4train, 2):
name = "{}_{}".format(c1,c2)
 new_col4train.append(name)
 train[name] = train[c1] + "_" + train[c2]
test[name] = test[c1] + "_" + test[c2]<define_variables>
def preprocessor_TabNet() :
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 train_drug = pd.read_csv(data_path + "train_drug.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv")
 if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp = train[cols_numeric].loc[:, mask]
train = pd.concat([train[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
test = pd.concat([test[["sig_id", "cp_type", "cp_time", "cp_dose"]], test.loc[:,cols_numeric]], axis = 1)
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
if scale == "rankgauss":
print(b_, "Rank Gauss")
 for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
else:
pass
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train = pd.concat([train, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test[GENES])
 pca_cells_test = pca_cells.transform(test[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test = pd.concat([test, pca_genes_test, pca_cells_test], axis = 1)
 else:
pass
if encoding == "dummy":
print(b_, "One-Hot")
 train = pd.get_dummies(train, columns = ["cp_time", "cp_dose"])
 test = pd.get_dummies(test, columns = ["cp_time", "cp_dose"])
 else:
pass
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
train["g_" + stats] = getattr(train[GENES], stats )(axis = 1)
 train["c_" + stats] = getattr(train[CELLS], stats )(axis = 1)
 train["gc_" + stats] = getattr(train[GENES + CELLS], stats )(axis = 1)
 test["g_" + stats] = getattr(test[GENES], stats )(axis = 1)
 test["c_" + stats] = getattr(test[CELLS], stats )(axis = 1)
 test["gc_" + stats] = getattr(test[GENES + CELLS], stats )(axis = 1)
 train = train.merge(targets, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 target_cols = [x for x in targets.columns if x != 'sig_id']
train_df = train
train_df.reset_index(drop=True, inplace=True)
 test.reset_index(drop = True, inplace = True)
 features_to_drop = ["sig_id", "cp_type"]
test.drop(features_to_drop, axis = 1, inplace = True)
 feature_cols = [c for c in train_df.columns if c not in targets.columns]
feature_cols = [c for c in feature_cols if c not in [ 'sig_id', 'drug_id', 'cp_type']]
return train, targets, test, submission, target_cols, feature_cols
cols_svd = ['MGR_ID_ROLE_CODE','MGR_ID_ROLE_DEPTNAME','MGR_ID_ROLE_FAMILY',
'RESOURCE_MGR_ID','RESOURCE_ROLE_CODE', 'RESOURCE_ROLE_FAMILY',
'RESOURCE_ROLE_ROLLUP_1','RESOURCE_ROLE_ROLLUP_2','RESOURCE',
'ROLE_DEPTNAME_ROLE_CODE','ROLE_DEPTNAME_ROLE_FAMILY',
'ROLE_FAMILY_DESC_ROLE_FAMILY','ROLE_FAMILY_ROLE_CODE',
'ROLE_FAMILY','ROLE_ROLLUP_1_ROLE_DEPTNAME',
'ROLE_ROLLUP_1_ROLE_FAMILY_DESC', 'ROLE_ROLLUP_1_ROLE_FAMILY',
'ROLE_ROLLUP_1','ROLE_ROLLUP_2']
cols_rnd = ['MGR_ID_ROLE_DEPTNAME','MGR_ID_ROLE_FAMILY','MGR_ID_ROLE_ROLLUP_1',
'MGR_ID_ROLE_ROLLUP_2','MGR_ID','RESOURCE_MGR_ID','RESOURCE_ROLE_CODE',
'RESOURCE_ROLE_FAMILY_DESC','RESOURCE_ROLE_FAMILY','RESOURCE_ROLE_ROLLUP_1',
'RESOURCE_ROLE_ROLLUP_2','ROLE_DEPTNAME_ROLE_FAMILY_DESC','ROLE_FAMILY_DESC_ROLE_CODE',
'ROLE_FAMILY_DESC_ROLE_FAMILY','ROLE_FAMILY','ROLE_ROLLUP_1_ROLE_CODE',
'ROLE_ROLLUP_1_ROLE_DEPTNAME','ROLE_ROLLUP_1_ROLE_FAMILY_DESC','ROLE_ROLLUP_2_ROLE_FAMILY']
cols_freq = ['MGR_ID_ROLE_DEPTNAME','RESOURCE_MGR_ID','RESOURCE_ROLE_CODE',
'RESOURCE_ROLE_DEPTNAME','RESOURCE_ROLE_FAMILY_DESC','RESOURCE_ROLE_FAMILY',
'RESOURCE_ROLE_ROLLUP_1','ROLE_DEPTNAME_ROLE_FAMILY_DESC','ROLE_DEPTNAME_ROLE_FAMILY',
'ROLE_DEPTNAME','ROLE_FAMILY_DESC_ROLE_CODE','ROLE_FAMILY_DESC_ROLE_FAMILY',
'ROLE_ROLLUP_1_ROLE_CODE','ROLE_ROLLUP_2_ROLE_DEPTNAME']
data_svd = transform_dataset(train[cols_svd], test[cols_svd], get_col_interactions_svd)
 data_rnd = transform_dataset(train[cols_rnd], test[cols_rnd],
assign_rnd_integer, {"number_of_times":5})
 data_freq = transform_dataset(train[cols_freq], test[cols_freq], get_freq_encoding )<concatenate>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 3
NFOLDS = 7
DRUG_THRESH = 18
train_df = make_cv_folds(train_df, SEEDS, NFOLDS, DRUG_THRESH )
data_train = pd.concat([x[0] for x in [data_svd, data_rnd, data_freq]], axis = 1)
 data_test = pd.concat([x[1] for x in [data_svd, data_rnd, data_freq]], axis = 1 )<set_options>
X_test = test_TabNet.values
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10 
)
 class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux)
 test_cv_preds = []
for s in range(SEEDS):
tabnet_params['seed'] = s
for fold_id in range(NFOLDS):
print(b_,"FOLDS: ", r_, fold_id + 1, y_, 'seed:', tabnet_params['seed'])
 print(g_, '*' * 60, c_)
 saved_path_name = '.. /input/tabnet-oof-seed3-fold7-112220/TabNet_seed_'\
+str(s)+'_fold_'+str(fold_id+1)
 saved_file = '/kaggle/working/model'
shutil.make_archive(saved_file,'zip',saved_path_name)
 print(f'Successfully saved model at {saved_file}.zip')
 device =('cuda' if torch.cuda.is_available() else 'cpu')
 with zipfile.ZipFile('model.zip')as z:
with z.open('model_params.json')as f:
loaded_params = json.load(f)
 with z.open('network.pt')as f:
try:
saved_state_dict = torch.load(f, map_location=device)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.bytesIO(f.read()),map_location = device)
 loaded_model = TabNetRegressor(**loaded_params)
 loaded_model._set_network()
loaded_model.network.load_state_dict(saved_state_dict)
 loaded_model.network.eval()
preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 test_preds_all = np.stack(test_cv_preds)
 saved_path_name = '.. /input/tabnet-oof-seed3-fold7-112220/'
oof_TabNet_all = np.load(saved_path_name + 'oof_TabNet_all.npy' )
del([data_svd, data_rnd, data_freq])
 gc.collect()
data_train = data_train.values
data_test = data_test.values
gc.collect()<train_model>
data_path = ".. /input/lish-moa/"
all_feat = [col for col in submission_TabNet.columns if col not in ["sig_id"]]
test_TabNet = pd.read_csv(data_path + "test_features.csv")
 sig_id = test_TabNet[test_TabNet["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission_TabNet = pd.merge(test_TabNet[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission_TabNet.fillna(0, inplace = True)
 submission_TabNet.to_csv("submission_TabNet.csv", index = None)
 submission_TabNet.head()
params = {'colsample_bytree': 0.312002398119274,
'lambda_l1': 1.919962415701389,
'learning_rate': 0.03363113877976891,
'max_bin': 484,
'max_depth': 10,
'min_child_weight': 0.035307873174480586,
'num_leaves': 220}
seeds = [42,146,78,6257,8373,737,5425,154,6262]
predictions_1 = []
for s in seeds:
model = get_model(params, random_state = s)
 model.fit(data_train, y)
 predictions_1.append(model.predict_proba(data_test)[:,1].ravel())
 predictions_1 = np.stack(predictions_1, -1 ).mean(-1 )<drop_column>
seed = 42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(seed)
 def preprocessor_nn_transfer() :
variance_threshould = 0.8
ncompo_genes = 600
ncompo_cells = 50
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 train_features = train_features[train_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test_features = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train_features = train_features.drop('cp_type', axis=1)
 test_features = test_features.drop('cp_type', axis=1)
 GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train_features[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train_features[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train_features = pd.concat([train_features, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test_features[GENES])
 pca_cells_test = pca_cells.transform(test_features[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test_features = pd.concat([test_features, pca_genes_test, pca_cells_test], axis = 1)
 PCA')
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
mask =(train_features[cols_numeric].var() >= variance_threshould ).values
tmp = train_features[cols_numeric].loc[:, mask]
train_features = pd.concat([train_features[["sig_id", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
test_features = pd.concat([test_features[["sig_id", "cp_time", "cp_dose"]], test_features.loc[:,cols_numeric]], axis = 1)
 train_features = pd.get_dummies(train_features, columns = ['cp_time', 'cp_dose'])
 test_features = pd.get_dummies(test_features, columns = ['cp_time', 'cp_dose'])
 Feature selection')
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 test = test_features
Join')
 target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 num_targets: {}'.format(num_targets))
')
 return train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols
del([data_train, data_test, model])
 gc.collect()<define_variables>
train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols = preprocessor_nn_transfer()
cols_svd = ['MGR_ID','RESOURCE_MGR_ID','RESOURCE_ROLE_CODE',
'RESOURCE_ROLE_DEPTNAME','RESOURCE_ROLE_FAMILY_DESC','RESOURCE_ROLE_FAMILY',
'RESOURCE_ROLE_ROLLUP_1','RESOURCE','ROLE_CODE',
'ROLE_DEPTNAME_ROLE_CODE','ROLE_DEPTNAME_ROLE_FAMILY','ROLE_FAMILY_DESC_ROLE_CODE',
'ROLE_FAMILY_DESC_ROLE_FAMILY','ROLE_FAMILY_DESC','ROLE_ROLLUP_1_ROLE_DEPTNAME',
'ROLE_ROLLUP_1_ROLE_FAMILY','ROLE_ROLLUP_1_ROLE_ROLLUP_2','ROLE_ROLLUP_2_ROLE_FAMILY_DESC',
'ROLE_ROLLUP_2_ROLE_FAMILY','ROLE_ROLLUP_2']
cols_rnd = ['MGR_ID_ROLE_CODE','MGR_ID_ROLE_DEPTNAME','MGR_ID_ROLE_FAMILY_DESC',
'MGR_ID_ROLE_ROLLUP_1','MGR_ID','RESOURCE_ROLE_DEPTNAME',
'RESOURCE_ROLE_FAMILY','RESOURCE_ROLE_ROLLUP_1','ROLE_CODE',
'ROLE_DEPTNAME_ROLE_FAMILY_DESC','ROLE_FAMILY_DESC_ROLE_CODE',
'ROLE_FAMILY_DESC_ROLE_FAMILY','ROLE_FAMILY_ROLE_CODE',
'ROLE_ROLLUP_1_ROLE_CODE','ROLE_ROLLUP_1_ROLE_FAMILY_DESC',
'ROLE_ROLLUP_1_ROLE_ROLLUP_2']
cols_freq = ['MGR_ID_ROLE_CODE','MGR_ID_ROLE_DEPTNAME','MGR_ID_ROLE_ROLLUP_1',
'MGR_ID_ROLE_ROLLUP_2','MGR_ID','RESOURCE_MGR_ID',
'RESOURCE_ROLE_DEPTNAME','RESOURCE_ROLE_FAMILY_DESC','RESOURCE_ROLE_ROLLUP_2',
'RESOURCE','ROLE_DEPTNAME_ROLE_FAMILY_DESC','ROLE_DEPTNAME',
'ROLE_FAMILY_DESC','ROLE_FAMILY','ROLE_ROLLUP_1_ROLE_FAMILY_DESC',
'ROLE_ROLLUP_1_ROLE_FAMILY','ROLE_ROLLUP_1_ROLE_ROLLUP_2',
'ROLE_ROLLUP_1','ROLE_ROLLUP_2_ROLE_CODE','ROLE_ROLLUP_2']
cols_te = ['MGR_ID','RESOURCE_MGR_ID','RESOURCE_ROLE_CODE',
'RESOURCE_ROLE_DEPTNAME','RESOURCE_ROLE_ROLLUP_2','RESOURCE',
'ROLE_CODE','ROLE_DEPTNAME_ROLE_FAMILY_DESC','ROLE_DEPTNAME_ROLE_FAMILY',
'ROLE_FAMILY_DESC_ROLE_CODE','ROLE_FAMILY_DESC','ROLE_FAMILY_ROLE_CODE',
'ROLE_ROLLUP_1_ROLE_FAMILY','ROLE_ROLLUP_2_ROLE_FAMILY','ROLE_ROLLUP_2']
data_svd = transform_dataset(train[cols_svd], test[cols_svd], get_col_interactions_svd)
 data_rnd = transform_dataset(train[cols_rnd], test[cols_rnd],
assign_rnd_integer, {"number_of_times":5})
 data_freq = transform_dataset(train[cols_freq], test[cols_freq], get_freq_encoding )<categorify>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
te = TargetEncoding(columns_names = cols_te)
 data_te_tr = get_CV_target_encoding(train[cols_te], y, te, 5)
 te.fit(train[cols_te], y)
 data_te_te = te.transform(test[cols_te] )<concatenate>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
feature_cols = [c for c in train.columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 model = Model(num_features, num_all_targets)
 def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 display(train.head())
 def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = train
test_ = test
kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/nn-transfer-learning-oof/SCORED_ONLY_SEED_{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 print(f"Pretrained_Model_Loaded_SCORED_ONLYSEED: {seed_id}, FOLD: {fold_id} ")
 return 0.02, predictions
data_train = pd.concat([x[0] for x in [data_svd, data_rnd, data_freq]], axis = 1)
 data_test = pd.concat([x[1] for x in [data_svd, data_rnd, data_freq]], axis = 1)
 data_train = pd.concat([data_train, data_te_tr], axis = 1)
 data_test = pd.concat([data_test, data_te_te], axis = 1)
 print("Dataset shape, Train: {}, Test: {}".format(data_train.shape, data_test.shape))
del([data_svd, data_rnd, data_freq, data_te_tr, data_te_te])
 gc.collect()
data_train = data_train.values
data_test = data_test.values
gc.collect()<train_model>
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
str(timedelta(seconds=time_diff))
data_dir = '.. /input/lish-moa/'
train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_scored.head()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1])
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 sub_nn_transfer = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub_nn_transfer.to_csv('submission_nn_transfer.csv', index=False )
params = {'colsample_bytree': 0.5280533549534434,
'lambda_l1': 0.1267270702844549,
'learning_rate': 0.012220447574715732,
'max_bin': 131,
'max_depth': 18,
'min_child_weight': 1.1518716916679328,
'num_leaves': 184}
predictions_2 = []
for s in seeds:
model = get_model(params, random_state = s)
 model.fit(data_train, y)
 predictions_2.append(model.predict_proba(data_test)[:,1].ravel())
 predictions_2 = np.stack(predictions_2, -1 ).mean(-1)
 del([data_train, data_test, model])
 gc.collect()<create_dataframe>
saved_path_name = '.. /input/nn-transfer-learning-oof/'
oof_nn_transfer = np.load(saved_path_name + 'oof_nn_transfer_all.npy')
 oof_nn_transfer.shape
submission = pd.DataFrame()
submission["Id"] = test["id"]
submission["ACTION"] =(predictions_1 + predictions_2)/ 2<save_to_csv>
def readin_data_resnet() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets_drug = train_targets_scored.merge(train_drug, on='sig_id')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
targets_drug = targets_drug.iloc[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/ttestpcarfelogisticregression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return train_features,labels_train,test_features,targets_drug,predictors,cs,gs
submission.to_csv("submission.csv", index = False )<set_options>
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
def resnet_preprocessor(train,test,cs,gs):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test
%matplotlib inline
split_char = '/'<load_from_csv>
def resnet_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.25),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model
breeds = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 colors = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 states = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv')
 train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv' )<feature_engineering>
def resnet_make_cv_folds(targets_drug,target_cols, SEEDS, NFOLDS, DRUG_THRESH):
vc = targets_drug.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = targets_drug.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = targets_drug.loc[targets_drug.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 targets_drug[kfold_col] = targets_drug.drug_id.map(dct1)
 targets_drug.loc[targets_drug[kfold_col].isna() , kfold_col] = targets_drug.loc[targets_drug[kfold_col].isna() , 'sig_id'].map(dct2)
 targets_drug[kfold_col] = targets_drug[kfold_col].astype('int8')
 return targets_drug
train['Age'] = train['Age'].fillna(method='pad')
 train.loc[train['Age'] > 20, 'Age'] = train['Age'].mean()<define_variables>
def predict_resnet(train_features,test_features,labels_train, targets_drug1,SEEDS, NFOLDS,cs,gs):
n_labels = labels_train.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
n_seeds = len(SEEDS)
 n_folds = NFOLDS
y_pred = np.zeros(( n_test,n_labels))
oof = np.zeros(( n_train,n_labels))
train_all,data_test = resnet_preprocessor(train_features.values,
test_features.drop('cp_type',axis=1 ).values,cs,gs)
 for seed in SEEDS:
kfold_col = 'kfold_{}'.format(seed)
 kf = targets_drug1[kfold_col]
for fold in range(NFOLDS):
train = targets_drug1[kfold_col] != fold
test = targets_drug1[kfold_col] == fold
X_train, X_test = train_all[train],train_all[test]
X_train_2 = train_features[train][predictors].values
X_test_2 = train_features[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = keras.models.load_model('/kaggle/input/multi-input-resnet-model/TwoHeads_seed_'+str(seed)+'_fold_'+str(fold), custom_objects={'logloss':logloss})
 y_val = model.predict([X_test,X_test_2])
 oof[test] += y_val/ n_seeds
y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 return oof,y_pred
breed_dict = dict(zip(breeds['BreedID'], breeds['BreedName']))<define_variables>
train_features,labels_train,test_features,targets_drug,predictors,cs,gs = readin_data_resnet()
p_min = 0.0005
p_max = 0.9995
DRUG_THRESH = 18
n_seeds = 7
np.random.seed(42)
 SEEDS = np.random.randint(0,100,size=n_seeds)
 NFOLDS = 10
target_cols = targets_drug.drop(['sig_id','drug_id'], axis=1 ).columns.values.tolist()
targets_drug_copy = targets_drug.copy()
targets_drug1 = resnet_make_cv_folds(targets_drug_copy,target_cols, SEEDS, NFOLDS, DRUG_THRESH)
 oof_rs, test_pred_rs = predict_resnet(train_features,test_features,labels_train, targets_drug1,SEEDS, NFOLDS,cs,gs )
animal_dict = {1: 'Dog', 2: 'Cat'}<categorify>
sub_rs = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_rs.iloc[:,1:] = np.clip(test_pred_rs,p_min,p_max)
 sub_rs.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_rs.to_csv('submission_rs.csv', index=False )
encodedColor1 = pd.get_dummies(train['Color1'], prefix="color")
 train = pd.concat([train, encodedColor1], axis='columns')
 encodedColor2 = pd.get_dummies(test['Color1'], prefix="color")
 test = pd.concat([test, encodedColor2], axis='columns' )<define_variables>
def log_loss_numpy(y_pred):
y_true_ravel = np.asarray(y_true ).ravel()
y_pred = np.asarray(y_pred ).ravel()
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 loss = np.where(y_true_ravel == 1, - np.log(y_pred), - np.log(1 - y_pred))
return loss.mean()
def func_numpy_metric(weights):
oof_blend = np.tensordot(weights, oof, axes =(( 0),(0)))
 return log_loss_numpy(oof_blend)
 def grad_func(weights):
oof_clip = np.clip(oof, 1e-15, 1 - 1e-15)
 gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
@njit
def grad_func_jit(weights):
oof_clip = np.minimum(1 - 1e-15, np.maximum(oof, 1e-15))
gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
img_size = 256
batch_size = 16<define_variables>
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv', index_col = 'sig_id' ).values
oof_dict = {'Model 1': '.. /input/nn-transfer-learning-oof/oof_nn_transfer_all.npy',
'Model 2': '.. /input/tabnet-oof-seed3-fold7-112220/oof_TabNet_all.npy',
'Resnet': '.. /input/prediction/resnet_oof.npy'
}
oof = np.zeros(( len(oof_dict), y_true.shape[0], y_true.shape[1]))
for i in range(oof.shape[0]):
oof[i] = np.load(list(oof_dict.values())[i] )
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1<define_variables>
%%time
log_loss_scores = {}
for n, key in enumerate(oof_dict.keys()):
score_oof = log_loss_numpy(oof[n])
 log_loss_scores[key] = score_oof
print(f'{key} CV:\t', score_oof)
 print('-' * 50 )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):
rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
test_weights = np.array([1 / oof.shape[0]] * oof.shape[0] )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<define_search_model>
tol = 1e-10
init_guess = [1 / oof.shape[0]] * oof.shape[0]
bnds = [(0, 1)for _ in range(oof.shape[0])]
cons = {'type': 'eq',
'fun': lambda x: np.sum(x)- 1,
'jac': lambda x: [1] * len(x)}
print('Inital Blend OOF:', func_numpy_metric(init_guess))
start_time = time()
res_scipy = minimize(fun = func_numpy_metric,
x0 = init_guess,
method = 'SLSQP',
jac = grad_func_jit,
bounds = bnds,
constraints = cons,
tol = tol)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}] Optimised Blend OOF:', res_scipy.fun)
 print('Optimised Weights:', res_scipy.x )
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )<predict_on_test>
print('Check the sum of all weights:', np.sum(res_scipy.x))
if np.sum(res_scipy.x)- 1 <= tol:
print('Great! The sum of all weights equals to 1!')
 else:
print('Manual adjustion is needed to modify the weights.' )
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] =(res_scipy.x[0]*sub_nn_transfer.iloc[:, 1:].values + \
res_scipy.x[1]*submission_TabNet.iloc[:,1:].values + \
res_scipy.x[2]*sub_rs.iloc[:,1:].values)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = ['pic_'+str(i)for i in range(train_feats.shape[1])]<define_variables>
!rm submission_TabNet.csv