Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<save_to_csv>
DO_SUBMISSION = True
DO_TRAIN_FOR_ENSEMBLE = False
DO_VIRTUAL_SUBMISSION = False
assert(sum([DO_SUBMISSION, DO_TRAIN_FOR_ENSEMBLE, DO_VIRTUAL_SUBMISSION])== 1), "select `ONE` mode"
sample_submission['target'] = final_test_preds
sample_submission.to_csv('submission.csv', index=False)
 sample_submission.head(5 )<load_from_csv>
%%time
!pip install --no-index --find-links.. /input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
train = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv")
 sample_submission = pd.read_csv('.. /input/tabular-playground-series-jan-2021/sample_submission.csv' )<count_missing_values>
%%time
!pip install.. /input/iterative-stratification/iterative-stratification-master/
train.isna().sum()<count_missing_values>
warnings.resetwarnings()
warnings.simplefilter('ignore', FutureWarning)
 warnings.simplefilter('ignore', DeprecationWarning)
 os.environ["CUDA_VISIBLE_DEVICES"]="0"
test.isna().sum()<prepare_x_and_y>
def generate_virtual_private(test_feat, smpl_sub, rate_to_pub=4):
assert(test_feat.sig_id == smpl_sub.sig_id ).all()
test_feat_list = [test_feat]
smpl_sub_list = [smpl_sub]
for i in range(1, rate_to_pub):
tmp_test_feat = test_feat.copy()
tmp_smpl_sub = smpl_sub.copy()
tmp_test_feat.sig_id = tmp_test_feat.sig_id + f"_{i}"
tmp_smpl_sub.sig_id = tmp_smpl_sub.sig_id + f"_{i}"
assert(tmp_test_feat.sig_id == tmp_smpl_sub.sig_id ).all()
tmp_test_feat.iloc[:, 4:] += i * 10
assert(tmp_test_feat.iloc[:, 4:] != test_feat.iloc[:, 4:] ).all().all()
test_feat_list.append(tmp_test_feat)
 smpl_sub_list.append(tmp_smpl_sub)
 test_feat_concat = pd.concat(test_feat_list, axis=0, ignore_index=True)
 smpl_sub_concat = pd.concat(smpl_sub_list, axis=0, ignore_index=True)
 return test_feat_concat, smpl_sub_concat
target = train['target'].values<count_unique_values>
MODEL_NAMES = ["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]
np.unique(target ).shape 
<define_variables>
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/nn-inference-0.01833.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-nn-use-train-public-inference.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape<train_model>
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/tabnet-inference-0.01840.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-tabnet-inference-add-param-n-shared.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
NUM_FOLDS = 10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
model = HistGradientBoostingRegressor(validation_fraction=None, max_iter=6500, l2_regularization=0.0, min_samples_leaf=28, max_depth=18, learning_rate = 0.01)
 model.fit(train_df, train_target)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(test[columns])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS<compute_test_metric>
%%time
if DO_SUBMISSION:
!python.. /input/moa-takapy-script/tf-rn-transfer-1layerother-selcol100.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-takapy-script/virtual-tf-rn-transfer-1layerother-selcol100.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
mean_squared_error(train_oof, target, squared=False )<save_model>
%%time
if DO_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/virtual-moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
np.save('train_oof', train_oof)
 np.save('test_preds', test_preds )<save_to_csv>
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
sample_submission['target'] = test_preds
sample_submission.to_csv('submission.csv', index=False )<set_options>
def del_control(df: pd.DataFrame())-> pd.DataFrame() :
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 df['cp_type'] = train['cp_type']
df = df.loc[df['cp_type']=='trt_cp'].reset_index(drop=True)
 df = df.drop('cp_type', axis=1)
 return df
def drop_sig_id(df: pd.DataFrame())-> pd.DataFrame() :
return df.drop('sig_id', axis=1)
 def fix_oof(df: pd.DataFrame())-> pd.DataFrame() :
if df.shape[0] != 21948:
df = del_control(df)
 if 'sig_id' in df.columns.tolist() :
df = drop_sig_id(df)
 return df
plt.style.use('fivethirtyeight')
 y_ = Fore.YELLOW
r_ = Fore.RED
g_ = Fore.GREEN
b_ = Fore.BLUE
m_ = Fore.MAGENTA
c_ = Fore.CYAN
sr_ = Style.RESET_ALL
warnings.filterwarnings('ignore')
<load_from_csv>
oof_list = []
oof_list.append(fix_oof(
 pd.read_pickle('.. /input/nn-use-train-public/oof.pkl')))
 oof_list.append(fix_oof(
 pd.DataFrame(
 np.load('.. /input/tabnet-train-public-add-n-shared-1/oof_tabnet.npy'),
columns=TARGET_COL)) 
)
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/oof_prediction_takapy_resnet.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-seed-cv/oof_prediction.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-drug-seed-cv/oof_prediction.csv")) )
path = '.. /input/tabular-playground-series-jan-2021/'
train_data = pd.read_csv(path + 'train.csv')
 test_data = pd.read_csv(path + 'test.csv')
 sample = pd.read_csv(path + 'sample_submission.csv' )<feature_engineering>
y_true = fix_oof(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
for i, name in enumerate(MODEL_NAMES):
print(f"[{name}]")
 print("shape:", oof_list[i].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[i].values.ravel()))
train_data['cont13_cont4_mul'] = train_data['cont13']*train_data['cont4']
train_data['cont13_cont11_mul'] = train_data['cont13']*train_data['cont11']
train_data['cont13_cont7_mul'] = train_data['cont13']*train_data['cont7']
train_data['cont13_cont2_mul'] = train_data['cont13']*train_data['cont2']
train_data['cont13_cont10_mul'] = train_data['cont13']*train_data['cont10']
test_data['cont13_cont4_mul'] = test_data['cont13']*test_data['cont4']
test_data['cont13_cont11_mul'] = test_data['cont13']*test_data['cont11']
test_data['cont13_cont7_mul'] = test_data['cont13']*test_data['cont7']
test_data['cont13_cont2_mul'] = test_data['cont13']*test_data['cont2']
test_data['cont13_cont10_mul'] = test_data['cont13']*test_data['cont10']<categorify>
class MoAStackingDataset(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
class MoAStackingDatasetForCNN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.reset_model_order()
def reset_model_order(self):
self.model_order = np.arange(self.feat.shape[-1])
 def shuffle_model_order(self, seed):
np.random.seed(seed)
 self.model_order = np.random.permutation(self.model_order)
 def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index][..., self.model_order] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
class MoAStackingDatasetForGCN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
num_bins = int(1 + np.log2(len(train_data)))
 train_data.loc[:,'bins'] = pd.cut(train_data['target'].to_numpy() ,bins=num_bins,labels=False)
 features = [f'cont{x}' for x in range(1,15)]
features += [
'cont13_cont4_mul',
'cont13_cont11_mul',
'cont13_cont7_mul',
'cont13_cont2_mul',
'cont13_cont10_mul',
]
target_feature = 'target'
train_data = train_data.query('target >=5')
 bins = train_data['bins'].to_numpy()
target = train_data[target_feature].to_numpy()
train_data = train_data[features].to_numpy()
test_data = test_data[features].to_numpy()
scaler = PowerTransformer()
train_data = scaler.fit_transform(train_data)
 test_data = scaler.transform(test_data )<compute_test_metric>
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU() ,
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
elif re.match(r"^htanh\_\d{4}$", activ_name):
bound = int(activ_name[-4:])/ 1000
return nn.Hardtanh(-bound, bound)
 else:
raise NotImplementedError
class LBAD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LBAD, self ).__init__()
layers = [nn.Linear(in_features, out_features)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
layers.append(get_activation(activ))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class BDLA(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(BDLA, self ).__init__()
layers = []
if use_bn:
layers.append(nn.BatchNorm1d(in_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
layers.append(nn.Linear(in_features, out_features))
if use_wn:
layers[-1] = nn.utils.weight_norm(layers[-1])
 layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class LABD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LABD, self ).__init__()
layers = [nn.Linear(in_features, out_features), get_activation(activ)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x )
def rmse_score(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))<init_hyperparams>
def vector_wise_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 X = torch.transpose(X, 0, 1)
 W = torch.transpose(W, 1, 2)
 H = torch.matmul(X, W)
 H = torch.transpose(H, 0, 1)
 return H
def vector_wise_shared_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 H = nn.functional.linear(X, W)
 return H
seed = 2021
nfolds = 5
params={
'objective':'regression',
'metrics':'rmse',
'boosting':'gbdt',
'min_data_per_group': 5,
'num_leaves': 256,
'max_depth': -1,
'learning_rate': 0.005,
'subsample_for_bin': 200000,
'lambda_l1': 1.074622455507616e-05,
'lambda_l2': 2.0521330798729704e-06,
'n_jobs': -1,
'cat_smooth': 1.0,
'silent': True,
'importance_type': 'gain',
'feature_pre_filter': False,
'bagging_fraction': 0.8206341150202605,
'min_data_in_leaf': 100,
'min_sum_hessian_in_leaf': 0.001,
'bagging_freq': 6,
'feature_fraction': 0.5,
'min_gain_to_split': 0.0,
'min_child_samples': 20}<train_model>
def _calculate_fan_in_and_fan_out_for_vwl(tensor)-> tp.Tuple[int]:

 dimensions = tensor.dim()
if dimensions < 2:
raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions")
 fan_in = tensor.size(-1)
 fan_out = tensor.size(-2)
 return fan_in, fan_out
def _calculate_correct_fan_for_vwl(tensor, mode)-> int:

 mode = mode.lower()
valid_modes = ['fan_in', 'fan_out']
if mode not in valid_modes:
raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes))
fan_in, fan_out = _calculate_fan_in_and_fan_out_for_vwl(tensor)
 return fan_in if mode == 'fan_in' else fan_out
def kaiming_uniform_for_vwl(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'):

 fan = _calculate_correct_fan_for_vwl(tensor, mode)
 gain = nn.init.calculate_gain(nonlinearity, a)
 std = gain / np.sqrt(fan)
 bound = np.sqrt(3.0)* std
with torch.no_grad() :
return tensor.uniform_(-bound, bound )
lgbm_preds = np.zeros(test_data.shape[0])
 kfold = StratifiedKFold(n_splits=nfolds,random_state=seed)
 lgbm_scores = list()
for train_idx, valid_idx in kfold.split(X=train_data,y=bins):
lgb_train = lgb.Dataset(train_data[train_idx],target[train_idx])
 lgb_valid = lgb.Dataset(train_data[valid_idx],target[valid_idx],reference=lgb_train)
 lgb_model = lgb.train(params,
lgb_train,
valid_sets=[lgb_train,lgb_valid],
num_boost_round=10000,
verbose_eval=200,
early_stopping_rounds=100, 
)
 lgbm_scores.append(rmse_score(target[valid_idx],lgb_model.predict(train_data[valid_idx])))
 lgbm_preds += lgb_model.predict(test_data)/nfolds
print("mean rmse score",np.mean(lgbm_scores))<init_hyperparams>
class VectorWiseLinear(nn.Module):

 def __init__(
 self,
in_features: int, out_features: int, n_vectors: int,
bias: bool=True, weight_shared: bool=True 
)-> None:

 super(VectorWiseLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
self.n_vectors = n_vectors
self.weight_shared = weight_shared
if self.weight_shared:
self.weight = nn.Parameter(
 torch.Tensor(self.out_features, self.in_features))
self.matmul_func = vector_wise_shared_matmul
else:
self.weight = nn.Parameter(
 torch.Tensor(self.n_vectors, self.out_features, self.in_features))
self.matmul_func = vector_wise_matmul
if bias:
self.bias = nn.Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 self.reset_parameters()
def reset_parameters(self)-> None:

 kaiming_uniform_for_vwl(self.weight, a=np.sqrt(5))
if self.bias is not None:
fan_in, _ = _calculate_fan_in_and_fan_out_for_vwl(self.weight)
 bound = 1 / np.sqrt(fan_in)
 nn.init.uniform_(self.bias, -bound, bound)
 def forward(self, X: torch.Tensor)-> torch.Tensor:

 H = self.matmul_func(X, self.weight)
 if self.bias is not None:
H = H + self.bias
return H
params = {'lambda': 0.0030282073258141168,
'alpha': 0.01563845128469084,
'colsample_bytree': 0.55,
'subsample': 0.7,
'learning_rate': 0.01,
'max_depth': 15,
'random_state': 2020,
'min_child_weight': 257,
}<train_model>
class GraphConv(nn.Module):

 def __init__(
 self,
in_channels: int, out_channels: int, n_nodes: int, shrare_msg: bool=True,
model_self: bool=True, share_model_self: bool=True,
bias: bool=True, share_bias: bool=True 
)-> None:

 self.in_channels = in_channels
self.out_channels = out_channels
self.n_nodes = n_nodes
self.model_self = model_self
super(GraphConv, self ).__init__()
self.msg = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, shrare_msg)
 if model_self:
self.model_self = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, share_model_self)
 if bias:
if share_bias:
self.bias = nn.Parameter(torch.Tensor(out_channels))
else:
self.bias = nn.Parameter(torch.Tensor(n_nodes, out_channels))
bound = 1 / np.sqrt(out_channels)
 nn.init.uniform_(self.bias, -bound, bound)
 else:
self.register_parameter('bias', None)
 def forward(self, X: torch.Tensor, A: torch.Tensor, W: torch.Tensor=None)-> torch.Tensor:

 if W is not None:
A = A * W
M = X
M = self.msg(M)
 M = torch.matmul(A, M)
 H = M
if self.model_self:
H = H + self.model_self(X)
 if self.bias is not None:
H = H + self.bias
return H
xgb_preds = np.zeros(test_data.shape[0])
 kfold = StratifiedKFold(n_splits=nfolds,random_state=seed)
 params['random_state'] = seed
xgb_scores = list()
for train_idx, valid_idx in kfold.split(X=train_data,y=bins):
xgb_train = xgb.DMatrix(train_data[train_idx],label=target[train_idx])
 xgb_valid = xgb.DMatrix(train_data[valid_idx],label=target[valid_idx])
 xgb_model = xgb.train(params,
xgb_train,
10000,
verbose_eval=200,
evals=[(xgb_train,'train'),(xgb_valid,'valid')],
early_stopping_rounds=100)
 xgb_scores.append(rmse_score(target[valid_idx],xgb_model.predict(xgb.DMatrix(train_data[valid_idx]))))
xgb_preds += xgb_model.predict(xgb.DMatrix(test_data)) /nfolds
print("mean rmse score",np.mean(xgb_scores))<init_hyperparams>
class MLP(nn.Module):

 def __init__(
 self, n_features_list: tp.List[int], use_tail_as_out: bool=False,
drop_rate: float=0.0, use_bn: bool=False, use_wn: bool=False,
activ:str="relu", block_name: str="LBAD" 
):

 super(MLP, self ).__init__()
n_layers = len(n_features_list)- 1
block_class = {
"LBAD": LBAD, "BDLA": BDLA, "LABD": LABD}[block_name]
layers = []
for i in range(n_layers):
in_feats, out_feats = n_features_list[i: i + 2]
if i == n_layers - 1 and use_tail_as_out:
if block_name in ["BDLA"]:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, "identity")
 else:
layer = nn.Linear(in_feats, out_feats)
 if use_wn:
layer = nn.utils.weight_norm(layer)
 else:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, activ)
 layers.append(layer)
 self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class CNNStacking1d(nn.Module):

 def __init__(
 self, n_models: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking1d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv1d(
 in_ch, out_ch, kernel_size=3, stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class CNNStacking2d(nn.Module):

 def __init__(
 self, n_models: int, n_classes: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking2d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv2d(
 in_ch, out_ch, kernel_size=(1, 3), stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class GCNStacking(nn.Module):

 def __init__(
 self, n_classes: int,
n_channels_list: tp.List[int],
add_self_loop: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(GCNStacking, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = GraphConv(
 in_ch, out_ch, n_classes,
shrare_msg=False, share_model_self=False, share_bias=False)
 setattr(self, "conv{}".format(i + 1), layer)
 self.relu = nn.ReLU(inplace=True)
 if add_self_loop:
adj_mat = torch.ones(n_classes, n_classes)/ n_classes
else:
adj_mat =(1 - torch.eye(n_classes, n_classes)) /(n_classes - 1)
 self.register_buffer("A", adj_mat.float())
 kwargs_head["n_features_list"][0] = n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, X: torch.FloatTensor)-> torch.Tensor:

 bs, n_classes = X.shape[:2]
H = X
for i in range(self.n_conv_layers):
H = getattr(self, "conv{}".format(i + 1))(H, self.A[None,...])
 H = self.relu(H)
 h = torch.reshape(H,(bs, -1))
h = self.head(h)
 return h
params = {'l2_leaf_reg': 0.02247766515106271,
'max_bin': 364,
'subsample': 0.6708650091202213,
'learning_rate': 0.010290546311954876,
'max_depth': 10,
'verbose':200,
'random_state': seed,
'min_data_in_leaf': 300,
'loss_function': 'RMSE',
'n_estimators': 25000,
'rsm':0.5,
'early_stopping_rounds':100}<find_best_model_class>
class EvalFuncManager(nn.Module):

 def __init__(
 self, iters_per_epoch: int,
evalfunc_dict: tp.Dict[str, nn.Module], prefix: str = "val" 
)-> None:

 super(EvalFuncManager, self ).__init__()
self.tmp_iter = 0
self.iters_per_epoch = iters_per_epoch
self.prefix = prefix
self.metric_names = []
for k, v in evalfunc_dict.items() :
setattr(self, k, v)
 self.metric_names.append(k)
 self.reset()
def reset(self)-> None:

 self.tmp_iter = 0
for name in self.metric_names:
getattr(self, name ).reset()
def __call__(self, y: torch.Tensor, t: torch.Tensor)-> None:

 for name in self.metric_names:
getattr(self, name ).update(y, t)
 self.tmp_iter += 1
if self.tmp_iter == self.iters_per_epoch:
ppe.reporting.report({
"{}/{}".format(self.prefix, name): getattr(self, name ).compute()
for name in self.metric_names})
 self.reset()
class MeanLoss(nn.Module):
def __init__(self):
super(MeanLoss, self ).__init__()
self.loss_sum = 0
self.n_examples = 0
def forward(self, y: torch.Tensor, t: torch.Tensor):

 return self.loss_func(y, t)
 def reset(self):

 self.loss_sum = 0
self.n_examples = 0
def update(self, y: torch.Tensor, t: torch.Tensor):

 self.loss_sum += self(y, t ).item() * y.shape[0]
self.n_examples += y.shape[0]
def compute(self):

 return self.loss_sum / self.n_examples
class MyLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLogLoss, self ).__init__()
self.loss_func = nn.BCEWithLogitsLoss(**params)
 class LSBCEWithLogitsLoss(nn.Module):

 def __init__(self, k: int, alpha: float=0.01):

 super(LSBCEWithLogitsLoss, self ).__init__()
self.k = k
self.alpha = alpha
self.loss_func = nn.BCEWithLogitsLoss()
def forward(self, y, t):

 t_s = t *(1 - self.alpha)+ self.alpha / self.k
loss = self.loss_func(y, t_s)
 return loss
class MyLSLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLSLogLoss, self ).__init__()
self.loss_func = LSBCEWithLogitsLoss(**params )
cat_preds = np.zeros(test_data.shape[0])
 kfold = StratifiedKFold(n_splits=nfolds,random_state=seed)
 params['random_state'] = seed
cat_scores = list()
for train_idx, valid_idx in kfold.split(X=train_data,y=bins):
cat_train = Pool(train_data[train_idx],target[train_idx])
 cat_valid = Pool(train_data[valid_idx],target[valid_idx])
 cat_model = CatBoost(params)
 cat_model.fit(cat_train,eval_set=cat_valid)
 cat_scores.append(rmse_score(target[valid_idx],cat_model.predict(train_data[valid_idx])))
 cat_preds += cat_model.predict(test_data)/nfolds
print('mean rmse score:',np.mean(cat_scores))<load_from_csv>
def run_train_loop(
 manager, args, model, device,
train_loader, optimizer, scheduler, loss_func 
):

 while not manager.stop_trigger:
model.train()
for batch in train_loader:
x, t = batch
with manager.run_iteration() :
optimizer.zero_grad()
y = model(x.to(device))
loss = loss_func(y, t.to(device))
ppe.reporting.report({'train/loss': loss.item() })
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
def run_eval(args, model, device, batch, eval_manager):

 model.eval()
x, t = batch
y = model(x.to(device))
eval_manager(y, t.to(device))
stacking_preds = pd.read_csv('.. /input/tps-simple-stacking/submission.csv')
 stacking_preds = stacking_preds.target.to_numpy()<save_to_csv>
def get_optimizer(settings, model):
optimizer = getattr(torch.optim, settings["optimizer"]["name"] )(
 model.parameters() , **settings["optimizer"]["params"])
 return optimizer
def get_scheduler(settings, train_loader, optimizer):
if settings["scheduler"]["name"] is None:
scheduler = None
else:
if settings["scheduler"]["name"] == "OneCycleLR":
settings["scheduler"]["params"]["epochs"] = settings["globals"]["max_epoch"]
settings["scheduler"]["params"]["steps_per_epoch"] = len(train_loader)
 scheduler = getattr(torch.optim.lr_scheduler, settings["scheduler"]["name"] )(
 optimizer, **settings["scheduler"]["params"])
 return scheduler
def get_loss_function(settings):
if hasattr(nn, settings["loss"]["name"]):
loss_func = getattr(nn, settings["loss"]["name"] )(**settings["loss"]["params"])
 else:
loss_func = eval(settings["loss"]["name"] )(**settings["loss"]["params"])
 return loss_func
def get_manager(
 settings, model, device, train_loader, val_loader, optimizer, eval_manager, output_path 
):
trigger = ppe.training.triggers.EarlyStoppingTrigger(
 check_trigger=(1, 'epoch'), monitor='val/metric', mode="min",
patience=settings["globals"]["patience"], verbose=True,
max_trigger=(settings["globals"]["max_epoch"], 'epoch'))
manager = ppe.training.ExtensionsManager(
 model, optimizer, settings["globals"]["max_epoch"],
iters_per_epoch=len(train_loader), stop_trigger=trigger, out_dir=output_path)
 log_extentions = [
ppe_extensions.observe_lr(optimizer=optimizer),
ppe_extensions.LogReport() ,
ppe_extensions.PlotReport(["train/loss", "val/loss"], 'epoch', filename='loss.png'),
ppe_extensions.PlotReport(["lr"], 'epoch', filename='lr.png'),
ppe_extensions.PrintReport([
"epoch", "iteration", "lr", "train/loss", "val/loss", "val/metric", "elapsed_time"])
 ]
for ext in log_extentions:
manager.extend(ext)
 manager.extend(
 ppe_extensions.Evaluator(
 val_loader, model,
eval_func=lambda *batch: run_eval(settings, model, device, batch, eval_manager)) ,
trigger=(1, "epoch"))
manager.extend(
 ppe_extensions.snapshot(target=model, filename="snapshot_epoch_{.epoch}.pth"),
trigger=ppe.training.triggers.MinValueTrigger(key="val/metric", trigger=(1, 'epoch')))
 return manager
sample.target =(0.6 * lgbm_preds.ravel() + 0.2 * xgb_preds.ravel() + 0.1 * cat_preds.ravel() + 0.1 * stacking_preds)
 sample.to_csv("submission.csv",index=False)
 sample.head()<set_options>
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 if deterministic:
torch.backends.cudnn.deterministic = True
sns.set(font_scale=1.4)
 warnings.filterwarnings("ignore")
<compute_test_metric>
def run_train_one_fold(
 settings, model, train_all_dataset, train_val_index, device, output_path):

 train_dataset = data.Subset(train_all_dataset, train_val_index[0])
 val_dataset = data.Subset(train_all_dataset, train_val_index[1])
 train_loader = data.DataLoader(train_dataset, **settings["loader"]["train"])
 val_loader = data.DataLoader(val_dataset, **settings["loader"]["val"])
 print("train: {}, val: {}".format(len(train_dataset), len(val_dataset)))
 model.to(device)
 optimizer = get_optimizer(settings, model)
 scheduler = get_scheduler(settings, train_loader, optimizer)
 loss_func = get_loss_function(settings)
 loss_func.to(device)
 eval_mgr = EvalFuncManager(len(val_loader), {
"loss": loss_func,
"metric": MyLogLoss() ,
})
 manager = get_manager(
 settings, model, device, train_loader, val_loader,
optimizer, eval_mgr, output_path)
 run_train_loop(
 manager, settings, model, device, train_loader, optimizer, scheduler, loss_func )
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)) )<categorify>
def get_drug_multilabel_index(target_in_drug_df, target_cols_list, vc_num, n_folds, seed):

 vc = target_in_drug_df.drug_id.value_counts()
vc1 = vc.loc[vc <= vc_num].index
vc2 = vc.loc[vc > vc_num].index
dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.groupby('drug_id')[target_cols_list].mean().loc[vc1]
if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.loc[target_in_drug_df.drug_id.isin(vc2)].reset_index(drop = True)
 if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 folds = []
target_in_drug_df['fold'] = target_in_drug_df.drug_id.map(dct1)
 target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'fold'] = target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'sig_id'].map(dct2)
 target_in_drug_df.fold = target_in_drug_df.fold.astype('int8')
 folds.append(target_in_drug_df.fold.values)
 return np.stack(folds)[0]
def drug_multilabel_stratified_kfold(k, target_in_drug_df, target_cols_list, n_folds, seed):

 folds_array = get_drug_multilabel_index(target_in_drug_df, target_cols_list, 18, n_folds, seed)
 train_idx = [i for i, x in enumerate(folds_array)if x != k]
validation_idx = [i for i, x in enumerate(folds_array)if x == k]
return train_idx, validation_idx
def add_pca(train_df, test_df, cols, n_comp=20, fit_test = True, prefix='pca_', fit_test_first=False):
pca = PCA(n_components=n_comp, random_state=42)
 pca_titles = [prefix+'_pca_'+str(x)for x in range(n_comp)]
temp_train = train_df.copy()
temp_test = test_df.copy()
for c in cols:
fv = temp_train[c].mean()
temp_train[c] = temp_train[c].fillna(value=fv)
 fv = temp_test[c].mean()
temp_test[c] = temp_test[c].fillna(value=fv)
 for p in pca_titles:
train_df[p] = 0.0
test_df[p] = 0.0
if fit_test==True:
pca_data = pd.concat([temp_train[cols], temp_test[cols]], axis=0)
 if fit_test_first==True:
pca.fit(pca_data[len(train_df):])
 pca_data = pca.transform(pca_data)
 else:
pca_data = pca.fit_transform(pca_data)
 train_df.loc[:, pca_titles] = pca_data[0:len(train_df)]
test_df.loc[:, pca_titles] = pca_data[len(train_df):]
return train_df, test_df, pca_titles
def split_distributions(train_df, test_df, cols, target, n_comp=20, prefix='gm', add_labels=False):
gm = GaussianMixture(n_components=n_comp,covariance_type='full', tol=0.001, reg_covar=1e-06, max_iter=100, n_init=1,
init_params='kmeans', weights_init=None, means_init=None, precisions_init=None,
random_state=42, warm_start=False, verbose=0, verbose_interval=10)
 temp_train = train_df.copy()
temp_test = test_df.copy()
gm_data = pd.concat([temp_train[[cols]], temp_test[[cols]]], axis=0 ).reset_index(drop=True)
 gm.fit(gm_data)
 gm_data['labels'] = gm.predict(gm_data)
 if add_labels:
gm_titles = [prefix+str(x)for x in range(n_comp)]
train_df[gm_titles]=0
test_df[gm_titles]=0
dummies = pd.get_dummies(gm_data['labels'], prefix=prefix)
 train_df.loc[:, gm_titles] = dummies[0:len(train_df)].values
test_df.loc[:, gm_titles] = dummies[len(train_df):].values
else:
gm_titles = []
train_df[prefix+'_label'] = gm.labels_[0:len(train_df)]
test_df[prefix+'_label'] = gm.labels_[len(train_df):]
means = train_df.groupby([prefix+'_label'])[target].mean()
train_df['menc_'+prefix+'_label'] = train_df[prefix+'_label'].map(means)
 test_df['menc_'+prefix+'_label'] = test_df[prefix+'_label'].map(means)
 gm_titles+=[prefix+'_label', 'menc_'+prefix+'_label']
return train_df, test_df, gm_titles
def assign_fold(df, label_column,fold_column, NFOLDS=5):
skf = StratifiedKFold(n_splits=NFOLDS)
 df[fold_column]=0
f=0
for trn_idx, val_idx in skf.split(df, df[label_column]):
df.loc[val_idx, 'fold']=f
f+=1
df[fold_column].value_counts()
return df
def st_scale(train_df, test_df, cols):
StSc = StandardScaler()
train_df[cols] = StSc.fit_transform(train_df[cols])
 test_df[cols] = StSc.transform(test_df[cols])
 return train_df, test_df
def seed_everything(seed=1234):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed )<load_from_csv>
target_in_drug = pd.merge(
 pd.read_csv(".. /input/lish-moa/train_targets_scored.csv"),
pd.read_csv(".. /input/lish-moa/train_drug.csv"),
how='left', on='sig_id')
 print(target_in_drug.shape)
 target_in_drug_trt = del_control(target_in_drug)
 print(target_in_drug_trt.shape )
PATH = '/kaggle/input/tabular-playground-series-jan-2021/'
train = pd.read_csv(PATH+'train.csv')
 test = pd.read_csv(PATH+'test.csv')
 submission = pd.read_csv(PATH+'sample_submission.csv')
 FT_COLS = [x for x in train.columns if 'cont' in x]
TARGET='target'
print(train.shape)
 train.head(10 )<define_variables>
X = np.concatenate([df.values for df in oof_list], axis=1)
 print(X.shape)
 y = y_true.values
print(y.shape)
 X_2d = np.stack([df.values for df in oof_list], axis=2)
 print(X_2d.shape)
 X_3d = X_2d[:, None,...]
print(X_3d.shape)
 X_node = np.stack([df.values for df in oof_list], axis=2 )
original_feature_dict = {'cont1': 5,
'cont2': 10,
'cont3': 8,
'cont4': 8,
'cont5': 5,
'cont6': 6,
'cont7': 4,
'cont8': 3,
'cont9': 7,
'cont10':3,
'cont11': 4,
'cont12': 2,
'cont13': 3,
'cont14': 5,}
pca_dict = {
'extra_pca_0': 4,
'extra_pca_1': 1,
'extra_pca_2': 1,
'extra_pca_3': 2,
'extra_pca_4': 1,
'extra_pca_5': 1 }
MULTIPLIER = 2.0
original_feature_dict = {a:int(b*MULTIPLIER)for(a,b)in original_feature_dict.items() }
ft_dict = {**original_feature_dict, **pca_dict}
ft_dict<split>
train_all_dataset = MoAStackingDatasetForCNN(X_2d, y)
mixture_title_cols=[]
for f in FT_COLS+pca_titles2:
train, test, titles = split_distributions(train, test,f,TARGET, n_comp=ft_dict[f], prefix=f+'_dim', add_labels=True)
 mixture_title_cols+=titles
mixture_value_cols = []
for count,f in enumerate(FT_COLS+pca_titles2):
for d in range(ft_dict[f]):
t_median = train[f][train[f+'_dim'+str(d)]==1].median()
t_std = train[f][train[f+'_dim'+str(d)]==1].std()
train[f+'_dim'+str(d)+'_dist'] = np.where(train[f+'_dim'+str(d)]==1, 
(train[f] - t_median)/t_std, np.nan)
 test[f+'_dim'+str(d)+'_dist'] = np.where(test[f+'_dim'+str(d)]==1, 
(test[f] - t_median)/t_std, np.nan)
 mixture_value_cols+=[f+'_dim'+str(d)+'_dist']<define_variables>
TRAIN_OUTPUT = Path.cwd().parent / "tmp"
TRAIN_OUTPUT.mkdir(exist_ok=True)
 WORK_DIR = Path.cwd()
print('Total Original Features', len(FT_COLS))
print('Total Submixtures Labels', len(mixture_title_cols))
print('Total Submixtures Values', len(mixture_value_cols))
print('Total Feature Columns', len(FT_COLS)+len(mixture_title_cols)+len(mixture_value_cols))<data_type_conversions>
settings_str =
settings = yaml.safe_load(settings_str )
NAN_VALUE = 0.0
for d in mixture_value_cols:
train[d] = train[d].fillna(value=NAN_VALUE)
 test[d] = test[d].fillna(value=NAN_VALUE )<normalization>
gc.collect()
SCALE = True
if SCALE:
train, test = st_scale(train, test, FT_COLS)
 for f in FT_COLS:
train[f] = np.clip(train[f], -2, 2)
 test[f] = np.clip(test[f], -2, 2)
 SCALE_DISTS = True
if SCALE_DISTS:
for d in mixture_value_cols:
TEMP_MAX = max(np.abs(train[d] ).max() ,
np.abs(test[d] ).max())
 train[d] = train[d] / TEMP_MAX
test[d] = test[d] / TEMP_MAX<feature_engineering>
device = torch.device(settings["globals"]["device"])
 cv_score_df_list = []
if settings["globals"]["fast_commit"]:
settings["globals"]["max_epoch"] = 1
stgs_list = []
for tmp_seed in settings["globals"]["seeds_for_avg"]:
stgs_list.append(deepcopy(settings))
stgs_list[-1]["globals"]["seed"] = tmp_seed
train['outlier_filter'] = np.where(train[TARGET]<4, True, False)
 print('<compute_train_metric>
for m_id, tmp_stgs in enumerate(stgs_list):
n_folds = settings["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
train_val_indexs = [
drug_multilabel_stratified_kfold(
 fold_id, target_in_drug_trt, TARGET_COL, n_folds, tmp_seed)
 for fold_id in range(n_folds)]
train_all_dataset.reset_model_order()
train_all_dataset.shuffle_model_order(tmp_seed)
 print("[model {}'s order]: {}".format(m_id, train_all_dataset.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
set_random_seed(tmp_seed , True)
 model = eval(tmp_stgs["model"]["name"] )(**tmp_stgs["model"]["params"])
 run_train_one_fold(
 tmp_stgs, model, train_all_dataset, train_val_indexs[fold_id], device,
TRAIN_OUTPUT / "fold{}_model{}".format(fold_id, m_id))
del model
gc.collect()
best_eval_list = []
for fold_id in range(n_folds):
output_dir = TRAIN_OUTPUT / "fold{}_model{}".format(fold_id, m_id)
 log_df = pd.read_json(output_dir / "log")
 best_log = log_df.iloc[log_df["val/metric"].idxmin() ][["epoch", "train/loss", "val/loss", "val/metric"]]
best_eval_list.append(best_log)
 best_epoch = int(best_log["epoch"])
 model_path = output_dir / "snapshot_epoch_{}.pth".format(best_epoch)
 cp_path = WORK_DIR / "best_model_fold{}_model{}.pth".format(fold_id, m_id)
 shutil.copy(model_path, cp_path)
 for model_path in output_dir.glob("*.pth"):
model_path.unlink()
cv_score_df = pd.DataFrame(best_eval_list)
 cv_score_df.insert(0, "fold", range(settings["globals"]["n_folds_split"]))
cv_score_df.insert(0, "model", m_id)
 del best_eval_list
cv_score_df_list.append(cv_score_df )
print('CV Benchmark - Random Guess of Median')
 cv_bm = np.sqrt(mse(train[TARGET], np.full(( train[TARGET].shape), train[TARGET].median())))
 cv_bm<train_model>
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )
def run_training(model, train_df, test_df,sample_submission, fold_col,
orig_features, mixture_val_cols,mixture_label_cols, target_col,
benchmark,
outlier_col=None, nn=False, epochs=10,batch_size=32, verbose=False,
dense=70, dout=0.15, dense_reg = 0.000001,act='elu'):
FOLD_VALUES = sorted([x for x in train_df[fold_col].unique() ])
 oof = np.zeros(( len(train_df),))
test_predictions = np.zeros(( len(sample_submission),))
fig,axes=plt.subplots(figsize=(15,6))
axes.set_ylim(0.69,0.75)
 RANDOM_SEEDS = [0,42,100,1000]
for rs in RANDOM_SEEDS:
seed_everything(seed=rs)
 for fold in FOLD_VALUES:
print(' --- ')
 print('running random seed', rs, 'fold', fold)
 if outlier_col:
trn_idx =(train_df[fold_col]!=fold)&(~train_df[outlier_col])
 else:
trn_idx = train_df[fold_col]!=fold
val_idx = train_df[fold_col]==fold
X_train_orig = train_df.loc[trn_idx,orig_features].values
X_train = train_df.loc[trn_idx,mixture_val_cols].values
X_train_mask = train_df.loc[trn_idx,mixture_label_cols].values
y_train = train_df.loc[trn_idx, target_col].values
X_val_orig = train_df.loc[val_idx,orig_features].values
X_val = train_df.loc[val_idx,mixture_val_cols].values
X_val_mask = train_df.loc[val_idx,mixture_label_cols].values
y_val = train_df.loc[val_idx, target_col].values
model = keras_model(orig_features, mixture_val_cols,mixture_label_cols,dout=dout,
dense=dense,act=act, dense_reg = dense_reg, descend_fraction = 0.9)
 K.clear_session()
back_count = 5
epoch_list=list(range(epochs-back_count, epochs))
class CustomModelCheckpoint(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
if epoch in epoch_list:
self.model.save_weights('model_fold_'+str(fold)+'_epoch_'+str(epoch)+'.h5', overwrite=True)
 else:
pass
cbk = CustomModelCheckpoint()
history = model.fit([X_train_orig, X_train,X_train_mask ], y_train, epochs=epochs, batch_size=batch_size,shuffle=True,
validation_data=([X_val_orig, X_val,X_val_mask], y_val), verbose=verbose,
callbacks=[cbk])
 print('Fold Last Epoch Train Error', history.history['root_mean_squared_error'][-1])
 print('Fold Last Epoch Valid Error', history.history['val_root_mean_squared_error'][-1])
 model.save_weights('model_final_'+str(fold)+'.h5')
 sns.lineplot(x=range(epochs), y=history.history['loss'], color='Blue')
 sns.lineplot(x=range(epochs), y=history.history['val_loss'], color='Red')
 val_preds = np.zeros(( len(X_val),))
for e in epoch_list:
model = keras_model(orig_features, mixture_val_cols,mixture_label_cols,dout=dout,
dense=dense,act=act, dense_reg = dense_reg, descend_fraction = 0.9)
 model.load_weights('model_fold_'+str(fold)+'_epoch_'+str(e)+'.h5')
 val_preds += model.predict([X_val_orig, X_val,X_val_mask] ).mean(axis=1)
 test_predictions += model.predict([test_df[orig_features].values,test_df[mixture_val_cols].values,
test_df[mixture_label_cols].values] ).mean(axis=1)
 val_preds = val_preds / len(epoch_list)
 oof[val_idx]+=val_preds
val_error = np.sqrt(mse(y_val, val_preds))
print('Fold multi epoch weight prediction error', val_error)
 oof = oof / len(RANDOM_SEEDS)
 total_val_error = np.sqrt(mse(train_df[target_col], oof))
print('final OOF MSE', total_val_error)
 fold_errors = []
for fold in train_df[fold_col].unique() :
val_idx = train_df[fold_col]==fold
fold_errors+=[np.sqrt(mse(train_df.loc[val_idx, target_col].values, oof[val_idx])) ]
test_predictions = test_predictions/(NFOLDS * len(RANDOM_SEEDS)* len(epoch_list))
return oof, test_predictions, fold_errors<choose_model_class>
if DO_SUBMISSION or DO_VIRTUAL_SUBMISSION:
sub_list = [
pd.read_csv('./submission-sinchir0-nn.csv'),
pd.read_csv('./submission_sinchir0_tabnet.csv'),
pd.read_csv("./submission_takapy_tf-resnet.csv"),
pd.read_csv('./submission_tawara_thrnn_seed_cv.csv'),
pd.read_csv('./submission_tawara_thrnn_drug_seed_cv.csv'),
]
else:
sub_list = [
pd.read_csv('.. /input/nn-use-train-public/submission.csv'),
pd.read_csv('.. /input/tabnet-train-public-add-n-shared-1/submission.csv'),
pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/submission.csv"),
pd.read_csv('.. /input/moa-weight-thrnn-seed-cv/submission.csv'),
pd.read_csv('.. /input/moa-weight-thrnn-drug-seed-cv/submission.csv'),
]
for i, name in enumerate(["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]):
print(f"[{name}]:", sub_list[i].shape )
def keras_model(ft_orig, mixture_values, mixture_labels, n_layer=3,bnorm=True,dout=0.2,
dense=20,act='elu', dense_reg = 0.000001, descend_fraction = 0.9):
input1 = L.Input(shape=(len(ft_orig)) , name='input_orig')
 input1_do = L.Dropout(0.1 )(input1)
 XA = L.Dense(dense, activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg), name='dense_orig' )(input1_do)
 if bnorm:
XA = L.BatchNormalization(name='bn_1' )(XA)
 XA1 = L.Dropout(dout, name='do_orig' )(XA)
 input2 = L.Input(shape=(len(mixture_values)) , name='input_new')
 input3 = L.Input(shape=(len(mixture_labels)) , name='input_new2')
 all_input_combo = tf.keras.layers.Concatenate(axis=1, name='concat_i2' )([input2, input3])
 input2_do = L.Dropout(0.2 )(all_input_combo)
 XB = L.Dense(dense, activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg), name='dense_new' )(input2_do)
 if bnorm:
XB = L.BatchNormalization(name='bn_2' )(XB)
 XB1 = L.Dropout(dout, name='do_new' )(XB)
 all_input_combo = tf.keras.layers.Concatenate(axis=1, name='concat_new' )([XA1, XB1])
 X = L.Dense(int(dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg))(all_input_combo)
 if bnorm:
X = L.BatchNormalization(name='bn_3' )(X)
 X2 = L.Dropout(dout, name='do_3' )(X)
 X2 = tf.keras.layers.concatenate([all_input_combo, X2], axis=1)
 X = L.Dense(int(( descend_fraction**2)*dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg))(X2)
 if bnorm:
X = L.BatchNormalization()(X)
 X3 = L.Dropout(dout )(X)
 X3 = tf.keras.layers.concatenate([X2, X3], axis=1)
 X = L.Dense(int(( descend_fraction**3)*dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg *2))(X3)
 if bnorm:
X = L.BatchNormalization()(X)
 X4 = L.Dropout(dout )(X)
 X4 = tf.keras.layers.concatenate([X3, X4],axis=1)
 X = L.Dense(int(( descend_fraction**4)*dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg*4))(X4)
 if bnorm:
X = L.BatchNormalization()(X)
 X5 = L.Dropout(dout )(X)
 X5 = tf.keras.layers.concatenate([X4, X5], axis=1)
 X = L.Dense(5, activation=act, activity_regularizer=tf.keras.regularizers.L2(0.0001))(X5)
 output1 = L.Dense(1, activation='linear' )(X)
 model = tf.keras.Model(inputs=[input1, input2, input3], outputs=output1)
 model.compile(loss=root_mean_squared_error,
optimizer=tfa.optimizers.AdamW(learning_rate=0.000809028893821181, weight_decay=9.83479875802558E-06),
metrics=tf.keras.metrics.RootMeanSquaredError())
 return model
K.clear_session()
model = keras_model(FT_COLS, mixture_title_cols, mixture_value_cols,
dense=70,
dout=0.15,
dense_reg = 0.000001,
act='elu',
descend_fraction = 0.9)
 model.summary()<train_model>
sub_list = [order_sub(sub_df)for sub_df in sub_list]
oof, test_predictions, fold_errors = run_training(model, train, test,submission, 'fold', FT_COLS,
mixture_value_cols, mixture_title_cols, TARGET, cv_bm,
outlier_col='outlier_filter',
epochs=30,
batch_size=256,
verbose=False,
dense=70,
dout=0.15,
dense_reg = 0.000001,
act='elu', )<save_to_csv>
def inference_function(settings, model, loader, device):
model.to(device)
 model.eval()
pred_list = []
with torch.no_grad() :
for x, t in loader:
y = model(x.to(device))
pred_list.append(y.sigmoid().detach().cpu().numpy())
 pred_arr = np.concatenate(pred_list)
 del pred_list
return pred_arr
print('Save Out of Fold Predictions')
 oof = pd.DataFrame(columns=['oof_prediction'], index=train['id'], data=oof + TARGET_MEAN)
 oof.to_csv('oof_predictions.csv', index=True)
 oof.head(10 )<compute_test_metric>
16 * 206
print('fold errors', fold_errors)
 print('fold error std', np.array(fold_errors ).std() )<save_to_csv>
X_test = np.concatenate([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=1)
 print(X_test.shape)
 X_test_2d = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 print(X_test_2d.shape)
 X_test_3d = X_test_2d[:, None,...]
print(X_test_3d.shape)
 X_test_node = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 test_dataset = MoAStackingDatasetForCNN(X_test_2d, None)
submission['target'] = test_predictions + TARGET_MEAN
submission.to_csv('submission.csv', index=False)
 submission.head(5 )<install_modules>
oof_pred_arr_avg = np.zeros(( len(X), 206))
test_pred_arr_avg = np.zeros(( len(X_test), 206))
for m_id, tmp_stgs in enumerate(stgs_list):
n_folds = tmp_stgs["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
train_val_indexs = [
drug_multilabel_stratified_kfold(
 fold_id, target_in_drug_trt, TARGET_COL, n_folds, tmp_seed)
 for fold_id in range(n_folds)]
oof_pred_arr = np.zeros(( len(X), 206))
test_preds_arr = np.zeros(( n_folds, len(X_test), 206))
train_all_dataset.reset_model_order()
train_all_dataset.shuffle_model_order(tmp_seed)
 test_dataset.model_order = train_all_dataset.model_order
print("[model {}'s order]: {}".format(m_id, train_all_dataset.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
model_path = WORK_DIR / "best_model_fold{}_model{}.pth".format(fold_id, m_id)
 model = eval(tmp_stgs["model"]["name"] )(**tmp_stgs["model"]["params"])
 model.load_state_dict(torch.load(model_path))
val_index = train_val_indexs[fold_id][1]
val_dataset = data.Subset(train_all_dataset, val_index)
 val_loader = data.DataLoader(val_dataset, **tmp_stgs["loader"]["val"])
 val_pred = inference_function(tmp_stgs, model, val_loader, device)
 oof_pred_arr[val_index] = val_pred
del val_dataset; del val_loader; del val_pred
test_loader = data.DataLoader(test_dataset, **tmp_stgs["loader"]["val"])
 test_pred = inference_function(tmp_stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del test_loader; del test_pred;
gc.collect()
oof_pred_arr_avg += oof_pred_arr
test_pred_arr_avg += test_preds_arr.mean(axis=0)
 oof_score = log_loss(y.ravel() , oof_pred_arr.ravel())
 cv_score_df_list[m_id] = cv_score_df_list[m_id].append(
 {"model": m_id, "fold": "oof", "val/metric": oof_score,},
ignore_index=True)
 oof_pred_arr_avg /= len(stgs_list)
 test_pred_arr_avg /= len(stgs_list )
!pip install -q -U git+https://github.com/mljar/mljar-supervised.git@master
!pip install -q -U matplotlib==3.1.3<import_modules>
cv_score_all = pd.concat(cv_score_df_list, axis=0)
 cv_score_all
import pandas as pd
from supervised import AutoML<load_from_csv>
print(log_loss(y.ravel() , oof_pred_arr_avg.ravel()))
train_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 x_cols = [f"cont{i}" for i in range(1,15)]<train_model>
if DO_SUBMISSION or DO_TRAIN_FOR_ENSEMBLE:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 elif DO_VIRTUAL_SUBMISSION:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 print(df_test.shape, submission.shape)
 df_test, submission = generate_virtual_private(df_test, submission)
 print(df_test.shape, submission.shape)
 else:
raise ValueError
automl = AutoML(mode="Explain")
 automl.fit(train_data[x_cols], train_data["target"] )<train_model>
df = submission.copy()
public_id = list(df['sig_id'].values)
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type =='ctl_vehicle'].sig_id] = 0
df_submit.to_csv('submission.csv',index=True )