Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<filter>
DO_SUBMISSION = True
DO_TRAIN_FOR_ENSEMBLE = False
DO_VIRTUAL_SUBMISSION = False
assert(sum([DO_SUBMISSION, DO_TRAIN_FOR_ENSEMBLE, DO_VIRTUAL_SUBMISSION])== 1), "select `ONE` mode"
train_df[train_df.landmark_id == 107382].id.values[1]<define_variables>
%%time
!pip install --no-index --find-links.. /input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
folder_path_number = []
for i in np.arange(0,3):
folder_path_number.append(train_df[train_df.landmark_id == 107382].id.values[0][i])
 folder_path_number<define_variables>
%%time
!pip install.. /input/iterative-stratification/iterative-stratification-master/
nameofimage = train_df[train_df.landmark_id == 107382].id.values[0]
image_path =(f'.. /input/landmark-retrieval-2020/train/{folder_path_number[0]}/{folder_path_number[1]}/{folder_path_number[2]}/{nameofimage}.jpg')
 image_path<define_variables>
warnings.resetwarnings()
warnings.simplefilter('ignore', FutureWarning)
 warnings.simplefilter('ignore', DeprecationWarning)
 os.environ["CUDA_VISIBLE_DEVICES"]="0"
number_image = train_df[train_df.landmark_id == 5724].count().values[1]
length = 8*number_image
fig, axes = plt.subplots(1, number_image, figsize=(length, 6))
for number in np.arange(0,number_image):
folder_path_number = []
for i in np.arange(0,3):
folder_path_number.append(train_df[train_df.landmark_id == 5724].id.values[number][i])
 nameofimage = train_df[train_df.landmark_id == 5724].id.values[number]
image_path =(f'.. /input/landmark-retrieval-2020/train/{folder_path_number[0]}/{folder_path_number[1]}/{folder_path_number[2]}/{nameofimage}.jpg')
 image = cv2.imread(image_path)
 axes[number].imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
axes[number].axis('off')
 axes[number].set_title(f'id of jpg image: {nameofimage}' )<define_variables>
def generate_virtual_private(test_feat, smpl_sub, rate_to_pub=4):
assert(test_feat.sig_id == smpl_sub.sig_id ).all()
test_feat_list = [test_feat]
smpl_sub_list = [smpl_sub]
for i in range(1, rate_to_pub):
tmp_test_feat = test_feat.copy()
tmp_smpl_sub = smpl_sub.copy()
tmp_test_feat.sig_id = tmp_test_feat.sig_id + f"_{i}"
tmp_smpl_sub.sig_id = tmp_smpl_sub.sig_id + f"_{i}"
assert(tmp_test_feat.sig_id == tmp_smpl_sub.sig_id ).all()
tmp_test_feat.iloc[:, 4:] += i * 10
assert(tmp_test_feat.iloc[:, 4:] != test_feat.iloc[:, 4:] ).all().all()
test_feat_list.append(tmp_test_feat)
 smpl_sub_list.append(tmp_smpl_sub)
 test_feat_concat = pd.concat(test_feat_list, axis=0, ignore_index=True)
 smpl_sub_concat = pd.concat(smpl_sub_list, axis=0, ignore_index=True)
 return test_feat_concat, smpl_sub_concat
def get_path(landmark_id):
number_image = train_df[train_df.landmark_id == landmark_id].count().values[1]
image_path = []
for number in np.arange(0,number_image):
folder_path_number = []
for i in np.arange(0,3):
folder_path_number.append(train_df[train_df.landmark_id == landmark_id].id.values[number][i])
 nameofimage = train_df[train_df.landmark_id == landmark_id].id.values[0]
image_path.append(( f'.. /input/landmark-retrieval-2020/train/{folder_path_number[0]}/{folder_path_number[1]}/{folder_path_number[2]}/{nameofimage}.jpg'))
return image_path<count_unique_values>
MODEL_NAMES = ["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]
print('The number of the unique landmark id in the dataset is', train_df.landmark_id.nunique() )<train_on_grid>
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/nn-inference-0.01833.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-nn-use-train-public-inference.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
files = glob.glob(".. /input/landmark-retrieval-2020/train/a/b/c/*.jpg")
 for i in range(10):
im = cv2.imread(files[i])
 print(im.shape )<choose_model_class>
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/tabnet-inference-0.01840.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-tabnet-inference-add-param-n-shared.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
vgg = VGG16(input_shape=(224,224,3), weights=None, include_top=False)
 vgg.load_weights(".. /input/keras-pretrained-models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5")
 input_image = Input(( 224,224,3))
x = vgg(input_image)
 output = GlobalMaxPooling2D()(x)
 model = Model(inputs=[input_image], outputs=[output])
 model.summary()<normalization>
%%time
if DO_SUBMISSION:
!python.. /input/moa-takapy-script/tf-rn-transfer-1layerother-selcol100.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-takapy-script/virtual-tf-rn-transfer-1layerother-selcol100.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
class MyModel(tf.keras.Model):
def __init__(self):
super(MyModel, self ).__init__()
self.model = model
@tf.function(input_signature=[
tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8, name='input_image')
 ])
 def call(self, input_image):
output_tensors = {}
im = tf.image.resize(input_image,(224,224))
im = preprocess_input(im)
 extracted_features = self.model(tf.convert_to_tensor([im], dtype=tf.uint8)) [0]
output_tensors['global_descriptor'] = tf.identity(extracted_features, name='global_descriptor')
 return output_tensors<load_pretrained>
%%time
if DO_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/virtual-moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError
m = MyModel()
served_function = m.call
tf.saved_model.save(
 m, export_dir="./my_model", signatures={'serving_default': served_function} )<load_pretrained>
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
with ZipFile('submission.zip','w')as zip:
zip.write('./my_model/saved_model.pb', arcname='saved_model.pb')
 zip.write('./my_model/variables/variables.data-00000-of-00001', arcname='variables/variables.data-00000-of-00001')
 zip.write('./my_model/variables/variables.index', arcname='variables/variables.index')
<load_pretrained>
def del_control(df: pd.DataFrame())-> pd.DataFrame() :
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 df['cp_type'] = train['cp_type']
df = df.loc[df['cp_type']=='trt_cp'].reset_index(drop=True)
 df = df.drop('cp_type', axis=1)
 return df
def drop_sig_id(df: pd.DataFrame())-> pd.DataFrame() :
return df.drop('sig_id', axis=1)
 def fix_oof(df: pd.DataFrame())-> pd.DataFrame() :
if df.shape[0] != 21948:
df = del_control(df)
 if 'sig_id' in df.columns.tolist() :
df = drop_sig_id(df)
 return df
with ZipFile('submission.zip','w')as zip:
zip.write('.. /input/landmark-retrieval-model/model/saved_model.pb', arcname='saved_model.pb')
 zip.write('.. /input/landmark-retrieval-model/model/variables/variables.data-00000-of-00001', arcname='variables/variables.data-00000-of-00001')
 zip.write('.. /input/landmark-retrieval-model/model/variables/variables.index', arcname='variables/variables.index' )<set_options>
oof_list = []
oof_list.append(fix_oof(
 pd.read_pickle('.. /input/nn-use-train-public/oof.pkl')))
 oof_list.append(fix_oof(
 pd.DataFrame(
 np.load('.. /input/tabnet-train-public-add-n-shared-1/oof_tabnet.npy'),
columns=TARGET_COL)) 
)
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/oof_prediction_takapy_resnet.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-seed-cv/oof_prediction.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-drug-seed-cv/oof_prediction.csv")) )
tf.get_logger().setLevel(logging.ERROR)
 warnings.filterwarnings("ignore")
 gpus = tf.config.experimental.list_physical_devices('GPU')
 num_gpus = len(gpus)
 if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(num_gpus, "Physical GPUs,", len(logical_gpus), "Logical GPUs")
 except RuntimeError as e:
print(e)
 if num_gpus == 0:
strategy = tf.distribute.OneDeviceStrategy(device='CPU')
 print("Setting strategy to OneDeviceStrategy(device='CPU')")
 elif num_gpus == 1:
strategy = tf.distribute.OneDeviceStrategy(device='GPU')
 print("Setting strategy to OneDeviceStrategy(device='GPU')")
 else:
strategy = tf.distribute.MirroredStrategy()
print("Setting strategy to MirroredStrategy() " )<init_hyperparams>
y_true = fix_oof(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
for i, name in enumerate(MODEL_NAMES):
print(f"[{name}]")
 print("shape:", oof_list[i].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[i].values.ravel()))
config = {
'learning_rate': 1e-3,
'momentum': 0.9,
'K': 4,
'margin': 0.3,
'clip_grad': 10.0,
'n_epochs': 4,
'batch_size': 16,
'input_size':(256, 256, 3),
'dense_units': 1024,
'dropout_rate': 0.0,
}<load_from_csv>
class MoAStackingDataset(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
class MoAStackingDatasetForCNN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.reset_model_order()
def reset_model_order(self):
self.model_order = np.arange(self.feat.shape[-1])
 def shuffle_model_order(self, seed):
np.random.seed(seed)
 self.model_order = np.random.permutation(self.model_order)
 def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index][..., self.model_order] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
class MoAStackingDatasetForGCN(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.model_order = None
def reset_model_order(self):

 pass
def shuffle_model_order(self, seed):

 pass
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
def read_df(input_path):
files_paths = glob.glob(input_path + 'train/*/*/*/*')
 mapping = {}
for path in files_paths:
mapping[path.split('/')[-1].split('.')[0]] = path
df = pd.read_csv(input_path + 'train.csv')
 df['path'] = df['id'].map(mapping)
 df =(
 df 
.groupby('landmark_id')['path'] 
.agg(lambda x: ','.join(x)) 
.reset_index() 
)
 return df
df = read_df('.. /input/landmark-retrieval-2020/')
 df.head(10 )<normalization>
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU() ,
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
elif re.match(r"^htanh\_\d{4}$", activ_name):
bound = int(activ_name[-4:])/ 1000
return nn.Hardtanh(-bound, bound)
 else:
raise NotImplementedError
class LBAD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LBAD, self ).__init__()
layers = [nn.Linear(in_features, out_features)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
layers.append(get_activation(activ))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class BDLA(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(BDLA, self ).__init__()
layers = []
if use_bn:
layers.append(nn.BatchNorm1d(in_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
layers.append(nn.Linear(in_features, out_features))
if use_wn:
layers[-1] = nn.utils.weight_norm(layers[-1])
 layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class LABD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LABD, self ).__init__()
layers = [nn.Linear(in_features, out_features), get_activation(activ)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x )
def _get_transform_matrix(rotation, shear, hzoom, wzoom, hshift, wshift):
def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 rotation = math.pi * rotation / 360.
 shear = math.pi * shear / 360.
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 rot_mat = get_3x3_mat([c1, s1, zero ,
-s1, c1, zero ,
zero, zero, one ])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_mat = get_3x3_mat([one, s2, zero ,
zero, c2, zero ,
zero, zero, one ])
 zoom_mat = get_3x3_mat([one/hzoom, zero, zero,
zero, one/wzoom, zero,
zero, zero, one])
 shift_mat = get_3x3_mat([one, zero, hshift,
zero, one, wshift,
zero, zero, one ])
 return tf.matmul(
 tf.matmul(rot_mat, shear_mat),
tf.matmul(zoom_mat, shift_mat)
)
 def _spatial_transform(image,
rotation=3.0,
shear=2.0,
hzoom=8.0,
wzoom=8.0,
hshift=8.0,
wshift=8.0):
ydim = tf.gather(tf.shape(image), 0)
 xdim = tf.gather(tf.shape(image), 1)
 xxdim = xdim % 2
yxdim = ydim % 2
rotation = rotation * tf.random.normal([1], dtype='float32')
 shear = shear * tf.random.normal([1], dtype='float32')
 hzoom = 1.0 + tf.random.normal([1], dtype='float32')/ hzoom
wzoom = 1.0 + tf.random.normal([1], dtype='float32')/ wzoom
hshift = hshift * tf.random.normal([1], dtype='float32')
 wshift = wshift * tf.random.normal([1], dtype='float32')
 m = _get_transform_matrix(
 rotation, shear, hzoom, wzoom, hshift, wshift)
 y = tf.repeat(tf.range(ydim//2, -ydim//2,-1), xdim)
 x = tf.tile(tf.range(-xdim//2, xdim//2), [ydim])
 z = tf.ones([ydim*xdim], dtype='int32')
 idx = tf.stack([y, x, z])
 idx2 = tf.matmul(m, tf.cast(idx, dtype='float32'))
idx2 = tf.cast(idx2, dtype='int32')
 idx2y = tf.clip_by_value(idx2[0,], -ydim//2+yxdim+1, ydim//2)
 idx2x = tf.clip_by_value(idx2[1,], -xdim//2+xxdim+1, xdim//2)
 idx2 = tf.stack([idx2y, idx2x, idx2[2,]])
 idx3 = tf.stack([ydim//2-idx2[0,], xdim//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
image = tf.reshape(d, [ydim, xdim, 3])
 return image
def _pixel_transform(image,
saturation_delta=0.3,
contrast_delta=0.1,
brightness_delta=0.2):
image = tf.image.random_saturation(
 image, 1-saturation_delta, 1+saturation_delta)
 image = tf.image.random_contrast(
 image, 1-contrast_delta, 1+contrast_delta)
 image = tf.image.random_brightness(
 image, brightness_delta)
 return image
def _random_fliplr(image, p=0.25):
r = tf.random.uniform(())
mirror_cond = tf.math.less(r, p)
 image = tf.cond(
 mirror_cond,
lambda: tf.reverse(image, [1]),
lambda: image 
)
 return image
def preprocess_input(image, target_size, augment=False):
image = tf.image.resize(
 image, target_size, method='bilinear')
 image = tf.cast(image, tf.uint8)
 if augment:
image = _spatial_transform(image)
 image = _random_fliplr(image)
 image = _pixel_transform(image)
 image = tf.cast(image, tf.float32)
 image /= 255.
 return image
def create_triplet_dataset(df, training, batch_size, input_size, K):
def sample_input(image_paths, label, K):
image_paths = tf.strings.split(image_paths, sep=',')
 labels = tf.tile([label], [K,])
 if K-len(image_paths)> 0:
image_paths = tf.random.shuffle(image_paths)
 for i in tf.range(K-len(image_paths)) :
image_paths = tf.concat(
 [image_paths, [tf.gather(image_paths, i)]], axis=0)
 return image_paths, labels
idx = tf.argsort(tf.random.uniform(tf.shape(image_paths)))
 idx = tf.gather(idx, tf.range(K))
image_paths = tf.gather(image_paths, idx)
 return image_paths, labels
def read_image(image_path):
image = tf.io.read_file(image_path)
 return tf.image.decode_jpeg(image, channels=3)
 def reshape(x, y):
x = tf.reshape(x,(-1, *input_size))
y = tf.reshape(y,(-1,))
return x, y
@tf.autograph.experimental.do_not_convert
def nested(x, y):
return(tf.data.Dataset.from_tensor_slices(( x, y)) 
.map(lambda x, y:(read_image(x), y),
tf.data.experimental.AUTOTUNE)
.map(lambda x, y:(preprocess_input(
 x, input_size[:2], True), y),
tf.data.experimental.AUTOTUNE)
.batch(K))
image_paths, labels = df.path, df.index
dataset = tf.data.Dataset.from_tensor_slices(( image_paths, labels))
if training:
dataset = dataset.shuffle(10_000)
 dataset = dataset.map(
 lambda x, y: sample_input(x, y, K), tf.data.experimental.AUTOTUNE)
 dataset = dataset.flat_map(nested)
 dataset = dataset.batch(batch_size)
 dataset = dataset.map(
 lambda x, y: reshape(x, y), tf.data.experimental.AUTOTUNE)
 return dataset<choose_model_class>
def vector_wise_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 X = torch.transpose(X, 0, 1)
 W = torch.transpose(W, 1, 2)
 H = torch.matmul(X, W)
 H = torch.transpose(H, 0, 1)
 return H
def vector_wise_shared_matmul(X: torch.Tensor, W: torch.Tensor)-> torch.Tensor:

 H = nn.functional.linear(X, W)
 return H
def create_model(input_shape, dense_units=512, dropout_rate=0.0):
backbone = tf.keras.applications.ResNet50(
 include_top=False,
input_shape=input_shape,
weights=('.. /input/imagenet-weights/' +
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')
)
 pooling = tf.keras.layers.GlobalAveragePooling2D(name='head/pooling')
 dropout = tf.keras.layers.Dropout(dropout_rate, name='head/dropout')
 dense = tf.keras.layers.Dense(dense_units, name='head/dense', dtype='float32')
 image = tf.keras.layers.Input(input_shape, name='input/image')
 x = backbone(image)
 x = pooling(x)
 x = dropout(x)
 x = dense(x)
 return tf.keras.Model(
 inputs=image, outputs=x)
 class DistributedModel:
def __init__(self,
input_size,
batch_size,
finetuned_weights,
dense_units,
dropout_rate,
margin,
optimizer,
strategy,
mixed_precision,
clip_grad):
self.model = create_model(
 input_shape=input_size,
dense_units=dense_units,
dropout_rate=dropout_rate)
 self.input_size = input_size
if finetuned_weights:
self.model.load_weights(finetuned_weights)
 self.mixed_precision = mixed_precision
self.optimizer = optimizer
self.strategy = strategy
self.clip_grad = clip_grad
self.loss_object = tfa.losses.TripletSemiHardLoss(
 margin=margin,
distance_metric='L2', 
)
 self.mean_loss_train = tf.keras.metrics.Mean()
if self.optimizer and self.mixed_precision:
self.optimizer = \
tf.keras.mixed_precision.experimental.LossScaleOptimizer(
 optimizer, loss_scale='dynamic')
 def _compute_loss(self, labels, embedding):
per_example_loss = self.loss_object(labels, embedding)
 return per_example_loss / strategy.num_replicas_in_sync
def _backprop_loss(self, tape, loss, weights):
gradients = tape.gradient(loss, weights)
 if self.mixed_precision:
gradients = self.optimizer.get_unscaled_gradients(gradients)
 clipped, _ = tf.clip_by_global_norm(gradients, clip_norm=self.clip_grad)
 self.optimizer.apply_gradients(zip(clipped, weights))
def _train_step(self, inputs):
with tf.GradientTape() as tape:
embedding = self.model(inputs[0], training=True)
 loss = self._compute_loss(inputs[1], embedding)
 self.mean_loss_train.update_state(loss)
 if self.mixed_precision:
loss = self.optimizer.get_scaled_loss(loss)
 self._backprop_loss(tape, loss, self.model.trainable_weights)
 @tf.function
def _distributed_train_step(self, dist_inputs):
per_replica_loss = self.strategy.run(self._train_step, args=(dist_inputs,))
return self.strategy.reduce(
 tf.distribute.ReduceOp.SUM, per_replica_loss, axis=None)
 def train(self, train_ds, epochs, save_path):
for epoch in range(epochs):
dist_train_ds = self.strategy.experimental_distribute_dataset(train_ds)
 dist_train_ds = tqdm.tqdm(dist_train_ds)
 for i, inputs in enumerate(dist_train_ds):
loss = self._distributed_train_step(inputs)
 dist_train_ds.set_description(
 "Loss {:.5f}".format(self.mean_loss_train.result().numpy())
)
 if save_path:
self.model.save_weights(save_path)
 self.mean_loss_train.reset_states()<create_dataframe>
def _calculate_fan_in_and_fan_out_for_vwl(tensor)-> tp.Tuple[int]:

 dimensions = tensor.dim()
if dimensions < 2:
raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions")
 fan_in = tensor.size(-1)
 fan_out = tensor.size(-2)
 return fan_in, fan_out
def _calculate_correct_fan_for_vwl(tensor, mode)-> int:

 mode = mode.lower()
valid_modes = ['fan_in', 'fan_out']
if mode not in valid_modes:
raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes))
fan_in, fan_out = _calculate_fan_in_and_fan_out_for_vwl(tensor)
 return fan_in if mode == 'fan_in' else fan_out
def kaiming_uniform_for_vwl(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'):

 fan = _calculate_correct_fan_for_vwl(tensor, mode)
 gain = nn.init.calculate_gain(nonlinearity, a)
 std = gain / np.sqrt(fan)
 bound = np.sqrt(3.0)* std
with torch.no_grad() :
return tensor.uniform_(-bound, bound )
train_ds = create_triplet_dataset(
 df=df,
training=True,
batch_size=config['batch_size'],
input_size=config['input_size'],
K=config['K'] 
)
 with strategy.scope() :
optimizer = tf.keras.optimizers.SGD(
 config['learning_rate'], momentum=config['momentum'])
 dist_model = DistributedModel(
 input_size=config['input_size'],
batch_size=config['batch_size'],
finetuned_weights=None,
dense_units=config['dense_units'],
dropout_rate=config['dropout_rate'],
margin=config['margin'],
optimizer=optimizer,
strategy=strategy,
mixed_precision=False,
clip_grad=config['clip_grad'])
 dist_model.train(
 train_ds=train_ds,
epochs=config['n_epochs'],
save_path='model.h5' )<init_hyperparams>
class VectorWiseLinear(nn.Module):

 def __init__(
 self,
in_features: int, out_features: int, n_vectors: int,
bias: bool=True, weight_shared: bool=True 
)-> None:

 super(VectorWiseLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
self.n_vectors = n_vectors
self.weight_shared = weight_shared
if self.weight_shared:
self.weight = nn.Parameter(
 torch.Tensor(self.out_features, self.in_features))
self.matmul_func = vector_wise_shared_matmul
else:
self.weight = nn.Parameter(
 torch.Tensor(self.n_vectors, self.out_features, self.in_features))
self.matmul_func = vector_wise_matmul
if bias:
self.bias = nn.Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 self.reset_parameters()
def reset_parameters(self)-> None:

 kaiming_uniform_for_vwl(self.weight, a=np.sqrt(5))
if self.bias is not None:
fan_in, _ = _calculate_fan_in_and_fan_out_for_vwl(self.weight)
 bound = 1 / np.sqrt(fan_in)
 nn.init.uniform_(self.bias, -bound, bound)
 def forward(self, X: torch.Tensor)-> torch.Tensor:

 H = self.matmul_func(X, self.weight)
 if self.bias is not None:
H = H + self.bias
return H
newmodel = tf.keras.Model(
 inputs=dist_model.model.get_layer('input/image' ).input,
outputs=dist_model.model.get_layer('head/dense' ).output)
 @tf.function(input_signature=[
tf.TensorSpec(
 shape=[None, None, 3],
dtype=tf.uint8,
name='input_image')
 ])
 def serving(input_image):
input_image = preprocess_input(
 input_image, target_size=config['input_size'][:2])
 outputs = newmodel(input_image[tf.newaxis])
 features = tf.math.l2_normalize(outputs[0])
 return {
'global_descriptor': tf.identity(features, name='global_descriptor')
 }
tf.saved_model.save(
 obj=newmodel,
export_dir='model',
signatures={'serving_default': serving})
 filepaths = []
for dirpath, _, filepath in os.walk('model'):
for fp in filepath:
filepaths.append(os.path.join(dirpath, fp))
with ZipFile('submission.zip', 'w')as zip:
for fp in filepaths:
print(fp, '/'.join(fp.split('/')[1:]))
zip.write(fp, arcname='/'.join(fp.split('/')[1:]))<set_options>
class GraphConv(nn.Module):

 def __init__(
 self,
in_channels: int, out_channels: int, n_nodes: int, shrare_msg: bool=True,
model_self: bool=True, share_model_self: bool=True,
bias: bool=True, share_bias: bool=True 
)-> None:

 self.in_channels = in_channels
self.out_channels = out_channels
self.n_nodes = n_nodes
self.model_self = model_self
super(GraphConv, self ).__init__()
self.msg = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, shrare_msg)
 if model_self:
self.model_self = VectorWiseLinear(
 in_channels, out_channels, n_nodes, False, share_model_self)
 if bias:
if share_bias:
self.bias = nn.Parameter(torch.Tensor(out_channels))
else:
self.bias = nn.Parameter(torch.Tensor(n_nodes, out_channels))
bound = 1 / np.sqrt(out_channels)
 nn.init.uniform_(self.bias, -bound, bound)
 else:
self.register_parameter('bias', None)
 def forward(self, X: torch.Tensor, A: torch.Tensor, W: torch.Tensor=None)-> torch.Tensor:

 if W is not None:
A = A * W
M = X
M = self.msg(M)
 M = torch.matmul(A, M)
 H = M
if self.model_self:
H = H + self.model_self(X)
 if self.bias is not None:
H = H + self.bias
return H
tf.get_logger().setLevel(logging.ERROR)
 warnings.filterwarnings("ignore")
 gpus = tf.config.experimental.list_physical_devices('GPU')
 num_gpus = len(gpus)
 if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(num_gpus, "Physical GPUs,", len(logical_gpus), "Logical GPUs")
 except RuntimeError as e:
print(e)
 if num_gpus == 0:
strategy = tf.distribute.OneDeviceStrategy(device='CPU')
 print("Setting strategy to OneDeviceStrategy(device='CPU')")
 elif num_gpus == 1:
strategy = tf.distribute.OneDeviceStrategy(device='GPU')
 print("Setting strategy to OneDeviceStrategy(device='GPU')")
 else:
strategy = tf.distribute.MirroredStrategy()
print("Setting strategy to MirroredStrategy() " )<init_hyperparams>
class MLP(nn.Module):

 def __init__(
 self, n_features_list: tp.List[int], use_tail_as_out: bool=False,
drop_rate: float=0.0, use_bn: bool=False, use_wn: bool=False,
activ:str="relu", block_name: str="LBAD" 
):

 super(MLP, self ).__init__()
n_layers = len(n_features_list)- 1
block_class = {
"LBAD": LBAD, "BDLA": BDLA, "LABD": LABD}[block_name]
layers = []
for i in range(n_layers):
in_feats, out_feats = n_features_list[i: i + 2]
if i == n_layers - 1 and use_tail_as_out:
if block_name in ["BDLA"]:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, "identity")
 else:
layer = nn.Linear(in_feats, out_feats)
 if use_wn:
layer = nn.utils.weight_norm(layer)
 else:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, activ)
 layers.append(layer)
 self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class CNNStacking1d(nn.Module):

 def __init__(
 self, n_models: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking1d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv1d(
 in_ch, out_ch, kernel_size=3, stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class CNNStacking2d(nn.Module):

 def __init__(
 self, n_models: int, n_classes: int,
n_channels_list: tp.List[int], use_bias: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(CNNStacking2d, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv2d(
 in_ch, out_ch, kernel_size=(1, 3), stride=1, padding=0, bias=use_bias),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h
class GCNStacking(nn.Module):

 def __init__(
 self, n_classes: int,
n_channels_list: tp.List[int],
add_self_loop: bool=False,
kwargs_head: tp.Dict={}, 
):

 super(GCNStacking, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = GraphConv(
 in_ch, out_ch, n_classes,
shrare_msg=False, share_model_self=False, share_bias=False)
 setattr(self, "conv{}".format(i + 1), layer)
 self.relu = nn.ReLU(inplace=True)
 if add_self_loop:
adj_mat = torch.ones(n_classes, n_classes)/ n_classes
else:
adj_mat =(1 - torch.eye(n_classes, n_classes)) /(n_classes - 1)
 self.register_buffer("A", adj_mat.float())
 kwargs_head["n_features_list"][0] = n_classes * n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, X: torch.FloatTensor)-> torch.Tensor:

 bs, n_classes = X.shape[:2]
H = X
for i in range(self.n_conv_layers):
H = getattr(self, "conv{}".format(i + 1))(H, self.A[None,...])
 H = self.relu(H)
 h = torch.reshape(H,(bs, -1))
h = self.head(h)
 return h
config = {
'learning_rate': 1e-3,
'momentum': 0.9,
'K': 4,
'margin': 0.3,
'clip_grad': 10.0,
'n_epochs': 4,
'batch_size': 16,
'input_size':(256, 256, 3),
'dense_units': 1024,
'dropout_rate': 0.0,
}<load_from_csv>
class EvalFuncManager(nn.Module):

 def __init__(
 self, iters_per_epoch: int,
evalfunc_dict: tp.Dict[str, nn.Module], prefix: str = "val" 
)-> None:

 super(EvalFuncManager, self ).__init__()
self.tmp_iter = 0
self.iters_per_epoch = iters_per_epoch
self.prefix = prefix
self.metric_names = []
for k, v in evalfunc_dict.items() :
setattr(self, k, v)
 self.metric_names.append(k)
 self.reset()
def reset(self)-> None:

 self.tmp_iter = 0
for name in self.metric_names:
getattr(self, name ).reset()
def __call__(self, y: torch.Tensor, t: torch.Tensor)-> None:

 for name in self.metric_names:
getattr(self, name ).update(y, t)
 self.tmp_iter += 1
if self.tmp_iter == self.iters_per_epoch:
ppe.reporting.report({
"{}/{}".format(self.prefix, name): getattr(self, name ).compute()
for name in self.metric_names})
 self.reset()
class MeanLoss(nn.Module):
def __init__(self):
super(MeanLoss, self ).__init__()
self.loss_sum = 0
self.n_examples = 0
def forward(self, y: torch.Tensor, t: torch.Tensor):

 return self.loss_func(y, t)
 def reset(self):

 self.loss_sum = 0
self.n_examples = 0
def update(self, y: torch.Tensor, t: torch.Tensor):

 self.loss_sum += self(y, t ).item() * y.shape[0]
self.n_examples += y.shape[0]
def compute(self):

 return self.loss_sum / self.n_examples
class MyLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLogLoss, self ).__init__()
self.loss_func = nn.BCEWithLogitsLoss(**params)
 class LSBCEWithLogitsLoss(nn.Module):

 def __init__(self, k: int, alpha: float=0.01):

 super(LSBCEWithLogitsLoss, self ).__init__()
self.k = k
self.alpha = alpha
self.loss_func = nn.BCEWithLogitsLoss()
def forward(self, y, t):

 t_s = t *(1 - self.alpha)+ self.alpha / self.k
loss = self.loss_func(y, t_s)
 return loss
class MyLSLogLoss(MeanLoss):
def __init__(self, **params):
super(MyLSLogLoss, self ).__init__()
self.loss_func = LSBCEWithLogitsLoss(**params )
def read_df(input_path):
files_paths = glob.glob(input_path + 'train/*/*/*/*')
 mapping = {}
for path in files_paths:
mapping[path.split('/')[-1].split('.')[0]] = path
df = pd.read_csv(input_path + 'train.csv')
 df['path'] = df['id'].map(mapping)
 df =(
 df 
.groupby('landmark_id')['path'] 
.agg(lambda x: ','.join(x)) 
.reset_index() 
)
 return df
df = read_df('.. /input/landmark-retrieval-2020/')
 df.head(10 )<normalization>
def run_train_loop(
 manager, args, model, device,
train_loader, optimizer, scheduler, loss_func 
):

 while not manager.stop_trigger:
model.train()
for batch in train_loader:
x, t = batch
with manager.run_iteration() :
optimizer.zero_grad()
y = model(x.to(device))
loss = loss_func(y, t.to(device))
ppe.reporting.report({'train/loss': loss.item() })
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
def run_eval(args, model, device, batch, eval_manager):

 model.eval()
x, t = batch
y = model(x.to(device))
eval_manager(y, t.to(device))
def _get_transform_matrix(rotation, shear, hzoom, wzoom, hshift, wshift):
def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 rotation = math.pi * rotation / 360.
 shear = math.pi * shear / 360.
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 rot_mat = get_3x3_mat([c1, s1, zero ,
-s1, c1, zero ,
zero, zero, one ])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_mat = get_3x3_mat([one, s2, zero ,
zero, c2, zero ,
zero, zero, one ])
 zoom_mat = get_3x3_mat([one/hzoom, zero, zero,
zero, one/wzoom, zero,
zero, zero, one])
 shift_mat = get_3x3_mat([one, zero, hshift,
zero, one, wshift,
zero, zero, one ])
 return tf.matmul(
 tf.matmul(rot_mat, shear_mat),
tf.matmul(zoom_mat, shift_mat)
)
 def _spatial_transform(image,
rotation=3.0,
shear=2.0,
hzoom=8.0,
wzoom=8.0,
hshift=8.0,
wshift=8.0):
ydim = tf.gather(tf.shape(image), 0)
 xdim = tf.gather(tf.shape(image), 1)
 xxdim = xdim % 2
yxdim = ydim % 2
rotation = rotation * tf.random.normal([1], dtype='float32')
 shear = shear * tf.random.normal([1], dtype='float32')
 hzoom = 1.0 + tf.random.normal([1], dtype='float32')/ hzoom
wzoom = 1.0 + tf.random.normal([1], dtype='float32')/ wzoom
hshift = hshift * tf.random.normal([1], dtype='float32')
 wshift = wshift * tf.random.normal([1], dtype='float32')
 m = _get_transform_matrix(
 rotation, shear, hzoom, wzoom, hshift, wshift)
 y = tf.repeat(tf.range(ydim//2, -ydim//2,-1), xdim)
 x = tf.tile(tf.range(-xdim//2, xdim//2), [ydim])
 z = tf.ones([ydim*xdim], dtype='int32')
 idx = tf.stack([y, x, z])
 idx2 = tf.matmul(m, tf.cast(idx, dtype='float32'))
idx2 = tf.cast(idx2, dtype='int32')
 idx2y = tf.clip_by_value(idx2[0,], -ydim//2+yxdim+1, ydim//2)
 idx2x = tf.clip_by_value(idx2[1,], -xdim//2+xxdim+1, xdim//2)
 idx2 = tf.stack([idx2y, idx2x, idx2[2,]])
 idx3 = tf.stack([ydim//2-idx2[0,], xdim//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
image = tf.reshape(d, [ydim, xdim, 3])
 return image
def _pixel_transform(image,
saturation_delta=0.3,
contrast_delta=0.1,
brightness_delta=0.2):
image = tf.image.random_saturation(
 image, 1-saturation_delta, 1+saturation_delta)
 image = tf.image.random_contrast(
 image, 1-contrast_delta, 1+contrast_delta)
 image = tf.image.random_brightness(
 image, brightness_delta)
 return image
def _random_fliplr(image, p=0.25):
r = tf.random.uniform(())
mirror_cond = tf.math.less(r, p)
 image = tf.cond(
 mirror_cond,
lambda: tf.reverse(image, [1]),
lambda: image 
)
 return image
def preprocess_input(image, target_size, augment=False):
image = tf.image.resize(
 image, target_size, method='bilinear')
 image = tf.cast(image, tf.uint8)
 if augment:
image = _spatial_transform(image)
 image = _random_fliplr(image)
 image = _pixel_transform(image)
 image = tf.cast(image, tf.float32)
 image /= 255.
 return image
def create_triplet_dataset(df, training, batch_size, input_size, K):
def sample_input(image_paths, label, K):
image_paths = tf.strings.split(image_paths, sep=',')
 labels = tf.tile([label], [K,])
 if K-len(image_paths)> 0:
image_paths = tf.random.shuffle(image_paths)
 for i in tf.range(K-len(image_paths)) :
image_paths = tf.concat(
 [image_paths, [tf.gather(image_paths, i)]], axis=0)
 return image_paths, labels
idx = tf.argsort(tf.random.uniform(tf.shape(image_paths)))
 idx = tf.gather(idx, tf.range(K))
image_paths = tf.gather(image_paths, idx)
 return image_paths, labels
def read_image(image_path):
image = tf.io.read_file(image_path)
 return tf.image.decode_jpeg(image, channels=3)
 def reshape(x, y):
x = tf.reshape(x,(-1, *input_size))
y = tf.reshape(y,(-1,))
return x, y
@tf.autograph.experimental.do_not_convert
def nested(x, y):
return(tf.data.Dataset.from_tensor_slices(( x, y)) 
.map(lambda x, y:(read_image(x), y),
tf.data.experimental.AUTOTUNE)
.map(lambda x, y:(preprocess_input(
 x, input_size[:2], True), y),
tf.data.experimental.AUTOTUNE)
.batch(K))
image_paths, labels = df.path, df.index
dataset = tf.data.Dataset.from_tensor_slices(( image_paths, labels))
if training:
dataset = dataset.shuffle(10_000)
 dataset = dataset.map(
 lambda x, y: sample_input(x, y, K), tf.data.experimental.AUTOTUNE)
 dataset = dataset.flat_map(nested)
 dataset = dataset.batch(batch_size)
 dataset = dataset.map(
 lambda x, y: reshape(x, y), tf.data.experimental.AUTOTUNE)
 return dataset<choose_model_class>
def get_optimizer(settings, model):
optimizer = getattr(torch.optim, settings["optimizer"]["name"] )(
 model.parameters() , **settings["optimizer"]["params"])
 return optimizer
def get_scheduler(settings, train_loader, optimizer):
if settings["scheduler"]["name"] is None:
scheduler = None
else:
if settings["scheduler"]["name"] == "OneCycleLR":
settings["scheduler"]["params"]["epochs"] = settings["globals"]["max_epoch"]
settings["scheduler"]["params"]["steps_per_epoch"] = len(train_loader)
 scheduler = getattr(torch.optim.lr_scheduler, settings["scheduler"]["name"] )(
 optimizer, **settings["scheduler"]["params"])
 return scheduler
def get_loss_function(settings):
if hasattr(nn, settings["loss"]["name"]):
loss_func = getattr(nn, settings["loss"]["name"] )(**settings["loss"]["params"])
 else:
loss_func = eval(settings["loss"]["name"] )(**settings["loss"]["params"])
 return loss_func
def get_manager(
 settings, model, device, train_loader, val_loader, optimizer, eval_manager, output_path 
):
trigger = ppe.training.triggers.EarlyStoppingTrigger(
 check_trigger=(1, 'epoch'), monitor='val/metric', mode="min",
patience=settings["globals"]["patience"], verbose=True,
max_trigger=(settings["globals"]["max_epoch"], 'epoch'))
manager = ppe.training.ExtensionsManager(
 model, optimizer, settings["globals"]["max_epoch"],
iters_per_epoch=len(train_loader), stop_trigger=trigger, out_dir=output_path)
 log_extentions = [
ppe_extensions.observe_lr(optimizer=optimizer),
ppe_extensions.LogReport() ,
ppe_extensions.PlotReport(["train/loss", "val/loss"], 'epoch', filename='loss.png'),
ppe_extensions.PlotReport(["lr"], 'epoch', filename='lr.png'),
ppe_extensions.PrintReport([
"epoch", "iteration", "lr", "train/loss", "val/loss", "val/metric", "elapsed_time"])
 ]
for ext in log_extentions:
manager.extend(ext)
 manager.extend(
 ppe_extensions.Evaluator(
 val_loader, model,
eval_func=lambda *batch: run_eval(settings, model, device, batch, eval_manager)) ,
trigger=(1, "epoch"))
manager.extend(
 ppe_extensions.snapshot(target=model, filename="snapshot_epoch_{.epoch}.pth"),
trigger=ppe.training.triggers.MinValueTrigger(key="val/metric", trigger=(1, 'epoch')))
 return manager
def create_model(input_shape, dense_units=512, dropout_rate=0.0):
backbone = tf.keras.applications.ResNet50(
 include_top=False,
input_shape=input_shape,
weights=('.. /input/imagenet-weights/' +
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')
)
 pooling = tf.keras.layers.GlobalAveragePooling2D(name='head/pooling')
 dropout = tf.keras.layers.Dropout(dropout_rate, name='head/dropout')
 dense = tf.keras.layers.Dense(dense_units, name='head/dense', dtype='float32')
 image = tf.keras.layers.Input(input_shape, name='input/image')
 x = backbone(image)
 x = pooling(x)
 x = dropout(x)
 x = dense(x)
 return tf.keras.Model(
 inputs=image, outputs=x)
 class DistributedModel:
def __init__(self,
input_size,
batch_size,
finetuned_weights,
dense_units,
dropout_rate,
margin,
optimizer,
strategy,
mixed_precision,
clip_grad):
self.model = create_model(
 input_shape=input_size,
dense_units=dense_units,
dropout_rate=dropout_rate)
 self.input_size = input_size
if finetuned_weights:
self.model.load_weights(finetuned_weights)
 self.mixed_precision = mixed_precision
self.optimizer = optimizer
self.strategy = strategy
self.clip_grad = clip_grad
self.loss_object = tfa.losses.TripletSemiHardLoss(
 margin=margin,
distance_metric='L2', 
)
 self.mean_loss_train = tf.keras.metrics.Mean()
if self.optimizer and self.mixed_precision:
self.optimizer = \
tf.keras.mixed_precision.experimental.LossScaleOptimizer(
 optimizer, loss_scale='dynamic')
 def _compute_loss(self, labels, embedding):
per_example_loss = self.loss_object(labels, embedding)
 return per_example_loss / strategy.num_replicas_in_sync
def _backprop_loss(self, tape, loss, weights):
gradients = tape.gradient(loss, weights)
 if self.mixed_precision:
gradients = self.optimizer.get_unscaled_gradients(gradients)
 clipped, _ = tf.clip_by_global_norm(gradients, clip_norm=self.clip_grad)
 self.optimizer.apply_gradients(zip(clipped, weights))
def _train_step(self, inputs):
with tf.GradientTape() as tape:
embedding = self.model(inputs[0], training=True)
 loss = self._compute_loss(inputs[1], embedding)
 self.mean_loss_train.update_state(loss)
 if self.mixed_precision:
loss = self.optimizer.get_scaled_loss(loss)
 self._backprop_loss(tape, loss, self.model.trainable_weights)
 @tf.function
def _distributed_train_step(self, dist_inputs):
per_replica_loss = self.strategy.run(self._train_step, args=(dist_inputs,))
return self.strategy.reduce(
 tf.distribute.ReduceOp.SUM, per_replica_loss, axis=None)
 def train(self, train_ds, epochs, save_path):
for epoch in range(epochs):
dist_train_ds = self.strategy.experimental_distribute_dataset(train_ds)
 dist_train_ds = tqdm.tqdm(dist_train_ds)
 for i, inputs in enumerate(dist_train_ds):
loss = self._distributed_train_step(inputs)
 dist_train_ds.set_description(
 "Loss {:.5f}".format(self.mean_loss_train.result().numpy())
)
 if save_path:
self.model.save_weights(save_path)
 self.mean_loss_train.reset_states()<create_dataframe>
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 if deterministic:
torch.backends.cudnn.deterministic = True
train_ds = create_triplet_dataset(
 df=df,
training=True,
batch_size=config['batch_size'],
input_size=config['input_size'],
K=config['K'] 
)
 with strategy.scope() :
optimizer = tf.keras.optimizers.SGD(
 config['learning_rate'], momentum=config['momentum'])
 dist_model = DistributedModel(
 input_size=config['input_size'],
batch_size=config['batch_size'],
finetuned_weights=None,
dense_units=config['dense_units'],
dropout_rate=config['dropout_rate'],
margin=config['margin'],
optimizer=optimizer,
strategy=strategy,
mixed_precision=False,
clip_grad=config['clip_grad'])
 dist_model.train(
 train_ds=train_ds,
epochs=config['n_epochs'],
save_path='model.h5' )<init_hyperparams>
def run_train_one_fold(
 settings, model, train_all_dataset, train_val_index, device, output_path):

 train_dataset = data.Subset(train_all_dataset, train_val_index[0])
 val_dataset = data.Subset(train_all_dataset, train_val_index[1])
 train_loader = data.DataLoader(train_dataset, **settings["loader"]["train"])
 val_loader = data.DataLoader(val_dataset, **settings["loader"]["val"])
 print("train: {}, val: {}".format(len(train_dataset), len(val_dataset)))
 model.to(device)
 optimizer = get_optimizer(settings, model)
 scheduler = get_scheduler(settings, train_loader, optimizer)
 loss_func = get_loss_function(settings)
 loss_func.to(device)
 eval_mgr = EvalFuncManager(len(val_loader), {
"loss": loss_func,
"metric": MyLogLoss() ,
})
 manager = get_manager(
 settings, model, device, train_loader, val_loader,
optimizer, eval_mgr, output_path)
 run_train_loop(
 manager, settings, model, device, train_loader, optimizer, scheduler, loss_func )
newmodel = tf.keras.Model(
 inputs=dist_model.model.get_layer('input/image' ).input,
outputs=dist_model.model.get_layer('head/dense' ).output)
 @tf.function(input_signature=[
tf.TensorSpec(
 shape=[None, None, 3],
dtype=tf.uint8,
name='input_image')
 ])
 def serving(input_image):
input_image = preprocess_input(
 input_image, target_size=config['input_size'][:2])
 outputs = newmodel(input_image[tf.newaxis])
 features = tf.math.l2_normalize(outputs[0])
 return {
'global_descriptor': tf.identity(features, name='global_descriptor')
 }
tf.saved_model.save(
 obj=newmodel,
export_dir='model',
signatures={'serving_default': serving})
 filepaths = []
for dirpath, _, filepath in os.walk('model'):
for fp in filepath:
filepaths.append(os.path.join(dirpath, fp))
with ZipFile('submission.zip', 'w')as zip:
for fp in filepaths:
print(fp, '/'.join(fp.split('/')[1:]))
zip.write(fp, arcname='/'.join(fp.split('/')[1:]))<import_modules>
def get_drug_multilabel_index(target_in_drug_df, target_cols_list, vc_num, n_folds, seed):

 vc = target_in_drug_df.drug_id.value_counts()
vc1 = vc.loc[vc <= vc_num].index
vc2 = vc.loc[vc > vc_num].index
dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.groupby('drug_id')[target_cols_list].mean().loc[vc1]
if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=seed, shuffle=True)
 tmp = target_in_drug_df.loc[target_in_drug_df.drug_id.isin(vc2)].reset_index(drop = True)
 if tmp.shape[0] != 0:
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[target_cols_list])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 folds = []
target_in_drug_df['fold'] = target_in_drug_df.drug_id.map(dct1)
 target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'fold'] = target_in_drug_df.loc[target_in_drug_df.fold.isna() ,'sig_id'].map(dct2)
 target_in_drug_df.fold = target_in_drug_df.fold.astype('int8')
 folds.append(target_in_drug_df.fold.values)
 return np.stack(folds)[0]
def drug_multilabel_stratified_kfold(k, target_in_drug_df, target_cols_list, n_folds, seed):

 folds_array = get_drug_multilabel_index(target_in_drug_df, target_cols_list, 18, n_folds, seed)
 train_idx = [i for i, x in enumerate(folds_array)if x != k]
validation_idx = [i for i, x in enumerate(folds_array)if x == k]
return train_idx, validation_idx
import os
import numpy as np
import pandas as pd
from sklearn import preprocessing
import xgboost as xgb
import gc<load_from_csv>
target_in_drug = pd.merge(
 pd.read_csv(".. /input/lish-moa/train_targets_scored.csv"),
pd.read_csv(".. /input/lish-moa/train_drug.csv"),
how='left', on='sig_id')
 print(target_in_drug.shape)
 target_in_drug_trt = del_control(target_in_drug)
 print(target_in_drug_trt.shape )
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print(train.shape)
 print(test.shape)
 y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test
many_null_cols = [col for col in X_train.columns if X_train[col].isnull().sum() / X_train.shape[0] > 0.96]
many_null_cols_X_test = [col for col in X_test.columns if X_test[col].isnull().sum() / X_test.shape[0] > 0.96]
big_top_value_cols = [col for col in X_train.columns if X_train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.96]
big_top_value_cols_X_test = [col for col in X_test.columns if X_test[col].value_counts(dropna=False, normalize=True ).values[0] > 0.96]
cols_to_drop = list(set(many_null_cols + many_null_cols_X_test + big_top_value_cols + big_top_value_cols_X_test))
len(cols_to_drop)
 print(cols_to_drop)
 X_train = X_train.drop(cols_to_drop, axis=1)
 X_test = X_test.drop(cols_to_drop, axis=1)
 X_train.drop('TransactionDT', axis=1, inplace=True)
 X_test.drop('TransactionDT', axis=1, inplace=True)
 print(X_train.shape)
 print(X_test.shape)
 for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))
X_train = X_train.fillna(-999)
 X_test = X_test.fillna(-999 )<train_model>
X = np.concatenate([df.values for df in oof_list], axis=1)
 print(X.shape)
 y = y_true.values
print(y.shape)
 X_2d = np.stack([df.values for df in oof_list], axis=2)
 print(X_2d.shape)
 X_3d = X_2d[:, None,...]
print(X_3d.shape)
 X_node = np.stack([df.values for df in oof_list], axis=2 )
print("Before OverSampling, counts of label '1': {}".format(sum(y_train==1)))
 print("Before OverSampling, counts of label '0': {}
".format(sum(y_train==0)))
 sm = KMeansSMOTE(random_state=99, sampling_strategy = 0.15, k_neighbors = 10,cluster_balance_threshold = 0.02, n_jobs=4)
 X_train_new, y_train_new = sm.fit_sample(X_train, y_train.ravel())
 X_train_new = pd.DataFrame(X_train_new)
 X_train_new.columns = X_train.columns
y_train_new = pd.DataFrame(y_train_new)
 print('After OverSampling, the shape of X_train_new: {}'.format(X_train_new.shape))
print('After OverSampling, the shape of y_train_new: {}
'.format(y_train_new.shape))<prepare_x_and_y>
train_all_dataset = MoAStackingDataset(X, y)
%%time
EPOCHS = 4
y_preds = np.zeros(sample_submission.shape[0])
 kf = StratifiedKFold(n_splits=EPOCHS, random_state= 99, shuffle=True)
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=17,
learning_rate=0.03,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
del clf
gc.collect()
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 del X_train_new
gc.collect()
sample_submission1a = sample_submission.copy()
sample_submission1a['isFraud'] = y_preds
sample_submission1a.to_csv('xgboost_oversample.csv')
 sample_submission1a['isFraud'].describe()<train_model>
TRAIN_OUTPUT = Path.cwd().parent / "tmp"
TRAIN_OUTPUT.mkdir(exist_ok=True)
 WORK_DIR = Path.cwd()
print("Before OverSampling, counts of label '1': {}".format(sum(y_train==1)))
 print("Before OverSampling, counts of label '0': {}
".format(sum(y_train==0)))
 sm = SMOTE(random_state=99, sampling_strategy = 0.15)
 X_train_new, y_train_new = sm.fit_sample(X_train, y_train.ravel())
 X_train_new = pd.DataFrame(X_train_new)
 X_train_new.columns = X_train.columns
y_train_new = pd.DataFrame(y_train_new)
 print('After OverSampling, the shape of X_train_new: {}'.format(X_train_new.shape))
print('After OverSampling, the shape of y_train_new: {}
'.format(y_train_new.shape))<prepare_x_and_y>
settings_str =
settings = yaml.safe_load(settings_str )
%%time
EPOCHS = 4
y_preds = np.zeros(sample_submission.shape[0])
 kf = StratifiedKFold(n_splits=EPOCHS, random_state= 99, shuffle=True)
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=17,
learning_rate=0.02,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
del clf
gc.collect()
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 del X_train_new
gc.collect()
sample_submission1 = sample_submission.copy()
sample_submission1['isFraud'] = y_preds
sample_submission1.to_csv('xgboost_oversample.csv')
 sample_submission1['isFraud'].describe()<train_model>
gc.collect()
print("Before OverSampling, counts of label '1': {}".format(sum(y_train==1)))
 print("Before OverSampling, counts of label '0': {}
".format(sum(y_train==0)))
 sm = RandomOverSampler(random_state=99, sampling_strategy = 0.12)
 X_train_new, y_train_new = sm.fit_sample(X_train, y_train.ravel())
 X_train_new = pd.DataFrame(X_train_new)
 X_train_new.columns = X_train.columns
y_train_new = pd.DataFrame(y_train_new)
 print('After OverSampling, the shape of X_train_new: {}'.format(X_train_new.shape))
print('After OverSampling, the shape of y_train_new: {}
'.format(y_train_new.shape))<prepare_x_and_y>
device = torch.device(settings["globals"]["device"])
 cv_score_df_list = []
if settings["globals"]["fast_commit"]:
settings["globals"]["max_epoch"] = 1
stgs_list = []
for tmp_seed in settings["globals"]["seeds_for_avg"]:
stgs_list.append(deepcopy(settings))
stgs_list[-1]["globals"]["seed"] = tmp_seed
%%time
EPOCHS = 4
y_preds = np.zeros(sample_submission.shape[0])
 kf = StratifiedKFold(n_splits=EPOCHS, random_state= 99, shuffle=True)
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=11,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999,
min_child_weight=1 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
del clf
gc.collect()
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 del X_train_new
gc.collect()
sample_submission2 = sample_submission.copy()
sample_submission2['isFraud'] = y_preds
sample_submission2.to_csv('xgboost_oversample_random.csv')
 sample_submission2['isFraud'].describe()<train_model>
for m_id, tmp_stgs in enumerate(stgs_list):
n_folds = settings["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
train_val_indexs = [
drug_multilabel_stratified_kfold(
 fold_id, target_in_drug_trt, TARGET_COL, n_folds, tmp_seed)
 for fold_id in range(n_folds)]
train_all_dataset.reset_model_order()
train_all_dataset.shuffle_model_order(tmp_seed)
 print("[model {}'s order]: {}".format(m_id, train_all_dataset.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
set_random_seed(tmp_seed , True)
 model = eval(tmp_stgs["model"]["name"] )(**tmp_stgs["model"]["params"])
 run_train_one_fold(
 tmp_stgs, model, train_all_dataset, train_val_indexs[fold_id], device,
TRAIN_OUTPUT / "fold{}_model{}".format(fold_id, m_id))
del model
gc.collect()
best_eval_list = []
for fold_id in range(n_folds):
output_dir = TRAIN_OUTPUT / "fold{}_model{}".format(fold_id, m_id)
 log_df = pd.read_json(output_dir / "log")
 best_log = log_df.iloc[log_df["val/metric"].idxmin() ][["epoch", "train/loss", "val/loss", "val/metric"]]
best_eval_list.append(best_log)
 best_epoch = int(best_log["epoch"])
 model_path = output_dir / "snapshot_epoch_{}.pth".format(best_epoch)
 cp_path = WORK_DIR / "best_model_fold{}_model{}.pth".format(fold_id, m_id)
 shutil.copy(model_path, cp_path)
 for model_path in output_dir.glob("*.pth"):
model_path.unlink()
cv_score_df = pd.DataFrame(best_eval_list)
 cv_score_df.insert(0, "fold", range(settings["globals"]["n_folds_split"]))
cv_score_df.insert(0, "model", m_id)
 del best_eval_list
cv_score_df_list.append(cv_score_df )
print("Before OverSampling, counts of label '1': {}".format(sum(y_train==1)))
 print("Before OverSampling, counts of label '0': {}
".format(sum(y_train==0)))
 sm = RandomOverSampler(random_state=999, sampling_strategy = 0.13)
 X_train_new, y_train_new = sm.fit_sample(X_train, y_train.ravel())
 X_train_new = pd.DataFrame(X_train_new)
 X_train_new.columns = X_train.columns
y_train_new = pd.DataFrame(y_train_new)
 print('After OverSampling, the shape of X_train_new: {}'.format(X_train_new.shape))
print('After OverSampling, the shape of y_train_new: {}
'.format(y_train_new.shape))<prepare_x_and_y>
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )
%%time
EPOCHS = 4
y_preds = np.zeros(sample_submission.shape[0])
 kf = StratifiedKFold(n_splits=EPOCHS, random_state= 99, shuffle=True)
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=17,
learning_rate=0.035,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
del clf
gc.collect()
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 del X_train_new
gc.collect()
sample_submission3 = sample_submission.copy()
sample_submission3['isFraud'] = y_preds
sample_submission3.to_csv('xgboost_oversample_random.csv')
 sample_submission3['isFraud'].describe()<define_variables>
if DO_SUBMISSION or DO_VIRTUAL_SUBMISSION:
sub_list = [
pd.read_csv('./submission-sinchir0-nn.csv'),
pd.read_csv('./submission_sinchir0_tabnet.csv'),
pd.read_csv("./submission_takapy_tf-resnet.csv"),
pd.read_csv('./submission_tawara_thrnn_seed_cv.csv'),
pd.read_csv('./submission_tawara_thrnn_drug_seed_cv.csv'),
]
else:
sub_list = [
pd.read_csv('.. /input/nn-use-train-public/submission.csv'),
pd.read_csv('.. /input/tabnet-train-public-add-n-shared-1/submission.csv'),
pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/submission.csv"),
pd.read_csv('.. /input/moa-weight-thrnn-seed-cv/submission.csv'),
pd.read_csv('.. /input/moa-weight-thrnn-drug-seed-cv/submission.csv'),
]
for i, name in enumerate(["NN(drugCV)", "TabNet", "ResNet", "ThrNN", "ThrNN(drugCV)"]):
print(f"[{name}]:", sub_list[i].shape )
%%time
EPOCHS = 4
TIMES = 24
frac_inc = np.arange(0,TIMES,1)/400 +0.1
print(frac_inc)
 y_preds = np.zeros(sample_submission.shape[0])
 for _ in range(TIMES):
kf = StratifiedKFold(n_splits=EPOCHS, random_state= _, shuffle=True)
 X_train_new = X_train[y_train == 0].sample(frac = frac_inc[_] ).append(X_train[y_train == 1])
 y_train_new = y_train[X_train_new.index].reset_index().drop(columns = 'TransactionID')
 X_train_new = X_train_new.reset_index().drop(columns = 'TransactionID')
 y_oof_new = np.zeros(X_train_new.shape[0])
 gc.collect()
for tr_idx, val_idx in kf.split(X_train_new, y_train_new):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=10,
learning_rate=0.04,
subsample=0.8,
colsample_bytree=0.9,
tree_method='gpu_hist',
missing=-999,
min_child_weight=2 
)
 X_tr, X_vl = X_train_new.iloc[tr_idx, :], X_train_new.iloc[val_idx, :]
y_tr, y_vl = y_train_new.iloc[tr_idx], y_train_new.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof_new[val_idx] = y_pred_train
y_preds+= clf.predict_proba(X_test)[:,1] / EPOCHS
print('ROC AUC oof_new {}'.format(roc_auc_score(y_train_new, y_oof_new)))
 sample_submission4 = sample_submission.copy()
sample_submission4['isFraud'] = y_preds/TIMES
sample_submission4.to_csv('xgboost_undersample.csv')
 sample_submission4['isFraud'].describe()<save_to_csv>
sub_list = [order_sub(sub_df)for sub_df in sub_list]
sample_submission_blend = sample_submission.copy()
sample_submission_blend['isFraud'] =(sample_submission1a['isFraud']+sample_submission1['isFraud'] + sample_submission2['isFraud']*0.5+ sample_submission3['isFraud']*0.5+ sample_submission4['isFraud'])/4
sample_submission_blend.to_csv('xgboost_under_over_blend.csv')
 sample_submission_blend2 = sample_submission.copy()
sample_submission_blend2['isFraud'] =(sample_submission1a['isFraud']*0.5+sample_submission1['isFraud']*0.5 + sample_submission2['isFraud']*0.25+ sample_submission3['isFraud']*0.25+ sample_submission4['isFraud'])/2.5
sample_submission_blend2.to_csv('xgboost_under_over_blend2.csv')
 sample_submission_blend_equal = sample_submission.copy()
sample_submission_blend_equal['isFraud'] =(sample_submission1a['isFraud']+sample_submission1['isFraud'] + sample_submission2['isFraud']+ sample_submission3['isFraud']+ sample_submission4['isFraud'])/5
sample_submission_blend_equal.to_csv('xgboost_under_over_blend_equal.csv' )<load_from_csv>
def inference_function(settings, model, loader, device):
model.to(device)
 model.eval()
pred_list = []
with torch.no_grad() :
for x, t in loader:
y = model(x.to(device))
pred_list.append(y.sigmoid().detach().cpu().numpy())
 pred_arr = np.concatenate(pred_list)
 del pred_list
return pred_arr
sub1 = pd.read_csv('.. /input/ieee-blend/lgb_sub.csv')
 sub2 = pd.read_csv('.. /input/ieee-blend/submission_IEEE(1 ).csv')
 sub3 = pd.read_csv('.. /input/ieee-blend/submission_IEEE.csv')
 sub4 = pd.read_csv('.. /input/ieee-blend/submission.csv')
 temp=pd.read_csv('.. /input/ieee-blend/lgb_sub.csv' )<save_to_csv>
16 * 206
temp['isFraud'] = 0.60*sub4['isFraud'] + 0.40*sub3['isFraud']
temp.to_csv('submission_p2_1.csv', index=False )<define_variables>
X_test = np.concatenate([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=1)
 print(X_test.shape)
 X_test_2d = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 print(X_test_2d.shape)
 X_test_3d = X_test_2d[:, None,...]
print(X_test_3d.shape)
 X_test_node = np.stack([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 test_dataset = MoAStackingDataset(X_test, None)
sub_path = ".. /input/ieee-blend"
all_files = os.listdir(sub_path)
 all_files<load_from_csv>
oof_pred_arr_avg = np.zeros(( len(X), 206))
test_pred_arr_avg = np.zeros(( len(X_test), 206))
for m_id, tmp_stgs in enumerate(stgs_list):
n_folds = tmp_stgs["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
train_val_indexs = [
drug_multilabel_stratified_kfold(
 fold_id, target_in_drug_trt, TARGET_COL, n_folds, tmp_seed)
 for fold_id in range(n_folds)]
oof_pred_arr = np.zeros(( len(X), 206))
test_preds_arr = np.zeros(( n_folds, len(X_test), 206))
train_all_dataset.reset_model_order()
train_all_dataset.shuffle_model_order(tmp_seed)
 test_dataset.model_order = train_all_dataset.model_order
print("[model {}'s order]: {}".format(m_id, train_all_dataset.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
model_path = WORK_DIR / "best_model_fold{}_model{}.pth".format(fold_id, m_id)
 model = eval(tmp_stgs["model"]["name"] )(**tmp_stgs["model"]["params"])
 model.load_state_dict(torch.load(model_path))
val_index = train_val_indexs[fold_id][1]
val_dataset = data.Subset(train_all_dataset, val_index)
 val_loader = data.DataLoader(val_dataset, **tmp_stgs["loader"]["val"])
 val_pred = inference_function(tmp_stgs, model, val_loader, device)
 oof_pred_arr[val_index] = val_pred
del val_dataset; del val_loader; del val_pred
test_loader = data.DataLoader(test_dataset, **tmp_stgs["loader"]["val"])
 test_pred = inference_function(tmp_stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del test_loader; del test_pred;
gc.collect()
oof_pred_arr_avg += oof_pred_arr
test_pred_arr_avg += test_preds_arr.mean(axis=0)
 oof_score = log_loss(y.ravel() , oof_pred_arr.ravel())
 cv_score_df_list[m_id] = cv_score_df_list[m_id].append(
 {"model": m_id, "fold": "oof", "val/metric": oof_score,},
ignore_index=True)
 oof_pred_arr_avg /= len(stgs_list)
 test_pred_arr_avg /= len(stgs_list )
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "ieee" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]<feature_engineering>
cv_score_all = pd.concat(cv_score_df_list, axis=0)
 cv_score_all
concat_sub['ieee_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['ieee_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['ieee_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['ieee_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )<define_variables>
print(log_loss(y.ravel() , oof_pred_arr_avg.ravel()))
cutoff_lo = 0.7
cutoff_hi = 0.3<save_to_csv>
if DO_SUBMISSION or DO_TRAIN_FOR_ENSEMBLE:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 elif DO_VIRTUAL_SUBMISSION:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 print(df_test.shape, submission.shape)
 df_test, submission = generate_virtual_private(df_test, submission)
 print(df_test.shape, submission.shape)
 else:
raise ValueError
<save_to_csv><EOS> 
df = submission.copy()
public_id = list(df['sig_id'].values)
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type =='ctl_vehicle'].sig_id] = 0
df_submit.to_csv('submission.csv',index=True )