Kaggle competition: Kannada MNIST
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train<init_hyperparams>
plt.style.use('ggplot')
 %matplotlib inline
,Flatten, Lambda, Reshape, LocallyConnected2D, BatchNormalization, MaxPool2D, Dropout
no_ctl = True
scale = 'rankgauss'
variance_threshould = 0.8
decompo = 'PCA'
ncompo_genes = 50
ncompo_cells = 8
encoding = 'dummy'
base_seed = 42
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 print(DEVICE)
 EPOCHS = 20
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
hidden_size=1024
SEEDS = 7
SEED = [0, 1, 2, 3, 4, 5, 6]
DRUG_THRESH = 18<load_from_csv>
dirname = '/kaggle/input/Kannada-MNIST'
output_dirname = '/kaggle/output/kaggle/working'
train_df = pd.read_csv(os.path.join(dirname, 'train.csv'))
test_df = pd.read_csv(os.path.join(dirname, 'test.csv'))
dig_df = pd.read_csv(os.path.join(dirname, 'Dig-MNIST.csv'))
sample_df = pd.read_csv(os.path.join(dirname, 'sample_submission.csv'))
seed_everything(base_seed)
 def preprocessor_Multi_heads() :
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train, test, targets = data_processing(train_features,test_features, train_targets_scored)
 print('length of target_cols:',len(target_cols))
train_comb = train.merge(targets, on='sig_id')
 train_comb.drop('cp_type', axis=1, inplace=True)
 print("combine train and targets to prepare for kfolds ", train_comb.shape)
 feature_cols = [c for c in process_data(train_comb ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id', 'drug_id']]
print('length of feature_cols:',len(feature_cols))
gen_cols = [c for c in feature_cols if 'g-' == c[:2]]
cell_cols = [c for c in feature_cols if 'c-' == c[:2]]
pca_gen_cols = [c for c in feature_cols if 'pca_g' == c[:5]]
pca_cell_cols = [c for c in feature_cols if 'pca_c' == c[:5]]
ohe_cols = [c for c in feature_cols if c not in gen_cols+cell_cols+pca_gen_cols+pca_cell_cols]
print(len(gen_cols), len(cell_cols), len(pca_gen_cols), len(pca_cell_cols),len(ohe_cols))
num_features=len(feature_cols)
 num_targets=len(target_cols)
 folds = train_comb.copy()
folds = folds.merge(train_drug, on='sig_id')
 folds = make_cv_folds(folds, SEEDS, NFOLDS, DRUG_THRESH)
 return folds, test, targets,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols,test_features
folds, test, targets,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols, test_features = preprocessor_Multi_heads()<choose_model_class>
X = train_df.drop('label', axis=1 ).to_numpy()
y = train_df['label'].to_numpy()
X_dig = dig_df.drop('label', axis=1 ).to_numpy()
y_dig = dig_df['label'].to_numpy()
X_test = test_df.drop('id', axis=1 ).to_numpy()
X = X.reshape(-1, 28, 28, 1)
 y = to_categorical(y)
 X_dig = X_dig.reshape(-1, 28, 28, 1)
 y_dig = to_categorical(y_dig)
 X_test = X_test.reshape(-1, 28, 28, 1)
 X = X / 255
X_dig = X_dig / 255
X_test = X_test / 255
seed = 33
np.random.RandomState(seed)
 valid_size=0.15
X_train, X_valid, y_train, y_valid = train_test_split(X, y,\
test_size=valid_size, random_state=seed )
class Model(nn.Module):
def __init__(self, num_gen_features, num_cell_features,num_targets, hidden_size):
super(Model, self ).__init__()
self.gen_batch_norm1 = nn.BatchNorm1d(num_gen_features)
 self.gen_dropout1 = nn.Dropout(0.2)
 self.gen_dense1 = nn.utils.weight_norm(nn.Linear(num_gen_features, hidden_size))
self.gen_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.gen_dropout2 = nn.Dropout(0.3)
 self.gen_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.cell_batch_norm1 = nn.BatchNorm1d(num_cell_features)
 self.cell_dropout1 = nn.Dropout(0.2)
 self.cell_dense1 = nn.utils.weight_norm(nn.Linear(num_cell_features, int(hidden_size/2)))
 self.cell_batch_norm2 = nn.BatchNorm1d(int(hidden_size/2))
self.cell_dropout2 = nn.Dropout(0.3)
 self.cell_dense2 = nn.utils.weight_norm(nn.Linear(int(hidden_size/2), int(hidden_size/2)))
 self.batch_norm3 = nn.BatchNorm1d(hidden_size + int(hidden_size/2))
self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size + int(hidden_size/2), num_targets))
def forward(self, gen_x, cell_x):
gen_x = self.gen_batch_norm1(gen_x)
 gen_x = self.gen_dropout1(gen_x)
 gen_x = F.relu(self.gen_dense1(gen_x))
gen_x = self.gen_batch_norm2(gen_x)
 gen_x = self.gen_dropout2(gen_x)
 gen_x = F.relu(self.gen_dense2(gen_x))
cell_x = self.cell_batch_norm1(cell_x)
 cell_x = self.cell_dropout1(cell_x)
 cell_x = F.relu(self.cell_dense1(cell_x))
cell_x = self.cell_batch_norm2(cell_x)
 cell_x = self.cell_dropout2(cell_x)
 cell_x = F.relu(self.cell_dense2(cell_x))
x = torch.cat(( gen_x,cell_x),dim=1)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<categorify>
def create_model() :
input_layer = Input(shape=(28, 28, 1))
x = Conv2D(filters=6, kernel_size=(5, 5), padding='Same', activation='tanh' )(input_layer)
 x = AveragePooling2D()(x)
 x = Activation('tanh' )(x)
 x = Conv2D(filters=16, kernel_size=(5, 5), activation='tanh' )(x)
 x = AveragePooling2D()(x)
 x = Activation('tanh' )(x)
 x = Conv2D(filters=120, kernel_size=(5, 5), activation='tanh' )(x)
 x = Flatten()(x)
 x = Dense(84, activation='tanh' )(x)
 output = Dense(10, activation='softmax' )(x)
 model = Model(inputs=[input_layer], outputs=[output])
 return model
class MoADataset:
def __init__(self, gen_features, cell_features, targets):
self.gen_features = gen_features
self.cell_features = cell_features
self.targets = targets
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, gen_features, cell_features):
self.gen_features = gen_features
self.cell_features = cell_features
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
model.compile(optimizer=keras.optimizers.SGD() ,
loss='categorical_crossentropy',
metrics=['accuracy'] )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
gen_x, cell_x = data['gen_x'].to(device), data['cell_x'].to(device)
 with torch.no_grad() :
outputs = model(gen_x, cell_x)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<prepare_x_and_y>
model = create_model()
model.summary()
model.compile(optimizer=keras.optimizers.SGD() ,
loss='categorical_crossentropy',
metrics=['accuracy'])
def get_oof_predictions(fold,seed):
seed_everything(seed)
 test_ = process_data(test)
 model = Model(
 num_gen_features=len(gen_cols + pca_gen_cols + ohe_cols),
num_cell_features=len(cell_cols + pca_cell_cols + ohe_cols),
num_targets=len(target_cols),
hidden_size=hidden_size)
 model.to(DEVICE)
 model.load_state_dict(torch.load(f".. /input/jp-multiheads/FOLD{fold}_SEED{seed}.pth"))
gen_x_test, cell_x_test = test_[gen_cols + pca_gen_cols + ohe_cols].values, test_[cell_cols + pca_cell_cols + ohe_cols].values
testdataset = TestDataset(gen_x_test, cell_x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 print('predict on test...', end='')
 predictions = np.zeros(( len(test_), targets.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 print(' done.
 ')
 return predictions<categorify>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
def run_k_fold(NFOLDS, seed):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
pred_ = get_oof_predictions(fold, seed)
 predictions += pred_ / NFOLDS
return predictions<categorify>
def create_my_model() :
input_layer = Input(shape=(28, 28, 1))
x = Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu' )(input_layer)
 x = Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu' )(x)
 x = BatchNormalization(momentum=.15 )(x)
 x = MaxPool2D(pool_size=(2,2))(x)
 x = Dropout(0.25 )(x)
 x = Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu' )(x)
 x = Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu' )(x)
 x = BatchNormalization(momentum=.15 )(x)
 x = MaxPool2D(pool_size=(2,2))(x)
 x = Dropout(0.25 )(x)
 x = Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu' )(x)
 x = Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu' )(x)
 x = BatchNormalization(momentum=.15 )(x)
 x = MaxPool2D(pool_size=(2,2))(x)
 x = Flatten()(x)
 x = Dense(256, activation="relu" )(x)
 x = Dropout(0.4 )(x)
 output = Dense(10, activation='softmax' )(x)
 model = Model(inputs=[input_layer], outputs=[output])
 return model
def InferenceMultiHeads(SEED):
predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
predictions_ = run_k_fold(NFOLDS, seed)
 predictions += predictions_ / len(SEED)
 print(predictions.shape)
 return predictions
predictions = InferenceMultiHeads(SEED )<create_dataframe>
my_model.compile(optimizer=keras.optimizers.Adam() ,
loss='categorical_crossentropy',
metrics=['accuracy'] )
predictions_clip = predictions
predictions_clip = pd.DataFrame(predictions_clip, columns=target_cols )<load_pretrained>
my_model.fit(datagen.flow(X_train, y_train, batch_size=64),
epochs=30,
verbose=1, validation_data =(X_valid, y_valid),
callbacks=[ReduceLROnPlateau(monitor='val_accuracy',
factor=0.5,
patieecne=3,
verbose=1,
min_lr=0.00001),] )
oof_multi_heads_all = np.load('.. /input/jp-multiheads/multi_heads_oof.npy' )<save_to_csv>
y_result = my_model.predict(X_test)
 y_result = np.argmax(y_result, axis=1)
 sample_df['label'] = y_result
sample_df.to_csv('submission.csv', index=False )
test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sig_id = test[test['cp_type']!='ctl_vehicle'].sig_id.reset_index(drop=True)
 predictions_clip['sig_id'] = sig_id
sub_mh = pd.merge(test[['sig_id']], predictions_clip, on='sig_id', how='left')
 sub_mh.fillna(0, inplace=True)
 sub_mh.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_mh.to_csv('submission_mh.csv', index=False )<create_dataframe>
optimizers = {
'sgd': 'SGD',
'sgd_mom': keras.optimizers.SGD(momentum=0.5),
'rmsprop': 'RMSprop',
'adam': 'Adam',
}
<compute_train_metric><EOS> 
historys = {}
models = {}
num_epochs = 1
for n, optimizer in enumerate(optimizers):
models[optimizer] = create_my_model()
models[optimizer].compile(optimizer=optimizers[optimizer],
loss='categorical_crossentropy',
metrics=['accuracy'])
 start = time.time()
h = models[optimizer].fit(X_train, y_train,
batch_size=1024,
epochs=num_epochs,
verbose=0,
validation_data=(X_valid, y_valid),
callbacks=[ReduceLROnPlateau(monitor='val_accuracy',
factor=0.5,
patieecne=3,
verbose=1,
min_lr=0.00001),
CSVLogger("{0}.log".format(optimizer)) ] 
)
 historys[optimizer] = h.history
print("{0} Optimizer: ".format(optimizer))
print("accuracy: {0}, val_accuracy {1}, time {2} minutes: ".format(\
historys[optimizer]['accuracy'][-1],
historys[optimizer]['val_accuracy'][-1], 
(time.time() - start)/ 60))