Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<choose_model_class>
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('.. /kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
MAX_EPOCH = 200
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 p_min = 0.0005
p_max = 0.9995
def InferenceTabNet(X_test, SEEDS, NFOLDS,DEVICE):
test_cv_preds = []
for s in SEEDS:
for fold_id in range(NFOLDS):
print(b_,"FOLDS: ", r_, fold_id + 1, y_, 'seed:', s)
 print(g_, '*' * 60, c_)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/TabNet_seed_'\
+str(s)+'_fold_'+str(fold_id+1)
 saved_file = '/kaggle/working/model'
shutil.make_archive(saved_file,'zip',saved_path_name)
 print(f'Successfully saved model at {saved_file}.zip')
 with zipfile.ZipFile('model.zip')as z:
with z.open('model_params.json')as f:
loaded_params = json.load(f)
 with z.open('network.pt')as f:
try:
saved_state_dict = torch.load(f, map_location=DEVICE)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.bytesIO(f.read()),map_location = DEVICE)
 loaded_model = TabNetRegressor(**loaded_params)
 loaded_model._set_network()
loaded_model.network.load_state_dict(saved_state_dict)
 loaded_model.network.eval()
preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 test_preds_all = np.stack(test_cv_preds)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/'
oof_TabNet_all = np.load(saved_path_name + 'oof_TabNet_all.npy')
 return oof_TabNet_all,test_preds_all
oof_TabNet_all,test_preds_all = InferenceTabNet(X_test, SEEDS, NFOLDS,DEVICE )<load_from_csv>
df_train = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 df_test = pd.read_csv('.. /input/Kannada-MNIST/Dig-MNIST.csv')
 y_train = df_train.iloc[:,0].values
y_test = df_test.iloc[:,0].values
df_train = df_train.drop(['label'], axis=1)
 df_test = df_test.drop(['label'], axis=1)
 N_train = len(y_train)
 N_test = len(y_test)
def to_submission_TabNet(test_preds_all,submission_TabNet):
data_path = ".. /input/lish-moa/"
all_feat = [col for col in submission_TabNet.columns if col not in ["sig_id"]]
test_TabNet = pd.read_csv(data_path + "test_features.csv")
 sig_id = test_TabNet[test_TabNet["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission_TabNet = pd.merge(test_TabNet[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission_TabNet.fillna(0, inplace = True)
 submission_TabNet.to_csv("submission_TabNet.csv", index = None)
 return submission_TabNet
submission_TabNet = to_submission_TabNet(test_preds_all,submission_TabNet)
 display(submission_TabNet.head())
 print(f"{b_}submission_TabNet.shape: {r_}{submission_TabNet.shape}" )<set_options>
x_train = np.uint8(df_train.values.reshape(-1,28,28))
x_test = np.uint8(df_test.values.reshape(-1,28,28))
%who DataFrame
del targets_TabNet, test_TabNet, train_df,target_cols, feature_cols
gc.collect()
%who DataFrame<set_options>
def preprocessing(img):
kernel = np.array([[0,-1.5,0],[-1.5,8.5,-1.5],[0,-1.5,0]])
 img = cv2.filter2D(img,-1,kernel)
 img_thresh = img/255.0 *2 -1
return img_thresh
x_train = np.array(list(map(preprocessing,x_train)))
 x_test = np.array(list(map(preprocessing, x_test)))
 x_train= x_train.reshape(-1,28,28,1)
 x_test= x_test.reshape(-1,28,28,1)
 y_train = tf.keras.utils.to_categorical(y_train,10)
 y_test = tf.keras.utils.to_categorical(y_test,10 )
seed = 42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(seed)
 def preprocessor_nn_transfer() :
variance_threshould = 0.8
ncompo_genes = 600
ncompo_cells = 50
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 train_features = train_features[train_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test_features = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train_features = train_features.drop('cp_type', axis=1)
 test_features = test_features.drop('cp_type', axis=1)
 GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train_features[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train_features[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train_features = pd.concat([train_features, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test_features[GENES])
 pca_cells_test = pca_cells.transform(test_features[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test_features = pd.concat([test_features, pca_genes_test, pca_cells_test], axis = 1)
 PCA')
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
mask =(train_features[cols_numeric].var() >= variance_threshould ).values
tmp = train_features[cols_numeric].loc[:, mask]
train_features = pd.concat([train_features[["sig_id", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
test_features = pd.concat([test_features[["sig_id", "cp_time", "cp_dose"]], test_features.loc[:,cols_numeric]], axis = 1)
 train_features = pd.get_dummies(train_features, columns = ['cp_time', 'cp_dose'])
 test_features = pd.get_dummies(test_features, columns = ['cp_time', 'cp_dose'])
 Feature selection')
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 test = test_features
Join')
 gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577',
'g-153','g-389','g-60','g-370','g-248','g-167',
'g-203','g-177','g-301','g-332','g-517','g-6',
'g-744','g-224','g-162','g-3','g-736','g-486',
'g-283','g-22','g-359','g-361','g-440','g-335',
'g-106','g-307','g-745','g-146','g-416','g-298',
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
for df in [train, test]:
df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in gsquarecols:
if feature in GENES:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in CELLS:
df[f'{feature}_squared'] = df[feature] ** 2
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 num_targets: {}'.format(num_targets))
')
 return train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols<split>
X_train, X_val, Y_train, Y_val = train_test_split(x_train, y_train, random_state=0, test_size=0.15)
 X_train.shape, X_val.shape, Y_train.shape, Y_val.shape
train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols = preprocessor_nn_transfer()<categorify>
i = Input(shape=(X_train[0].shape))
x1 = Conv2D(32,(3,3), padding="same" )(i)
 x1 = LeakyReLU(0.2 )(x1)
 x1 = BatchNormalization()(x1)
 x1 = Conv2D(32,(3,3), padding="same" )(x1)
 x1 = LeakyReLU(0.2 )(x1)
 x1 = BatchNormalization()(x1)
 x1 = Conv2D(64,(3,3), padding="same" )(x1)
 x1 = LeakyReLU(0.2 )(x1)
 b1 = BatchNormalization()(x1)
 x2 = Conv2D(64,(3,3), padding="same" )(b1)
 x2 = LeakyReLU(0.2 )(x2)
 x2 = BatchNormalization()(x2)
 x2 = Conv2D(64,(5,5), padding="same" )(x2)
 x2 = LeakyReLU(0.2 )(x2)
 x2 = BatchNormalization()(x2)
 x3 = Add()([b1,x2])
 x3 = MaxPooling2D()(x3)
 x3 = Dropout(0.2 )(x3)
 x3 = Conv2D(128,(3,3), padding="same" )(x3)
 x3 = LeakyReLU(0.1 )(x3)
 x3 = BatchNormalization()(x3)
 x3 = Conv2D(128,(3,3), padding="same" )(x3)
 x3 = LeakyReLU(0.1 )(x3)
 x3 = Conv2D(256,(3,3), padding="same" )(x3)
 x3 = LeakyReLU(0.1 )(x3)
 b2 = BatchNormalization()(x3)
 x3 = Conv2D(256,(5,5), padding="same" )(b2)
 x3 = LeakyReLU(0.1 )(x3)
 x3 = BatchNormalization()(x3)
 x = Add()([b2,x3])
 x = MaxPooling2D()(x)
 x = Conv2D(256,(3,3), padding="same" )(x)
 x = LeakyReLU(0.1 )(x)
 x = GlobalMaxPooling2D()(x)
 x = Flatten()(x)
 x = Dense(128, activation='relu', kernel_regularizer=l2(0.01))(x)
 x = Dropout(0.2 )(x)
 x = Dense(128, activation='relu', kernel_regularizer=l2(0.01))(x)
 x = Dropout(0.2 )(x)
 x = Dense(10, kernel_regularizer=l2(0.01))(x)
 model = Model(i,x)
 plot_model(model, "model.png", show_shapes=True )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<define_variables>
model.compile(optimizer=Adam(lr=0.001, beta_1=0.9),
loss='squared_hinge',
metrics=['accuracy'])
 dataGen = ImageDataGenerator(rotation_range=15,width_shift_range=0.2, height_shift_range=0.2,zoom_range=0.15)
 dataGen.fit(X_train)
 batch_size=32
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.3,
patience=3, min_lr=1e-10)
 r = model.fit_generator(dataGen.flow(X_train, Y_train, batch_size=32),
steps_per_epoch=X_train.shape[0]//batch_size, epochs=50, validation_data=(X_val,Y_val), verbose=2,callbacks=[reduce_lr] )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
SEEDS = list(range(42,49))
NFOLDS = 10
DRUG_THRESH = 18
feature_cols = [c for c in train.columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
<choose_model_class>
model.evaluate(x_test, y_test )
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 test_ = test
model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/nn-transferlearning-7seeds4248-10folds/SCORED_ONLY_SEED_{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 print(f"Pretrained_Model_Loaded_SCORED_ONLYSEED: {seed_id}, FOLD: {fold_id} ")
 return predictions
def run_k_fold(NFOLDS, seed_id):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
return predictions<feature_engineering>
df_final = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 df_final = df_final.drop(['id'], axis=1)
 X_final = np.uint8(df_final.values.reshape(-1,28,28))
X_final = np.array(list(map(preprocessing,X_final)))
 X_final= X_final.reshape(-1,28,28,1)
 pred = model.predict(X_final ).argmax(axis=1 )
<save_to_csv><EOS> 
sub = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 sub['label'] = pred
sub['label']= sub['label'].astype(int)
 sub.to_csv('submission.csv', index=False )