Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<init_hyperparams>
torch.cuda.is_available()
no_ctl = True
scale = 'rankgauss'
variance_threshould = 0.8
decompo = 'PCA'
ncompo_genes = 50
ncompo_cells = 8
encoding = 'dummy'
base_seed = 42
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 print(DEVICE)
 EPOCHS = 20
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
hidden_size=1024
SEEDS = 7
SEED = [0, 1, 2, 3, 4, 5, 6]
DRUG_THRESH = 18<load_from_csv>
print("Is a GPU available? {}".format(is_available()))
print("The name of the device is {}".format(get_device_name()))
print("Device capability is {}".format(get_device_capability(0)))
 print("Available GPU memory is {0:.{1}f} GB ".format(get_device_properties(0 ).total_memory /(1024 ** 3), 1))
seed_everything(base_seed)
 def preprocessor_Multi_heads() :
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train, test, targets = data_processing(train_features,test_features, train_targets_scored)
 print('length of target_cols:',len(target_cols))
train_comb = train.merge(targets, on='sig_id')
 train_comb.drop('cp_type', axis=1, inplace=True)
 print("combine train and targets to prepare for kfolds ", train_comb.shape)
 feature_cols = [c for c in process_data(train_comb ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id', 'drug_id']]
print('length of feature_cols:',len(feature_cols))
gen_cols = [c for c in feature_cols if 'g-' == c[:2]]
cell_cols = [c for c in feature_cols if 'c-' == c[:2]]
pca_gen_cols = [c for c in feature_cols if 'pca_g' == c[:5]]
pca_cell_cols = [c for c in feature_cols if 'pca_c' == c[:5]]
ohe_cols = [c for c in feature_cols if c not in gen_cols+cell_cols+pca_gen_cols+pca_cell_cols]
print(len(gen_cols), len(cell_cols), len(pca_gen_cols), len(pca_cell_cols),len(ohe_cols))
num_features=len(feature_cols)
 num_targets=len(target_cols)
 folds = train_comb.copy()
folds = folds.merge(train_drug, on='sig_id')
 folds = make_cv_folds(folds, SEEDS, NFOLDS, DRUG_THRESH)
 return folds, test, targets,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols,test_features
folds, test, targets,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols, test_features = preprocessor_Multi_heads()<choose_model_class>
df = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 os.makedirs("images")
 def bbox(image):

 HEIGHT = image.shape[0]
WIDTH = image.shape[1]
for i in range(image.shape[1]):
if(image[:, i] > 0 ).sum() >= 1:
x_min = i - 1 if(i > 1)else 0
break
for i in reversed(range(image.shape[1])) :
if(image[:, i] > 0 ).sum() >= 1:
x_max = i + 2 if(i < WIDTH - 2)else WIDTH
break
for i in range(image.shape[0]):
if(image[i] > 0 ).sum() >= 1:
y_min = i - 1 if(i > 1)else 0
break
for i in reversed(range(image.shape[0])) :
if(image[i] > 0 ).sum() >= 1:
y_max = i + 2 if(i < HEIGHT - 2)else HEIGHT
break
return x_min, y_min, x_max, y_max
ids = []
labels = []
for i,row in df.iterrows() :
id = "{}".format(i)
 img = np.reshape(row[1:].values, newshape=(28,28)).astype(np.uint8)
 x_min, y_min, x_max, y_max = bbox(img)
 img = img[y_min:y_max, x_min:x_max]
x_min, y_min, x_max, y_max = bbox(img)
 img = img[y_min:y_max, x_min:x_max]
img = np.pad(img,(2,2), "constant", constant_values=(0,0))
img[img >= 50] = 255
img[img < 50] = 0
img = Image.fromarray(img)
 img = img.resize(( 28,28), Image.ANTIALIAS)
 label = 0
img.save("images/{}.png".format(id))
ids.append(id)
 labels.append(label)
 label_df = pd.DataFrame({"id": ids, "label": labels})
class Model(nn.Module):
def __init__(self, num_gen_features, num_cell_features,num_targets, hidden_size):
super(Model, self ).__init__()
self.gen_batch_norm1 = nn.BatchNorm1d(num_gen_features)
 self.gen_dropout1 = nn.Dropout(0.2)
 self.gen_dense1 = nn.utils.weight_norm(nn.Linear(num_gen_features, hidden_size))
self.gen_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.gen_dropout2 = nn.Dropout(0.3)
 self.gen_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.cell_batch_norm1 = nn.BatchNorm1d(num_cell_features)
 self.cell_dropout1 = nn.Dropout(0.2)
 self.cell_dense1 = nn.utils.weight_norm(nn.Linear(num_cell_features, int(hidden_size/2)))
 self.cell_batch_norm2 = nn.BatchNorm1d(int(hidden_size/2))
self.cell_dropout2 = nn.Dropout(0.3)
 self.cell_dense2 = nn.utils.weight_norm(nn.Linear(int(hidden_size/2), int(hidden_size/2)))
 self.batch_norm3 = nn.BatchNorm1d(hidden_size + int(hidden_size/2))
self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size + int(hidden_size/2), num_targets))
def forward(self, gen_x, cell_x):
gen_x = self.gen_batch_norm1(gen_x)
 gen_x = self.gen_dropout1(gen_x)
 gen_x = F.relu(self.gen_dense1(gen_x))
gen_x = self.gen_batch_norm2(gen_x)
 gen_x = self.gen_dropout2(gen_x)
 gen_x = F.relu(self.gen_dense2(gen_x))
cell_x = self.cell_batch_norm1(cell_x)
 cell_x = self.cell_dropout1(cell_x)
 cell_x = F.relu(self.cell_dense1(cell_x))
cell_x = self.cell_batch_norm2(cell_x)
 cell_x = self.cell_dropout2(cell_x)
 cell_x = F.relu(self.cell_dense2(cell_x))
x = torch.cat(( gen_x,cell_x),dim=1)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<categorify>
class KannadaMNIST(object):
def __init__(self
, image_folder : str
, labels : pd.DataFrame
, transforms : list = None 
):

 self.image_folder = image_folder
self.labels = labels
self.transforms = transforms
self.index = np.arange(self.labels.shape[0])
 def __len__(self):
return len(self.index)
 def __getitem__(self, idx):
label = self.labels.iloc[self.index[idx]]
img = Image.open(self.image_folder + "/" + label["id"] + ".png")
 img_ = self.transforms(img)
 return {"image": img_, "id":int(label["id"])}
class MoADataset:
def __init__(self, gen_features, cell_features, targets):
self.gen_features = gen_features
self.cell_features = cell_features
self.targets = targets
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, gen_features, cell_features):
self.gen_features = gen_features
self.cell_features = cell_features
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
def conv1d(ni, no, ks=1, stride=1, padding=0, bias=False):
	"Create and initialize a `nn.Conv1d` layer with spectral normalization."
	conv = nn.Conv1d(ni, no, ks, stride=stride, padding=padding, bias=bias)
 	nn.init.kaiming_normal_(conv.weight)
 	if bias: conv.bias.data.zero_()
	return nn.utils.spectral_norm(conv)
 class SimpleSelfAttention(nn.Module):
	
	def __init__(self, n_in, ks=1, sym=False):
		super().__init__()
		self.conv = conv1d(n_in, n_in, ks, padding=ks//2, bias=False)
 		self.gamma = nn.Parameter(torch.Tensor([0.]))
		self.sym = sym
		self.n_in = n_in
		
	def forward(self, x):
		if self.sym:
			
			c = self.conv.weight.view(self.n_in,self.n_in)
 			c =(c + c.t())/2
			self.conv.weight = c.view(self.n_in,self.n_in,1)
 				
		size = x.size()
		x = x.view(*size[:2],-1)
 		
		
		
		
		convx = self.conv(x)
 		xxT = torch.bmm(x, x.permute(0,2,1 ).contiguous())
 		o = torch.bmm(xxT, convx)
 		o = self.gamma * o + x
		
		return o.view(*size ).contiguous()
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
gen_x, cell_x = data['gen_x'].to(device), data['cell_x'].to(device)
 with torch.no_grad() :
outputs = model(gen_x, cell_x)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>

 class KannadaNet(nn.Module):
def __init__(self
, input_shape
, n_class 
):

 super(KannadaNet, self ).__init__()
self._build_layers(input_shape=input_shape, n_class=n_class)
 self._init_weight()
print("Model contains {} parameters".format(sum(p.numel() for p in self.parameters() if p.requires_grad)))
 def forward(self, x):
x = self._conv_forward(x)
 x = self.fc_1(x)
 x = F.relu(x)
 x = self.bn_1(x)
 x = self.fc_2(x)
 x = F.relu(x)
 x = self.bn_2(x)
 confidence = self.confidence(x)
 out = self.classifier(x)
 return {"prediction":out.cpu().float() , "confidence":confidence.cpu().float() }
def _build_layers(self, input_shape, n_class):

 self.stem = Conv2D_BN(in_channels =input_shape[0]
, out_channels=32
, padding =1 
)
 self.inception_res_1 = InceptionBlock(in_shape=[32,28,28], residual=True)
 self.attention_1 = SimpleSelfAttention(self.inception_res_1.output_shape[0])
 self.inception_res_2 = InceptionBlock(in_shape=[int(x / y)for x, y in zip(self.inception_res_1.output_shape, [1,2,2])], residual=True)
 self.attention_2 = SimpleSelfAttention(self.inception_res_2.output_shape[0])
 self.inception_res_3 = InceptionBlock(in_shape=[int(x / y)for x, y in zip(self.inception_res_2.output_shape, [1, 2, 2])], residual=True)
 self.attention_3 = SimpleSelfAttention(self.inception_res_3.output_shape[0])
 self.pool = nn.MaxPool2d(kernel_size=2)
 conv_out_dims = self._conv_forward(torch.zeros([1] + input_shape, dtype=torch.float32))
self.fc_1 = nn.Linear(in_features=conv_out_dims.shape[1], out_features=512)
 self.bn_1 = nn.BatchNorm1d(num_features=512)
 self.fc_2 = nn.Linear(in_features=512, out_features=512)
 self.bn_2 = nn.BatchNorm1d(num_features=512)
 self.confidence = nn.Linear(in_features=512, out_features=1)
 self.classifier = nn.Linear(in_features=512, out_features=n_class)
 def _conv_forward(self, x):

 x = self.stem(x)
 x = self.inception_res_1(x)
 x = self.attention_1(x)
 x = self.pool(x)
 x = self.inception_res_2(x)
 x = self.attention_2(x)
 x = self.pool(x)
 x = self.inception_res_3(x)
 x = self.attention_3(x)
 x = self.pool(x)
 x = torch.flatten(x, start_dim=1)
 return x
def _init_weight(self):
for m in self.modules() :
print(m)
 if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1.0)
 m.bias.data.zero_()
class InceptionBlock(nn.Module):
def __init__(self, in_shape, residual=True):
super(InceptionBlock, self ).__init__()
self.residual = residual
self.conv_1x1_1 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]/2))
self.bn_1 = nn.BatchNorm2d(num_features=self.conv_1x1_1.out_channels)
 self.conv_1x1_2 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]))
self.bn_2 = nn.BatchNorm2d(num_features=self.conv_1x1_2.out_channels)
 self.conv_1x1_3 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]/2))
self.bn_3 = nn.BatchNorm2d(num_features=self.conv_1x1_3.out_channels)
 self.conv_1x1_4 = nn.Conv2d(kernel_size=1, in_channels=in_shape[0], out_channels=int(in_shape[0]/2))
self.bn_4 = nn.BatchNorm2d(num_features=self.conv_1x1_4.out_channels)
 self.conv_3x3_1 = nn.Conv2d(kernel_size=3, in_channels=self.conv_1x1_2.out_channels, out_channels=self.conv_1x1_1.out_channels * 2, padding=1)
 self.bn_5 = nn.BatchNorm2d(num_features=self.conv_3x3_1.out_channels)
 self.conv_3x3_2 = nn.Conv2d(kernel_size=3, in_channels=self.conv_1x1_3.out_channels, out_channels=int(in_shape[0]/4), padding=1)
 self.bn_6 = nn.BatchNorm2d(num_features=self.conv_3x3_2.out_channels)
 self.conv_3x3_3 = nn.Conv2d(kernel_size=3, in_channels=self.conv_3x3_2.out_channels, out_channels=int(in_shape[0]/4), padding=1)
 self.bn_7 = nn.BatchNorm2d(num_features=self.conv_3x3_3.out_channels)
 self.pool_3x3 = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)
 self.output_shape = self._get_out_shape(in_shape)
 def forward(self, x):
x_1 = self.conv_1x1_1(x)
 x_1 = F.relu(x_1)
 x_1 = self.bn_1(x_1)
 x_2 = self.conv_1x1_2(x)
 x_2 = F.relu(x_2)
 x_2 = self.bn_2(x_2)
 x_2 = self.conv_3x3_1(x_2)
 x_2 = F.relu(x_2)
 x_2 = self.bn_5(x_2)
 x_3 = self.conv_1x1_3(x)
 x_3 = F.relu(x_3)
 x_3 = self.bn_3(x_3)
 x_3 = self.conv_3x3_2(x_3)
 x_3 = F.relu(x_3)
 x_3 = self.bn_6(x_3)
 x_3 = self.conv_3x3_3(x_3)
 x_3 = F.relu(x_3)
 x_3 = self.bn_7(x_3)
 x_4 = self.pool_3x3(x)
 x_4 = self.conv_1x1_4(x_4)
 x_4 = F.relu(x_4)
 x_4 = self.bn_4(x_4)
 if self.residual:
x = torch.cat([x_1, x_2, x_3, x_4, x], dim=1)
 else:
x = torch.cat([x_1, x_2, x_3, x_4], dim=1)
 return x
def _get_out_shape(self, input_shape):
return list(self.forward(torch.zeros([1] + input_shape)).shape[1:])
 class Conv2D_BN(nn.Module):
def __init__(self, in_channels, out_channels, kernel=3, stride=1, padding=1, activation=True):

 super(Conv2D_BN, self ).__init__()
self.activation = activation
self.conv = nn.Conv2d(in_channels =in_channels
, out_channels=out_channels
, kernel_size =kernel
, stride =stride
, padding =padding
, bias =True 
)
 self.bn = nn.BatchNorm2d(num_features=out_channels, momentum=0.9)
 def forward(self, x):
x = self.conv(x)
 if self.activation:
x = F.relu(x)
 x = self.bn(x)
 return x
def block_forward(layers, x):
for layer in layers:
x = layer(x)
 return x
def Mish(x):
r
return x * torch.tanh(F.softplus(x))
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<prepare_x_and_y>
dataset = KannadaMNIST(image_folder="images", labels=label_df, transforms=transforms.ToTensor())
 loader = torch.utils.data.DataLoader(dataset =dataset
, batch_size =256
, shuffle =False
, pin_memory =True
)
def get_oof_predictions(fold,seed):
seed_everything(seed)
 test_ = process_data(test)
 model = Model(
 num_gen_features=len(gen_cols + pca_gen_cols + ohe_cols),
num_cell_features=len(cell_cols + pca_cell_cols + ohe_cols),
num_targets=len(target_cols),
hidden_size=hidden_size)
 model.to(DEVICE)
 model.load_state_dict(torch.load(f".. /input/jp-multiheads/FOLD{fold}_SEED{seed}.pth"))
gen_x_test, cell_x_test = test_[gen_cols + pca_gen_cols + ohe_cols].values, test_[cell_cols + pca_cell_cols + ohe_cols].values
testdataset = TestDataset(gen_x_test, cell_x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 print('predict on test...', end='')
 predictions = np.zeros(( len(test_), targets.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 print(' done.
 ')
 return predictions<categorify>
for sample in loader:
test = sample["image"][0]
break
test = transforms.functional.to_pil_image(test)
 test
def run_k_fold(NFOLDS, seed):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
pred_ = get_oof_predictions(fold, seed)
 predictions += pred_ / NFOLDS
return predictions<categorify>
net = KannadaNet(input_shape=[1, 28, 28]
, n_class =10 
)
 net.load_state_dict(torch.load(".. /input/pytorch-kannada-v2/CustomNetV2 Submission Weights.tar"), strict=True)
 net.cuda()
net.eval()
def InferenceMultiHeads(SEED):
predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
predictions_ = run_k_fold(NFOLDS, seed)
 predictions += predictions_ / len(SEED)
 print(predictions.shape)
 return predictions
predictions = InferenceMultiHeads(SEED )<create_dataframe>
label_df['label'] = predictions
label_df['id'] = ids
label_df = label_df.sort_values("id")
 print(label_df.head())
 label_df.to_csv("submission.csv",index=False )
predictions_clip = predictions
predictions_clip = pd.DataFrame(predictions_clip, columns=target_cols )<load_pretrained>
shutil.rmtree("images" )