Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<create_dataframe>
%matplotlib inline
BATCH_SIZE = 256
print(os.listdir('.. /input/Kannada-MNIST/'))
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device )
n_comp_GENES = 463
n_comp_CELLS = 60
train_features, test_features = add_PC_to_dataframe(train_features, test_features, n_comp_GENES, n_comp_CELLS, GENES, CELLS )<split>
df_train = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 target = df_train['label']
df_train.drop('label', axis=1, inplace=True)
 X_test = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 X_test.drop('id', axis=1, inplace=True)
 X_test.head()
VarianceThreshold_for_FS = 0.9
train, test, target, target_cols, slected_feats = select_features(train_features, test_features, train_targets_scored, VarianceThreshold_for_FS)
<define_variables>
X_train, X_dev, y_train, y_dev = train_test_split(df_train, target, stratify=target, random_state=42, test_size=0.2)
 print('X_train', len(X_train))
print('X_dev', len(X_dev))
print('X_test', len(X_test))
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols )<merge>
class CharData(Dataset):

 def __init__(self,
images,
labels=None,
transform=None, 
):
self.X = images
self.y = labels
self.transform = transform
def __len__(self):
return len(self.X)
 def __getitem__(self, idx):
img = np.array(self.X.iloc[idx, :], dtype='uint8' ).reshape([28, 28, 1])
 if self.transform is not None:
img = self.transform(img)
 if self.y is not None:
y = np.zeros(10, dtype='float32')
 y[self.y.iloc[idx]] = 1
return img, y
else:
return img
train = train.merge(train_targets_nonscored, on='sig_id' )<categorify>
train_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomAffine(degrees=5, translate=(0.1, 0.1), scale=(0.9, 1.1), shear=5),
transforms.ToTensor()
])
 test_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.ToTensor()
] )
folds = get_folds(train, target )<define_variables>
train_dataset = CharData(X_train, y_train, train_transform)
 dev_dataset = CharData(X_dev, y_dev, test_transform)
 test_dataset = CharData(X_test, transform=test_transform )
feature_cols = [c for c in process_data(folds ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<choose_model_class>
train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)
 dev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)
 test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4 )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size, Dropout_Model):
super(Model, self ).__init__()
self.hidden_size = [hidden_size]
self.dropout_value = [Dropout_Model]
self.batch_norm_0 = nn.BatchNorm1d(num_features)
 self.dense_0 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm_1 = nn.BatchNorm1d(hidden_size)
 self.dropout_1 = nn.Dropout(Dropout_Model)
 self.dense_1 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm_last = nn.BatchNorm1d(hidden_size)
 self.dropout_last = nn.Dropout(Dropout_Model)
 self.dense_last = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def recalibrate_layer(self, layer):
if(torch.isnan(layer.weight_v ).sum() > 0):
print('recalibrate layer.weight_v')
 layer.weight_v = torch.nn.Parameter(torch.where(torch.isnan(layer.weight_v), torch.zeros_like(layer.weight_v), layer.weight_v))
layer.weight_v = torch.nn.Parameter(layer.weight_v + 1e-7)
 if(torch.isnan(layer.weight ).sum() > 0):
print('recalibrate layer.weight')
 layer.weight = torch.where(torch.isnan(layer.weight), torch.zeros_like(layer.weight), layer.weight)
 layer.weight += 1e-7
def forward(self, x):
x = self.batch_norm_0(x)
 self.recalibrate_layer(self.dense_0)
 x = F.leaky_relu(self.dense_0(x))
x = self.batch_norm_1(x)
 x = self.dropout_1(x)
 self.recalibrate_layer(self.dense_1)
 x = F.leaky_relu(self.dense_1(x))
x = self.batch_norm_last(x)
 x = self.dropout_last(x)
 self.recalibrate_layer(self.dense_last)
 x = self.dense_last(x)
 return x<train_model>
DEPTH_MULT = 4
class ConvLayer(nn.Module):
def __init__(self, input_size, output_size, kernel_size=3):
super(ConvLayer, self ).__init__()
self.ops = nn.Sequential(
 nn.Conv2d(input_size, output_size, kernel_size=kernel_size, stride=1, padding=kernel_size//2),
nn.BatchNorm2d(output_size),
nn.ReLU(inplace=True)
)
 def forward(self, x):
return self.ops(x)
 class FCLayer(nn.Module):
def __init__(self, input_size, output_size):
super(FCLayer, self ).__init__()
self.ops = nn.Sequential(
 nn.Linear(input_size, output_size),
nn.BatchNorm1d(output_size),
nn.ReLU(inplace=True)
)
 def forward(self, x):
return self.ops(x)
 def mixup(x, shuffle, lam, i, j):
if shuffle is not None and lam is not None and i == j:
x = lam * x +(1 - lam)* x[shuffle]
return x
class Net(nn.Module):
def __init__(self, num_classes):
super(Net, self ).__init__()
self.conv1 = ConvLayer(1, DEPTH_MULT * 32)
 self.conv2 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)
 self.conv3 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)
 self.conv4 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)
 self.conv5 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 64)
 self.conv6 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.conv7 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.conv8 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.conv9 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.conv10 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)
 self.mp = nn.MaxPool2d(kernel_size=2, stride=2)
 self.fc1 = FCLayer(DEPTH_MULT * 64 * 7 * 7, DEPTH_MULT * 512)
 self.fc2 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)
 self.fc3 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)
 self.fc4 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)
 self.projection = nn.Linear(DEPTH_MULT * 512, 10)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = self.conv3(x)
 x = self.conv4(x)
 x = self.mp(x)
 x = self.conv5(x)
 x = self.conv6(x)
 x = self.conv7(x)
 x = self.conv8(x)
 x = self.conv9(x)
 x = self.conv10(x)
 x = self.mp(x)
 x = x.view(x.size(0), -1)
 x = self.fc1(x)
 x = self.fc2(x)
 x = self.fc3(x)
 x = self.fc4(x)
 x = self.projection(x)
 return x
def run_training(fold, seed, folds, model, feature_cols, target_cols, target, BATCH_SIZE, DEVICE, LEARNING_RATE, WEIGHT_DECAY, EPOCHS, EARLY_STOPPING_STEPS, EARLY_STOP):
seed_everything(seed)
 train = process_data(folds)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_{seed}.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
return oof
def run_predicition(fold, model, test, feature_cols, DEVICE, BATCH_SIZE):
test_ = process_data(test)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model.load_state_dict(torch.load(f"FOLD{fold}_{seed}.pth"))
model.to(DEVICE)
 predictions = inference_fn(model, testloader, DEVICE)
 return predictions<prepare_x_and_y>
def criterion(input, target, size_average=True):

 l = -(target * torch.log(F.softmax(input, dim=1)+ 1e-10)).sum(1)
 if size_average:
l = l.mean()
else:
l = l.sum()
return l
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold}_{seed}.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(
 num_features=num_features,
num_targets=num_all_targets,
hidden_size=hidden_size,
Dropout_Model=Dropout_Model)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model =Model(
 num_features=num_features,
num_targets=num_all_targets,
hidden_size=hidden_size,
Dropout_Model=Dropout_Model)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold}_{seed}.pth"))
pretrained_model.to(DEVICE)
 model_new = Model(
 num_features=num_features,
num_targets=num_all_targets,
hidden_size=hidden_size,
Dropout_Model=Dropout_Model)
 model_new.load_state_dict(pretrained_model.state_dict())
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, model_new, num_targets, DEVICE)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 return oof
def run_predicition(fold, model, test, feature_cols, DEVICE, BATCH_SIZE):
test_ = process_data(test)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold}_{seed}.pth"))
model.to(DEVICE)
 predictions = inference_fn(model, testloader, DEVICE)
 return predictions<define_variables>
model = Net(10)
 model = model.to(device)
 n_epochs = 80
optimizer = optim.Adam(model.parameters() , lr=0.001)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=n_epochs // 4, gamma=0.1 )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
Dropout_Model = 0.25
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size = 1550<define_variables>
def train(epoch, history=None):
model.train()
for batch_idx,(data, target)in enumerate(train_loader):
data = data.to(device)
 target = target.to(device)
 optimizer.zero_grad()
output = model(data)
 loss = criterion(output, target)
 if history is not None:
history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()
loss.backward()
optimizer.step()
if(batch_idx + 1)% 100 == 0:
print('Train Epoch: {} [{}/{}({:.0f}%)]\tLR: {:.6f}\tLoss: {:.6f}'.format(
 epoch,(batch_idx + 1)* len(data), len(train_loader.dataset),
100.*(batch_idx + 1)/ len(train_loader),
optimizer.state_dict() ['param_groups'][0]['lr'],
loss.data))
exp_lr_scheduler.step()
def evaluate(epoch, history=None):
model.eval()
loss = 0
correct = 0
with torch.no_grad() :
for data, target in dev_loader:
data = data.to(device)
 target = target.to(device)
 output = model(data)
 loss += criterion(output, target, size_average=False ).data
pred = output.data.max(1, keepdim=True)[1]
correct += pred.eq(target.max(1, keepdim=True)[1].data.view_as(pred)).cpu().sum().numpy()
loss /= len(dev_loader.dataset)
 accuracy = correct / len(dev_loader.dataset)
 if history is not None:
history.loc[epoch, 'dev_loss'] = loss.cpu().numpy()
history.loc[epoch, 'dev_accuracy'] = accuracy
print('Dev loss: {:.4f}, Dev accuracy: {}/{}({:.3f}%)
 '.format(
 loss, correct, len(dev_loader.dataset),
100.* accuracy))
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 28
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1<train_model>
model.eval()
predictions = []
for data in tqdm(test_loader):
data = data.to(device)
 output = model(data ).max(dim=1)[1]
predictions += list(output.data.cpu().numpy() )
<import_modules><EOS> 
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 submission['label'] = predictions
submission.to_csv('submission.csv', index=False)
 submission.head()