Kaggle competition: Digit Recognizer
def inference(models, test_loader, device):
tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for model in models:
with torch.no_grad() :
y_preds1 = model(images)
 y_preds2 = model(images.flip(-1))
y_preds =(y_preds1.sigmoid().to('cpu' ).numpy() + y_preds2.sigmoid().to('cpu' ).numpy())/ 2
avg_preds.append(y_preds)
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<choose_model_class>
class my_params:
seed = 1
batch_size = 256
test_size = 0.1
model = ResNet200D()
model.load_state_dict(torch.load(MODEL_PATH)['model'])
 model.eval()
models = [model.to(device)]<load_pretrained>
train_pd = pd.read_csv(".. /input/digit-recognizer/train.csv", dtype=np.float32)
 final_test = pd.read_csv(".. /input/digit-recognizer/test.csv", dtype=np.float32)
 sample_sub = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 train_pd.info()
test_dataset = TestDataset(test, transform=get_transforms())
 test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False,
num_workers=4 , pin_memory=True)
 predictions = inference(models, test_loader, device )<feature_engineering>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device )
test[target_cols] = predictions
test[target_cols].head(5 )<load_from_csv>
labels_np = train_pd.label.values
features_np = train_pd.loc[:, train_pd.columns != 'label'].values/255
features_train, features_val, labels_train, labels_val = train_test_split(features_np, labels_np,
test_size = my_params.test_size,
random_state = my_params.seed
)
Final_Submission = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv')
 Final_Submission[target_cols] =(test[target_cols]**0.5 + submission[target_cols]**0.5)/2<save_to_csv>
featuresTrain = torch.from_numpy(features_train)
 labelsTrain = torch.from_numpy(labels_train ).type(torch.LongTensor)
 featuresVal = torch.from_numpy(features_val)
 labelsVal = torch.from_numpy(labels_val ).type(torch.LongTensor )
Final_Submission.to_csv("submission.csv", index=False )<define_variables>
train_transforms = transforms.Compose([transforms.ToPILImage() ,
transforms.RandomRotation(degrees=(-10, 10)) ,
transforms.RandomAffine(0, translate=(0.1,0.1)) ,
transforms.ToTensor() ] )
sys.path = [
'.. /input/smp20210127/segmentation_models.pytorch-master/segmentation_models.pytorch-master/',
'.. /input/smp20210127/EfficientNet-PyTorch-master/EfficientNet-PyTorch-master',
'.. /input/smp20210127/pytorch-image-models-master/pytorch-image-models-master',
'.. /input/smp20210127/pretrained-models.pytorch-master/pretrained-models.pytorch-master',
] + sys.path<set_options>
class custom_mnist(Dataset):
def __init__(self, feat_tens, label_tens, transform=None):
self.data = feat_tens.reshape(len(feat_tens), 1, 28, 28)
 self.label_data = label_tens
self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
image = self.data[index]
label = self.label_data[index]
if self.transform is not None:
image = self.transform(image)
 return image, label
%matplotlib inline
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu' )<define_variables>
train_dataset = custom_mnist(featuresTrain, labelsTrain)
 val_dataset = custom_mnist(featuresVal, labelsVal)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size = my_params.batch_size, shuffle = True)
 val_loader = torch.utils.data.DataLoader(val_dataset)

print("train_loader: {}".format(len(train_loader)) , "
val_loader: {}".format(len(val_loader)) )
data_dir = '.. /input/ranzcr-clip-catheter-line-classification'
model_dir = '.. /input/ranzcr-public-model-qishen'
num_workers = 2
image_size = 512
batch_size = 8<load_from_csv>
img, lab = train_dataset.__getitem__(3)
 print("image_shape: ", img.shape)
 print("image_label: ", lab)
 plt.imshow(np.squeeze(img))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
df_sub = df_sub.iloc[:358] if df_sub.shape[0] == 3582 else df_sub<data_type_conversions>
class NeuralNetwork(nn.Module):
def __init__(self):

 super(NeuralNetwork, self ).__init__()
self.features = torch.nn.Sequential(
 nn.Conv2d(in_channels=1, out_channels=32, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(num_features=32),
nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(num_features=32),
nn.MaxPool2d(kernel_size =(2,2), stride=2),
nn.Dropout(0.25),
nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(num_features=64),
nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, padding=1),
nn.ReLU() ,
nn.BatchNorm2d(num_features=64),
nn.MaxPool2d(kernel_size =(2,2), stride=2),
nn.Dropout(0.25),
nn.Flatten() ,
nn.Linear(in_features=3136, out_features=512),
nn.BatchNorm1d(num_features=512),
nn.Dropout(0.25),
nn.Linear(in_features=512, out_features=1024),
nn.BatchNorm1d(num_features=1024),
nn.Dropout(0.50),
nn.Linear(in_features=1024, out_features=10),
nn.ReLU() ,
nn.Softmax(dim=1), 
)
 def forward(self, x):
logits = self.features(x)
 return logits
NeuralNetwork()
class RANZCRDatasetTest(Dataset):
def __init__(self, df):
self.df = df.reset_index(drop=True)
 def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
image = cv2.imread(os.path.join(data_dir, 'test', row.StudyInstanceUID + '.jpg')) [:, :, ::-1]
image1024 = cv2.resize(image ,(1024, 1024)).astype(np.float32 ).transpose(2, 0, 1)/ 255.
 image512 = cv2.resize(image ,(512, 512)).astype(np.float32 ).transpose(2, 0, 1)/ 255.
 return {
'1024': torch.tensor(image1024),
'512': torch.tensor(image512),
}
dataset_test = RANZCRDatasetTest(df_sub)
 test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=batch_size, shuffle=False, num_workers=num_workers )<categorify>
model = NeuralNetwork()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adamax(model.parameters() , lr=0.0001, betas=(0.9, 0.999), eps=1e-08, weight_decay=0.8)
 epochs = 40
steps = 0
train_losses, val_losses, val_accs = [], [], []
early_stop_count = 0
class SegModel(nn.Module):
def __init__(self, backbone):
super(SegModel, self ).__init__()
self.seg = smp.UnetPlusPlus(encoder_name=backbone, encoder_weights=None, classes=2, activation=None)
 def forward(self,x):
global_features = self.seg.encoder(x)
 seg_features = self.seg.decoder(*global_features)
 seg_features = self.seg.segmentation_head(seg_features)
 return seg_features
class enetv2(nn.Module):
def __init__(self, backbone):
super(enetv2, self ).__init__()
self.enet = timm.create_model(backbone, False)
 self.enet.conv_stem.weight = nn.Parameter(self.enet.conv_stem.weight.repeat(1,5//3+1,1,1)[:, :5])
 self.myfc = nn.Linear(self.enet.classifier.in_features, 12)
 self.enet.classifier = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x, mask):
mask = F.interpolate(mask, x.shape[2])
 x = torch.cat([x, mask], 1)
 x = self.extract(x)
 x = self.myfc(x)
 return x<define_variables>
if torch.cuda.is_available() :
model = model.cuda()
criterion = criterion.cuda()
print("GPU Training")
 else:
print("CPU Training" )
enet_type_seg = 'timm-efficientnet-b1'
kernel_type_seg = 'unetb1_2cbce_1024T15tip_lr1e4_bs4_augv2_30epo'
enet_type_cls = 'tf_efficientnet_b1_ns'
kernel_type_cls = 'enetb1_5ch_512_lr3e4_bs32_30epo'<load_pretrained>
print("---- Starting Model Training ----
")
 for e in range(epochs):
if early_stop_count >= 10:
print("
Validation Accuracy not improved for {} epochs.val_acc: {}.".format(early_stop_count, sorted(val_accs)[-1]))
break
print("
-- Epoch {}/{} -- ".format(e+1, epochs))
running_loss = 0
steps = 0
for images, labels in train_loader:
if torch.cuda.is_available() :
images = images.to(device)
 labels = labels.to(device)
 steps += 1
optimizer.zero_grad()
log_ps = model(images)
 loss = criterion(log_ps, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
if steps % len(train_loader)== 0:
val_loss = 0
val_accuracy = 0
train_accuracy = 0
with torch.no_grad() :
model.eval()
for images, labels in val_loader:
if torch.cuda.is_available() :
images = images.to(device)
 labels = labels.to(device)
 log_ps = model(images)
 val_loss += criterion(log_ps, labels)
 ps = torch.exp(log_ps)
 top_p, top_class = ps.topk(1, dim=1)
 equals = top_class == labels.view(*top_class.shape)
 val_accuracy += torch.mean(equals.type(torch.FloatTensor))
for images, labels in train_loader:
if torch.cuda.is_available() :
images = images.to(device)
 labels = labels.to(device)
 log_ps = model(images)
 ps = torch.exp(log_ps)
 top_p, top_class = ps.topk(1, dim=1)
 equals = top_class == labels.view(*top_class.shape)
 train_accuracy += torch.mean(equals.type(torch.FloatTensor))
model.train()
train_losses.append(running_loss/len(train_loader))
val_losses.append(val_loss/len(val_loader))
val_accs.append(val_accuracy/len(val_loader))
if val_accs[-1] == sorted(val_accs)[-1]:
best_model_params = model.state_dict()
early_stop_count = 0
print("+")
 print("Training Loss: {:.4f} ".format(train_losses[-1]),
"Training Acc: {:.4f}".format(train_accuracy/len(train_loader)) ,
"Valid Loss: {:.4f} ".format(val_losses[-1]),
"Valid Accuracy: {:.4f}".format(val_accs[-1]))
early_stop_count += 1
if e ==(epochs-1):
print("
Top Validation Accuracy: {}".format(sorted(val_accs)[-1]))
models_seg = []
for fold in range(5):
model = SegModel(enet_type_seg)
 model = model.to(device)
 model_file = os.path.join(model_dir, f'{kernel_type_seg}_best_fold{fold}.pth')
 model.load_state_dict(torch.load(model_file), strict=True)
 model.eval()
models_seg.append(model)
 models_cls = []
for fold in range(5):
model = enetv2(enet_type_cls)
 model = model.to(device)
 model_file = os.path.join(model_dir, f'{kernel_type_cls}_best_fold{fold}.pth')
 model.load_state_dict(torch.load(model_file), strict=True)
 model.eval()
models_cls.append(model )<create_dataframe>
model.load_state_dict(best_model_params )
df_subs = [df_sub.copy() for _ in range(PROBS.shape[0])]
for i, this_sub in enumerate(df_subs):
this_sub[[
'ETT - Abnormal',
'ETT - Borderline',
'ETT - Normal',
'NGT - Abnormal',
'NGT - Borderline',
'NGT - Incompletely Imaged',
'NGT - Normal',
'CVC - Abnormal',
'CVC - Borderline',
'CVC - Normal',
'Swan Ganz Catheter Present'
]] = PROBS[i]
this_sub[[
'ETT - Abnormal',
'ETT - Borderline',
'ETT - Normal',
'NGT - Abnormal',
'NGT - Borderline',
'NGT - Incompletely Imaged',
'NGT - Normal',
'CVC - Abnormal',
'CVC - Borderline',
'CVC - Normal',
'Swan Ganz Catheter Present'
]] = this_sub[[
'ETT - Abnormal',
'ETT - Borderline',
'ETT - Normal',
'NGT - Abnormal',
'NGT - Borderline',
'NGT - Incompletely Imaged',
'NGT - Normal',
'CVC - Abnormal',
'CVC - Borderline',
'CVC - Normal',
'Swan Ganz Catheter Present'
]].rank(pct=True)
 df_sub[[
'ETT - Abnormal',
'ETT - Borderline',
'ETT - Normal',
'NGT - Abnormal',
'NGT - Borderline',
'NGT - Incompletely Imaged',
'NGT - Normal',
'CVC - Abnormal',
'CVC - Borderline',
'CVC - Normal',
'Swan Ganz Catheter Present'
]] = np.stack([this_sub[[
'ETT - Abnormal',
'ETT - Borderline',
'ETT - Normal',
'NGT - Abnormal',
'NGT - Borderline',
'NGT - Incompletely Imaged',
'NGT - Normal',
'CVC - Abnormal',
'CVC - Borderline',
'CVC - Normal',
'Swan Ganz Catheter Present'
]].values for this_sub in df_subs], 0 ).mean(0)
 df_sub.to_csv('submission.csv', index=False )<set_options>
final_test_np = final_test.values/255
test_labels = np.zeros(final_test_np.shape)
 test_tn = torch.from_numpy(final_test_np)
 test_labels = torch.from_numpy(test_labels )
sys.path.append('.. /input/pytorch-images-seresnet')
 warnings.filterwarnings('ignore')
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<define_variables>
class test_mnist(Dataset):
def __init__(self, feat_tens):
self.data = feat_tens.reshape(len(feat_tens), 1, 28, 28)
 def __len__(self):
return len(self.data)
 def __getitem__(self, index):
image = self.data[index]
return image
submission_dataset = test_mnist(test_tn)
 submission_loader = torch.utils.data.DataLoader(submission_dataset, batch_size = 256, shuffle = False )
<load_from_csv><EOS> 
submission = [['ImageId', 'Label']]
with torch.no_grad() :
model.eval()
image_id = 1
for images in submission_loader:
if torch.cuda.is_available() :
images = images.to(device)
 log_ps = model(images)
 ps = torch.exp(log_ps)
 top_p, top_class = ps.topk(1, dim=1)
 for prediction in top_class:
submission.append([image_id, prediction.item() ])
 image_id += 1
submission_df = pd.DataFrame(submission)
 submission_df.columns = submission_df.iloc[0]
submission_df = submission_df.drop(0, axis=0)
 submission_df.to_csv("submission.csv", index=False )