Kaggle competition: iMet Collection 2019 - FGVC6
transforms_test = albumentations.Compose([
Resize(image_size, image_size),
Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2()
] )<load_from_csv>
def seed_all(seed=27):

 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_all(27 )
test = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv')
 test['file_path'] = test.StudyInstanceUID.apply(lambda x: os.path.join('.. /input/ranzcr-clip-catheter-line-classification/test', f'{x}.jpg'))
target_cols = test.iloc[:, 1:12].columns.tolist()
test_dataset = RANZCRDataset(test, 'test', transform=transforms_test)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=24 )<choose_model_class>
BATCH_SIZE = 128
NUM_EPOCHS = 20
PERCENTILE = 99.7
LEARNING_RATE = 0.0001
DISABLE_TQDM = True
if submit:
test_preds = []
for i in range(len(enet_type)) :
if enet_type[i] == 'resnet200d':
print('resnet200d loaded')
 model = RANZCRResNet200D(enet_type[i], out_dim=len(target_cols))
model = model.to(device)
 model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))
if tta:
test_preds += [tta_inference_func(test_loader)]
else:
test_preds += [inference_func(test_loader)]
submission = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv')
 submission[target_cols] = np.mean(test_preds, axis=0)
 submission.to_csv('submission.csv', index=False)
 else:
pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv' ).to_csv('submission.csv', index=False )<load_from_csv>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )
if fast_sub:
pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv',usecols=[0],index_col=0 ).join(pd.read_csv(fast_sub_path ).set_index('StudyInstanceUID')).fillna(0 ).to_csv('submission.csv' )<import_modules>
df = pd.read_csv(labels_csv)
 attribute_dict = dict(zip(df.attribute_id,df.attribute_name))
del df,labels_csv
SQRT2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
build_model, get_cosine_annealing_lr_callback, get_fold_indices, DataGenerator, make_lungmask)
 Laplace_metric, Laplace_log_likelihood, experimental_loss_function)
 WANDB = False
SUBMIT = True
TRAIN_ON_BACKWARD_WEEKS = False
PSEUDO_TEST_PATIENTS = 0<define_variables>
tag_count = 0
culture_count = 0
for idx,data in attribute_dict.items() :
if data.split("::")[0] == 'tag':
tag_count+=1
if data.split("::")[0] == 'culture':
culture_count+=1
print('total_categories: {0}
tag_categories: {1}
culture_categories: {2} ' \ 
.format(len(attribute_dict),tag_count,culture_count))
assert tag_count+culture_count == len(attribute_dict)
 output_dim = len(attribute_dict )
if SUBMIT:
PSEUDO_TEST_PATIENTS = 0
WANDB = False<install_modules>
df = pd.read_csv(train_csv)
 labels_dict = dict(zip(df.id,df.attribute_ids))
if WANDB:
user_secrets = UserSecretsClient()
wandb_key = user_secrets.get_secret("wandb_key")
 assert wandb_key, "Please create a key.txt or Kaggle Secret with your W&B API key"
!pip install wandb -qqq --upgrade
!wandb login $wandb_key<define_variables>
idx = len(os.listdir(train_dir))
number = randint(0,idx)
 image_name = os.listdir(train_dir)[number]
def imshow(image):
plt.figure(figsize=(6, 6))
plt.imshow(image)
 plt.show()
x = Image.open(os.path.join(train_dir,image_name))
for i in labels_dict[os.listdir(train_dir)[number].split('.')[0]].split() :
print(attribute_dict[int(i)])
 np.array(x ).shape
imshow(x )
PREDICT_SLOPE = False
USE_IMAGES = False
APPLY_LUNGMASK = False
DIM = 224
IMG_FEATURES = 22
EFFNET = 0
USE_THREE_LAYERS = True
OPTIMAL_SIGMA_LOSS = False
COSINE_CYCLES = 5
DROP_OUT_RATE = 0
DROP_OUT_LAYERS = []
L2_REGULARIZATION = False
REGULARIZATION_CONSTANT = 0.0001<define_variables>
data_transforms = transforms.Compose([
transforms.Resize(( 224,224)) ,
transforms.ToTensor() ,
] )
FOLDS = 5
BATCH_SIZE = 128
NUMBER_FEATURES = 10<define_variables>
class ImageData(data.Dataset):
def __init__(self,df,dirpath,transform,test = False):
self.df = df
self.test = test
self.dirpath = dirpath
self.conv_to_tensor = transform
if not self.test:
self.image_arr = np.asarray(str(self.dirpath)+'/'+self.df.iloc[:, 0]+'.png')
 else:
self.image_arr = np.asarray(str(self.dirpath)+'/'+self.df.iloc[:, 0])
 if not self.test:
self.label_df = self.df.iloc[:,1]
self.data_len = len(self.df.index)
 def __len__(self):
return self.data_len
def __getitem__(self, idx):
image_name = self.image_arr[idx]
img = Image.open(image_name)
 img_tensor = self.conv_to_tensor(img)
 if not self.test:
image_labels = self.label_df[idx]
label_tensor = torch.zeros(( 1, output_dim))
for label in image_labels.split() :
label_tensor[0, int(label)] = 1
image_label = torch.tensor(label_tensor,dtype= torch.float32)
 return(img_tensor,image_label.squeeze())
 return(img_tensor )
TTA_STEPS = 1
TTA_MULTIPLIER = 0
HIDDEN_LAYERS = [32,32]
NOISE_SDS = [10,10,1] + [500, 0, 10] + [0.25] + 3*[0.25]
GAUSSIAN_NOISE_FVC_CORRELATED = True
ADD_NOISE_FVC_TO_PERCENT = True
ACTIVATION_FUNCTION = 'swish'
LOSS_MODIFICATION = 1
BATCH_NORMALIZATION = False
PRE_BATCH_NORMALIZATION = False
BATCH_RENORMALIZATION = False
EPOCHS = 500
INPUT_NORMALIZATION = True
OUTPUT_NORMALIZATION = True
NEGATIVE_NORMALIZATION = False
LEARNING_RATE_SCHEDULER = 'exp'
MAX_LEARNING_RATE = 0.001
EPOCHS_PER_OOM_DECAY = 300
MODEL_NAME = "TestNoPercentNoOverfit"
config = dict(NUMBER_FEATURES = NUMBER_FEATURES, L2_REGULARIZATION = L2_REGULARIZATION, INPUT_NORMALIZATION = INPUT_NORMALIZATION, BATCH_RENORMALIZATION = BATCH_RENORMALIZATION,
ACTIVATION_FUNCTION = ACTIVATION_FUNCTION, DROP_OUT_RATE = DROP_OUT_RATE, OUTPUT_NORMALIZATION = OUTPUT_NORMALIZATION, PRE_BATCH_NORMALIZATION = PRE_BATCH_NORMALIZATION,
EPOCHS = EPOCHS, MAX_LEARNING_RATE = MAX_LEARNING_RATE, LOSS_MODIFICATION = LOSS_MODIFICATION, NOISE_SDS = NOISE_SDS, OPTIMAL_SIGMA_LOSS = OPTIMAL_SIGMA_LOSS,
COSINE_CYCLES = COSINE_CYCLES, MODEL_NAME=MODEL_NAME, LEARNING_RATE_SCHEDULER = LEARNING_RATE_SCHEDULER, PREDICT_SLOPE = PREDICT_SLOPE,
HIDDEN_LAYERS = HIDDEN_LAYERS, REGULARIZATION_CONSTANT = REGULARIZATION_CONSTANT, EPOCHS_PER_OOM_DECAY = EPOCHS_PER_OOM_DECAY,
DROP_OUT_LAYERS = DROP_OUT_LAYERS, BATCH_SIZE = BATCH_SIZE, GAUSSIAN_NOISE_FVC_CORRELATED = GAUSSIAN_NOISE_FVC_CORRELATED, TTA_STEPS = TTA_STEPS,
ADD_NOISE_FVC_TO_PERCENT = ADD_NOISE_FVC_TO_PERCENT, NEGATIVE_NORMALIZATION = NEGATIVE_NORMALIZATION, BATCH_NORMALIZATION = BATCH_NORMALIZATION,
APPLY_LUNGMASK = APPLY_LUNGMASK, USE_IMAGES = USE_IMAGES, DIM = DIM, IMG_FEATURES = IMG_FEATURES, EFFNET = EFFNET, TTA_MULTIPLIER = TTA_MULTIPLIER )<save_to_csv>
df = pd.read_csv(train_csv)
 train_df,val_df = train_test_split(df, test_size=0.20)
 train_df = train_df.reset_index(drop=True)
 val_df = val_df.reset_index(drop=True)
 print(f"Validation_Data Length: {len(val_df)}
Train_Data Length: {len(train_df)}" )
if SUBMIT:
test_data, submission = get_test_data(".. /input/osic-pulmonary-fibrosis-progression/test.csv")
 test_data["Percent"] = 0
train_data, train_images, train_labels = get_train_data('.. /input/osic-pulmonary-fibrosis-progression/train.csv', PSEUDO_TEST_PATIENTS, TRAIN_ON_BACKWARD_WEEKS, USE_IMAGES, APPLY_LUNGMASK, DIM)
 train_data["Percent"] = 0
np.save("train_data.npy", train_data.to_numpy())
 np.save("train_images.npy", np.array(train_images))
np.save("train_labels.npy", train_labels.to_numpy())
 if PSEUDO_TEST_PATIENTS > 0:
test_data, test_check = get_pseudo_test_data('.. /input/osic-pulmonary-fibrosis-progression/train.csv', PSEUDO_TEST_PATIENTS, INPUT_NORMALIZATION )<feature_engineering>
train_dataset = ImageData(train_df,train_dir,data_transforms)
 train_loader = data.DataLoader(dataset=train_dataset,batch_size=BATCH_SIZE,shuffle=False)
 val_dataset = ImageData(val_df,train_dir,data_transforms)
 val_loader = data.DataLoader(dataset=val_dataset,batch_size=BATCH_SIZE,shuffle=False)
 test_df = pd.DataFrame(os.listdir(test_dir))
test_dataset = ImageData(test_df,test_dir,data_transforms,test = True)
 test_loader = data.DataLoader(dataset=test_dataset,batch_size=BATCH_SIZE,shuffle=False)
 dataloaders_dict = {'train':train_loader, 'val':val_loader}
fold_pos = get_fold_indices(FOLDS, train_data)
 print(fold_pos )<choose_model_class>
resnet_cls = models.resnet50()
resnet_cls.load_state_dict(torch.load(resnet_weights_path))
class AvgPool(nn.Module):
def forward(self, x):
return F.avg_pool2d(x, x.shape[2:])
 class ResNet50(nn.Module):
def __init__(self,num_outputs):
super(ResNet50,self ).__init__()
self.resnet = resnet_cls
layer4 = self.resnet.layer4
self.resnet.layer4 = nn.Sequential(
 nn.Dropout(0.5),
layer4 
)
 self.resnet.avgpool = AvgPool()
self.resnet.fc = nn.Linear(2048, num_outputs)
 for param in self.resnet.parameters() :
param.requires_grad = False
for param in self.resnet.layer4.parameters() :
param.requires_grad = True
for param in self.resnet.fc.parameters() :
param.requires_grad = True
def forward(self,x):
out = self.resnet(x)
 return out
NeuralNet = ResNet50(num_outputs = output_dim )
predictions = []
for fold in range(FOLDS):
train_ID = list(range(fold_pos[0],fold_pos[fold])) + list(range(fold_pos[fold+1],fold_pos[-1]))
val_ID = list(range(fold_pos[fold], fold_pos[fold+1]))
training_generator = DataGenerator(train_ID, config)
 validation_generator = DataGenerator(val_ID, config, validation = True)
 model = build_model(config)
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 callbacks = [sv]
if LEARNING_RATE_SCHEDULER == 'exp':
callbacks.append(get_exponential_decay_lr_callback(config))
if LEARNING_RATE_SCHEDULER == 'cos':
callbacks.append(get_cosine_annealing_lr_callback(config))
print(fold+1, "of", FOLDS)
 if WANDB:
name = MODEL_NAME + '-F{}'.format(fold+1)
 config.update({'fold': fold+1})
 wandb.init(project="pulfib", name = name, config=config)
 wandb_cb = WandbCallback()
callbacks.append(wandb_cb)
 history = model.fit(training_generator, validation_data = validation_generator, epochs = EPOCHS,
verbose = 0, callbacks = callbacks)
 if SUBMIT or PSEUDO_TEST_PATIENTS > 0:
model.load_weights('fold-%i.h5'%fold)
 TTA_test_data = TTA_on_test(test_data.to_numpy() , config)
 for j in range(TTA_STEPS):
predictions.append(model.predict(TTA_test_data[:,:,j], batch_size = 256))
if WANDB:
wandb.join()<save_to_csv>
total_params = sum(p.numel() for p in NeuralNet.parameters())
 print(f'{total_params:,} total parameters.')
 total_trainable_params = sum(p.numel() for p in NeuralNet.parameters() if p.requires_grad)
 print(f'{total_trainable_params:,} training parameters.' )
if SUBMIT:
predictions = np.abs(predictions)
 predictions[:,:,1] = np.power(predictions[:,:,1],2)
 predictions = np.mean(predictions, axis = 0)
 predictions[:,1] = np.power(predictions[:,1],0.5)
 for i in range(1,len(test_data)+1):
submission.loc[i,"FVC"] = predictions[i-1,0]
submission.loc[i, "Confidence"] = predictions[i-1,1]
submission.to_csv("submission.csv", index = False)
<feature_engineering>
print("TRAINING")
 print("training examples: ",len(train_dataset))
print("batch size: ",BATCH_SIZE)
 print("batches available: ",len(train_loader))
print()
print("TESTING")
 print("validation examples: ",len(val_dataset))
print("batch size: ",BATCH_SIZE)
 print("batches available: ",len(val_loader))
print()
print("VALIDATION")
 print("testing examples: ",len(test_dataset))
print("batch size: ",BATCH_SIZE)
 print("batches available: ",len(test_loader))
if PSEUDO_TEST_PATIENTS > 0:
quadraticmeans = []
for j in range(0,11):
result = []
for i in range(-20,20):
postprocess = np.abs(predictions[j])
 if i == 0:
postprocess[:,:,1] = gmean(postprocess[:,:,1], axis = 0)
 postprocess = np.mean(postprocess, axis = 0)
 else:
postprocess[:,:,1] = np.power(postprocess[:,:,1],i)
 postprocess = np.mean(postprocess, axis = 0)
 postprocess[:,1] = np.power(postprocess[:,1],1/i)
 FVC_true = test_check["TargetFVC"].values
FVC_pred = postprocess[:,0]
sigma = postprocess[:,1]
sigma_clip = np.maximum(np.abs(sigma), 70)
 delta = np.abs(FVC_true - FVC_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 loss =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip * sq2)
 result.append(np.mean(loss))
if i == 2:
quadraticmeans.append(result[-1])
 plt.plot(np.arange(-20,20),result)
 plt.show()
plt.plot(0.1*np.arange(0,11),quadraticmeans)
 plt.show()<import_modules>
NeuralNet = NeuralNet.to(device)
 optimizer = optim.Adam(NeuralNet.parameters() ,lr = LEARNING_RATE)
 loss_func = torch.nn.BCEWithLogitsLoss()
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience = 2)
 best_loss = np.inf
for epoch in range(NUM_EPOCHS):
for phase in ['train', 'val']:
start_time = time.time()
if phase == 'train':
NeuralNet.train()
else:
NeuralNet.eval()
running_loss = 0.0
for images_batch, labels_batch in tqdm(dataloaders_dict[phase],disable = DISABLE_TQDM):
images_batch = images_batch.to(device)
 labels_batch = labels_batch.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
pred_batch = NeuralNet(images_batch)
 loss = loss_func(pred_batch,labels_batch)
 if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * images_batch.size(0)
 epoch_loss = running_loss / len(dataloaders_dict[phase].dataset)
 if phase == 'val' and epoch_loss < best_loss:
print("model val_loss Improved from {:.8f} to {:.8f}".format(best_loss,epoch_loss))
best_loss = epoch_loss
best_model_wts = copy.deepcopy(NeuralNet.state_dict())
 if phase == 'val':
scheduler.step(epoch_loss)
 elapsed_time = time.time() -start_time
print("Phase: {} | Epoch: {}/{} | {}_loss:{:.8f} | Time: {:.4f}s".format(phase,
epoch+1,
NUM_EPOCHS,
phase,
epoch_loss,
elapsed_time))
NeuralNet.load_state_dict(best_model_wts )
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from pandas_profiling import ProfileReport<load_from_csv>
NeuralNet.eval()
predictions = np.zeros(( len(test_dataset), output_dim))
i = 0
for test_batch in tqdm(test_loader,disable = DISABLE_TQDM):
test_batch = test_batch.to(device)
 batch_prediction = NeuralNet(test_batch ).detach().cpu().numpy()
predictions[i * BATCH_SIZE:(i+1)* BATCH_SIZE, :] = batch_prediction
i+=1
train=pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv')
 print('Train Data:')
 print(train.head())
 test=pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')
 print('
Test Data:')
 print(test.head())
 sub=pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 print('
Submission File:')
 print(sub.head() )<load_from_csv>
predicted_class_idx = []
for i in range(len(predictions)) :
idx_list = np.where(predictions[i] > np.percentile(predictions[i],PERCENTILE))
predicted_class_idx.append(idx_list[0] )
data_dir='.. /input/osic-pulmonary-fibrosis-progression/train'
patients=os.listdir(data_dir)
 labels_df=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv',index_col=0)
 labels_df.head()<define_variables>
test_df['attribute_ids'] = predicted_class_idx
test_df['attribute_ids'] = test_df['attribute_ids'].apply(lambda x : ' '.join(map(str,list(x))))
test_df = test_df.rename(columns={0: 'id'})
 test_df['id'] = test_df['id'].apply(lambda x : x.split('.')[0])
 test_df.head()
<define_variables><EOS> 
test_df.to_csv('submission.csv',index = False )