Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<save_to_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import linear_model, ensemble
from sklearn.metrics import mean_squared_error, mean_absolute_error
import tensorflow as tf
from tqdm.notebook import tqdm
import os
from PIL import Image
sample_submission.to_csv("submission.csv", index=False )<set_options>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 submission = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )
warnings.filterwarnings('ignore' )<define_variables>
train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'] )
DATA_FOLDER = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection'
EPOCHES = 2
BSIZE = 6
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'<load_pretrained>
submission['Patient'] =(
 submission['Patient_Week'] 
.apply(
 lambda x:x.split('_')[0] 
)
)
 submission['Weeks'] =(
 submission['Patient_Week'] 
.apply(
 lambda x: int(x.split('_')[-1])
)
)
 submission = submission[['Patient','Weeks', 'Confidence','Patient_Week']]
submission = submission.merge(test.drop('Weeks', axis=1), on="Patient" )
ds = VinBigDataDS(DATA_FOLDER)
 loader = D.DataLoader(ds, batch_size=BSIZE, shuffle=True, num_workers=0,
collate_fn=collate_fn )<choose_model_class>
train['Dataset'] = 'train'
test['Dataset'] = 'test'
submission['Dataset'] = 'submission'
def get_model(num_classes):
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)
 in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
 return model<choose_model_class>
all_data = train.append([test, submission])
 all_data = all_data.reset_index()
all_data = all_data.drop(columns=['index'] )
model = get_model(15)
 model = model.to(DEVICE)
 params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.AdamW(params, lr=1e-3 )<categorify>
all_data['FirstWeek'] = all_data['Weeks']
all_data.loc[all_data.Dataset=='submission','FirstWeek'] = np.nan
all_data['FirstWeek'] = all_data.groupby('Patient')['FirstWeek'].transform('min' )
for epoch in range(1,EPOCHES+1):
model.train()
loss_history = { 'loss_classifier': [],
'loss_box_reg': [],
'loss_objectness': [],
'loss_rpn_box_reg':[]}
for images, targets in tqdm(loader):
images = list(image.to(DEVICE)for image in images)
 targets = [{k: v.to(DEVICE)for k, v in t.items() } for t in targets]
loss_dict = model(images, targets)
 losses = sum(loss for loss in loss_dict.values())
 for k, v in loss_dict.items() :
loss_history[k].append(v.item())
 optimizer.zero_grad()
losses.backward()
optimizer.step()
print(f"Loss classifier: {np.array(loss_history['loss_classifier'] ).mean() :6.2f}, \
Loss BoxReg: {np.array(loss_history['loss_box_reg'] ).mean() :6.2f},\
Loss Objectness: {np.array(loss_history['loss_objectness'] ).mean() :6.2f},\
Loss RPN Box Reg: {np.array(loss_history['loss_rpn_box_reg'] ).mean() :6.2f}" )<categorify>
first_fvc =(
 all_data 
.loc[all_data.Weeks == all_data.FirstWeek][['Patient','FVC']] 
.rename({'FVC': 'FirstFVC'}, axis=1)
.groupby('Patient')
.first() 
.reset_index() 
)
 all_data = all_data.merge(first_fvc, on='Patient', how='left' )
model.eval() ;
filenames = glob.glob(os.path.join(DATA_FOLDER, 'test', '*.dicom'))
outputs = []
for filename in tqdm(filenames):
dfile = dcm.read_file(filename)
 image = dfile.pixel_array
image = historgam(image)
 image = ds.transforms(image.astype(np.float32))
with torch.no_grad() :
prediction = model([image.to(DEVICE)])[0]
line = ''
for i, label in enumerate(prediction['labels']):
if(label < 14)&(prediction['scores'][i].cpu().item() >0.4):
line += str(label.cpu().item())+ ' ' + str(round(prediction['scores'][i].cpu().item() , 4)) + ' '
line += ' '.join([str(round(val.item())) for val in prediction['boxes'][i]])+ ' '
if not line:
line = '14 1.0 0 0 1 1'
outputs.append([filename.split('/')[-1][:-6], line] )<save_to_csv>
all_data['WeeksPassed'] = all_data['Weeks'] - all_data['FirstWeek']
submission = pd.DataFrame(outputs, columns=['image_id', 'PredictionString'])
 submission.to_csv('submission.csv', index=False )<install_modules>
def calculate_height(row):
if row['Sex'] == 'Male':
return row['FirstFVC'] /(27.63 - 0.112 * row['Age'])
 else:
return row['FirstFVC'] /(21.78 - 0.101 * row['Age'])
 all_data['Height'] = all_data.apply(calculate_height, axis=1 )
!pip install efficientnet_pytorch torchtoolbox<set_options>
all_data = pd.concat([
all_data,
pd.get_dummies(all_data.Sex),
pd.get_dummies(all_data.SmokingStatus)
 ], axis=1)
 all_data = all_data.drop(columns=['Sex', 'SmokingStatus'] )
%matplotlib inline<set_options>
def scale_feature(series):
return(series - series.min())/(series.max() - series.min())
 all_data['Weeks'] = scale_feature(all_data['Weeks'])
 all_data['Percent'] = scale_feature(all_data['Percent'])
 all_data['Age'] = scale_feature(all_data['Age'])
 all_data['FirstWeek'] = scale_feature(all_data['FirstWeek'])
 all_data['FirstFVC'] = scale_feature(all_data['FirstFVC'])
 all_data['WeeksPassed'] = scale_feature(all_data['WeeksPassed'])
 all_data['Height'] = scale_feature(all_data['Height'] )
warnings.simplefilter('ignore')
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(47 )<set_options>
feature_columns = [
'Percent',
'Age',
'FirstWeek',
'FirstFVC',
'WeeksPassed',
'Height',
'Female',
'Male',
'Currently smokes',
'Ex-smoker',
'Never smoked',
]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<create_dataframe>
train = all_data.loc[all_data.Dataset == 'train']
test = all_data.loc[all_data.Dataset == 'test']
submission = all_data.loc[all_data.Dataset == 'submission']
class MelanomaDataset(Dataset):
def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):

 self.df = df
self.imfolder = imfolder
self.transforms = transforms
self.train = train
self.meta_features = meta_features
def __getitem__(self, index):
im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')
 x = cv2.imread(im_path)
 meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)
 if self.transforms:
x = self.transforms(x)
 if self.train:
y = self.df.iloc[index]['target']
return(x, meta), y
else:
return(x, meta)
 def __len__(self):
return len(self.df)
 class Net(nn.Module):
def __init__(self, arch, n_meta_features: int):
super(Net, self ).__init__()
self.arch = arch
if 'ResNet' in str(arch.__class__):
self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)
 if 'EfficientNet' in str(arch.__class__):
self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)
 self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),
nn.BatchNorm1d(500),
nn.ReLU() ,
nn.Dropout(p=0.2),
nn.Linear(500, 250),
nn.BatchNorm1d(250),
nn.ReLU() ,
nn.Dropout(p=0.2))
self.ouput = nn.Linear(500 + 250, 1)
 def forward(self, inputs):

 x, meta = inputs
cnn_features = self.arch(x)
 meta_features = self.meta(meta)
 features = torch.cat(( cnn_features, meta_features), dim=1)
 output = self.ouput(features)
 return output<categorify>
model = linear_model.HuberRegressor(max_iter=200 )
class AdvancedHairAugmentation:

 def __init__(self, hairs: int = 5, hairs_folder: str = ""):
self.hairs = hairs
self.hairs_folder = hairs_folder
def __call__(self, img):

 n_hairs = random.randint(0, self.hairs)
 if not n_hairs:
return img
height, width, _ = img.shape
hair_images = [im for im in os.listdir(self.hairs_folder)if 'png' in im]
for _ in range(n_hairs):
hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))
 hair = cv2.flip(hair, random.choice([-1, 0, 1]))
hair = cv2.rotate(hair, random.choice([0, 1, 2]))
h_height, h_width, _ = hair.shape
roi_ho = random.randint(0, img.shape[0] - hair.shape[0])
 roi_wo = random.randint(0, img.shape[1] - hair.shape[1])
 roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 hair_fg = cv2.bitwise_and(hair, hair, mask=mask)
 dst = cv2.add(img_bg, hair_fg)
 img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder="{self.hairs_folder}")'<set_options>
model.fit(train[feature_columns], train['FVC'] )
class DrawHair:

 def __init__(self, hairs:int = 4, width:tuple =(1, 2)) :
self.hairs = hairs
self.width = width
def __call__(self, img):

 if not self.hairs:
return img
width, height, _ = img.shape
for _ in range(random.randint(0, self.hairs)) :
origin =(random.randint(0, width), random.randint(0, height // 2))
end =(random.randint(0, width), random.randint(0, height))
color =(0, 0, 0)
 cv2.line(img, origin, end, color, random.randint(self.width[0], self.width[1]))
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, width={self.width})'<categorify>
predictions = model.predict(train[feature_columns] )
class Microscope:

 def __init__(self, p: float = 0.5):
self.p = p
def __call__(self, img):

 if random.random() < self.p:
circle = cv2.circle(( np.ones(img.shape)* 255 ).astype(np.uint8), 
(img.shape[0]//2, img.shape[1]//2),
random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), 
(0, 0, 0),
-1)
 mask = circle - 255
img = np.multiply(img, mask)
 return img
def __repr__(self):
return f'{self.__class__.__name__}(p={self.p})'<normalization>
mse = mean_squared_error(
 train['FVC'],
predictions,
squared=False 
)
 mae = mean_absolute_error(
 train['FVC'],
predictions 
)
 print('MSE Loss: {0:.2f}'.format(mse))
print('MAE Loss: {0:.2f}'.format(mae))
train_transform = transforms.Compose([
AdvancedHairAugmentation(hairs_folder='/kaggle/input/melanoma-hairs'),
transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
Microscope(p=0.5),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
 ])
 test_transform = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
 ] )<load_pretrained>
def competition_metric(trueFVC, predFVC, predSTD):
clipSTD = np.clip(predSTD, 70 , 9e9)
 deltaFVC = np.clip(np.abs(trueFVC - predFVC), 0 , 1000)
 return np.mean(-1 *(np.sqrt(2)* deltaFVC / clipSTD)- np.log(np.sqrt(2)* clipSTD))
print(
 'Competition metric: ',
competition_metric(train['FVC'].values, predictions, 285)
 )
arch = EfficientNet.from_pretrained('efficientnet-b1' )<load_from_csv>
train['prediction'] = predictions
train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')
 test_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv' )<categorify>
sub_predictions = model.predict(submission[feature_columns])
 submission['FVC'] = sub_predictions
concat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)
 dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')
 train_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)
 test_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)
 train_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})
 test_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})
 train_df['sex'] = train_df['sex'].fillna(-1)
 test_df['sex'] = test_df['sex'].fillna(-1)
 train_df['age_approx'] /= train_df['age_approx'].max()
test_df['age_approx'] /= test_df['age_approx'].max()
train_df['age_approx'] = train_df['age_approx'].fillna(0)
 test_df['age_approx'] = test_df['age_approx'].fillna(0)
 train_df['patient_id'] = train_df['patient_id'].fillna(0 )<drop_column>
submission = submission[['Patient_Week', 'FVC']]
submission['Confidence'] = 285
meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]
meta_features.remove('anatom_site_general_challenge' )<create_dataframe>
submission.to_csv('submission.csv', index=False )
<choose_model_class><EOS> 
submission.to_csv('submission.csv', index=False )