Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<find_best_model_class>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
epochs = 12
es_patience = 3
TTA = 3
oof = np.zeros(( len(train_df), 1))
preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 skf = KFold(n_splits=5, shuffle=True, random_state=47)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X=np.zeros(len(train_df)) , y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):
print('=' * 20, 'Fold', fold, '=' * 20)
 model_path = f'model_{fold}.pth'
best_val = 0
patience = es_patience
arch = EfficientNet.from_pretrained('efficientnet-b1')
 model = Net(arch=arch, n_meta_features=len(meta_features))
model = model.to(device)
 optim = torch.optim.Adam(model.parameters() , lr=0.001)
 scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)
 criterion = nn.BCEWithLogitsLoss()
train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=train_transform,
meta_features=meta_features)
 val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=test_transform,
meta_features=meta_features)
 train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)
 val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)
 test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)
 for epoch in range(epochs):
start_time = time.time()
correct = 0
epoch_loss = 0
model.train()
for x, y in train_loader:
x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)
 x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)
 y = torch.tensor(y, device=device, dtype=torch.float32)
 optim.zero_grad()
z = model(x)
 loss = criterion(z, y.unsqueeze(1))
loss.backward()
optim.step()
pred = torch.round(torch.sigmoid(z))
correct +=(pred.cpu() == y.cpu().unsqueeze(1)).sum().item()
epoch_loss += loss.item()
train_acc = correct / len(train_idx)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))
val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())
 print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(
 epoch + 1,
epoch_loss,
train_acc,
val_acc,
val_roc,
str(datetime.timedelta(seconds=time.time() - start_time)) [:7]))
scheduler.step(val_roc)
 if val_roc >= best_val:
best_val = val_roc
patience = es_patience
torch.save(model, model_path)
 else:
patience -= 1
if patience == 0:
print('Early stopping.Best Val roc_auc: {:.3f}'.format(best_val))
break
model = torch.load(model_path)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
oof[val_idx] = val_preds.cpu().numpy()
tta_preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 for _ in range(TTA):
for i, x_test in enumerate(test_loader):
x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)
 x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)
 z_test = model(x_test)
 z_test = torch.sigmoid(z_test)
 tta_preds[i*test_loader.batch_size:i*test_loader.batch_size + x_test[0].shape[0]] += z_test
preds += tta_preds / TTA
preds /= skf.n_splits<compute_test_metric>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)) )<save_to_csv>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
pd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False )<save_to_csv>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 sub['target'] = preds.cpu().numpy().reshape(-1,)
 sub.to_csv('submission.csv', index=False )<install_modules>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
!pip install efficientnet_pytorch torchtoolbox<set_options>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
%matplotlib inline<set_options>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
warnings.simplefilter('ignore')
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(47 )<set_options>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.5 )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b6']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<create_dataframe>
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size= 0.8 )
class MelanomaDataset(Dataset):
def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):

 self.df = df
self.imfolder = imfolder
self.transforms = transforms
self.train = train
self.meta_features = meta_features
def __getitem__(self, index):
im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')
 x = cv2.imread(im_path)
 meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)
 if self.transforms:
x = self.transforms(x)
 if self.train:
y = self.df.iloc[index]['target']
return(x, meta), y
else:
return(x, meta)
 def __len__(self):
return len(self.df)
 class Net(nn.Module):
def __init__(self, arch, n_meta_features: int):
super(Net, self ).__init__()
self.arch = arch
if 'ResNet' in str(arch.__class__):
self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)
 if 'EfficientNet' in str(arch.__class__):
self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)
 self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),
nn.BatchNorm1d(500),
nn.ReLU() ,
nn.Dropout(p=0.2),
nn.Linear(500, 250),
nn.BatchNorm1d(250),
nn.ReLU() ,
nn.Dropout(p=0.2))
self.ouput = nn.Linear(500 + 250, 1)
 def forward(self, inputs):

 x, meta = inputs
cnn_features = self.arch(x)
 meta_features = self.meta(meta)
 features = torch.cat(( cnn_features, meta_features), dim=1)
 output = self.ouput(features)
 return output<categorify>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
class AdvancedHairAugmentation:

 def __init__(self, hairs: int = 5, hairs_folder: str = ""):
self.hairs = hairs
self.hairs_folder = hairs_folder
def __call__(self, img):

 n_hairs = random.randint(0, self.hairs)
 if not n_hairs:
return img
height, width, _ = img.shape
hair_images = [im for im in os.listdir(self.hairs_folder)if 'png' in im]
for _ in range(n_hairs):
hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))
 hair = cv2.flip(hair, random.choice([-1, 0, 1]))
hair = cv2.rotate(hair, random.choice([0, 1, 2]))
h_height, h_width, _ = hair.shape
roi_ho = random.randint(0, img.shape[0] - hair.shape[0])
 roi_wo = random.randint(0, img.shape[1] - hair.shape[1])
 roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 hair_fg = cv2.bitwise_and(hair, hair, mask=mask)
 dst = cv2.add(img_bg, hair_fg)
 img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder="{self.hairs_folder}")'<set_options>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
class DrawHair:

 def __init__(self, hairs:int = 4, width:tuple =(1, 2)) :
self.hairs = hairs
self.width = width
def __call__(self, img):

 if not self.hairs:
return img
width, height, _ = img.shape
for _ in range(random.randint(0, self.hairs)) :
origin =(random.randint(0, width), random.randint(0, height // 2))
end =(random.randint(0, width), random.randint(0, height))
color =(0, 0, 0)
 cv2.line(img, origin, end, color, random.randint(self.width[0], self.width[1]))
return img
def __repr__(self):
return f'{self.__class__.__name__}(hairs={self.hairs}, width={self.width})'<categorify>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
class Microscope:

 def __init__(self, p: float = 0.5):
self.p = p
def __call__(self, img):

 if random.random() < self.p:
circle = cv2.circle(( np.ones(img.shape)* 255 ).astype(np.uint8), 
(img.shape[0]//2, img.shape[1]//2),
random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), 
(0, 0, 0),
-1)
 mask = circle - 255
img = np.multiply(img, mask)
 return img
def __repr__(self):
return f'{self.__class__.__name__}(p={self.p})'<normalization>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
train_transform = transforms.Compose([
AdvancedHairAugmentation(hairs_folder='/kaggle/input/melanoma-hairs'),
transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
Microscope(p=0.5),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
 ])
 test_transform = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
 ] )<load_pretrained>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
arch = EfficientNet.from_pretrained('efficientnet-b1' )<load_from_csv>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')
 test_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv' )<categorify>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
concat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)
 dummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')
 train_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)
 test_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)
 train_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})
 test_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})
 train_df['sex'] = train_df['sex'].fillna(-1)
 test_df['sex'] = test_df['sex'].fillna(-1)
 train_df['age_approx'] /= train_df['age_approx'].max()
test_df['age_approx'] /= test_df['age_approx'].max()
train_df['age_approx'] = train_df['age_approx'].fillna(0)
 test_df['age_approx'] = test_df['age_approx'].fillna(0)
 train_df['patient_id'] = train_df['patient_id'].fillna(0 )<drop_column>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]
meta_features.remove('anatom_site_general_challenge' )<create_dataframe>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
test = MelanomaDataset(df=test_df,
imfolder='/kaggle/input/melanoma-external-malignant-256/test/test/',
train=False,
transforms=train_transform,
meta_features=meta_features )<choose_model_class>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
skf = GroupKFold(n_splits=5 )<find_best_model_class>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
epochs = 12
es_patience = 3
TTA = 3
oof = np.zeros(( len(train_df), 1))
preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 skf = KFold(n_splits=5, shuffle=True, random_state=47)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X=np.zeros(len(train_df)) , y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):
print('=' * 20, 'Fold', fold, '=' * 20)
 model_path = f'model_{fold}.pth'
best_val = 0
patience = es_patience
arch = EfficientNet.from_pretrained('efficientnet-b1')
 model = Net(arch=arch, n_meta_features=len(meta_features))
model = model.to(device)
 optim = torch.optim.Adam(model.parameters() , lr=0.001)
 scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)
 criterion = nn.BCEWithLogitsLoss()
train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=train_transform,
meta_features=meta_features)
 val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True),
imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/',
train=True,
transforms=test_transform,
meta_features=meta_features)
 train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)
 val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)
 test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)
 for epoch in range(epochs):
start_time = time.time()
correct = 0
epoch_loss = 0
model.train()
for x, y in train_loader:
x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)
 x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)
 y = torch.tensor(y, device=device, dtype=torch.float32)
 optim.zero_grad()
z = model(x)
 loss = criterion(z, y.unsqueeze(1))
loss.backward()
optim.step()
pred = torch.round(torch.sigmoid(z))
correct +=(pred.cpu() == y.cpu().unsqueeze(1)).sum().item()
epoch_loss += loss.item()
train_acc = correct / len(train_idx)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))
val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())
 print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(
 epoch + 1,
epoch_loss,
train_acc,
val_acc,
val_roc,
str(datetime.timedelta(seconds=time.time() - start_time)) [:7]))
scheduler.step(val_roc)
 if val_roc >= best_val:
best_val = val_roc
patience = es_patience
torch.save(model, model_path)
 else:
patience -= 1
if patience == 0:
print('Early stopping.Best Val roc_auc: {:.3f}'.format(best_val))
break
model = torch.load(model_path)
 model.eval()
val_preds = torch.zeros(( len(val_idx), 1), dtype=torch.float32, device=device)
 with torch.no_grad() :
for j,(x_val, y_val)in enumerate(val_loader):
x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)
 x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)
 y_val = torch.tensor(y_val, device=device, dtype=torch.float32)
 z_val = model(x_val)
 val_pred = torch.sigmoid(z_val)
 val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val[0].shape[0]] = val_pred
oof[val_idx] = val_preds.cpu().numpy()
tta_preds = torch.zeros(( len(test), 1), dtype=torch.float32, device=device)
 for _ in range(TTA):
for i, x_test in enumerate(test_loader):
x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)
 x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)
 z_test = model(x_test)
 z_test = torch.sigmoid(z_test)
 tta_preds[i*test_loader.batch_size:i*test_loader.batch_size + x_test[0].shape[0]] += z_test
preds += tta_preds / TTA
preds /= skf.n_splits<compute_test_metric>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)) )<save_to_csv>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
pd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False )<save_to_csv>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 sub['target'] = preds.cpu().numpy().reshape(-1,)
 sub.to_csv('submission.csv', index=False )<import_modules>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
import pandas as pd
import matplotlib.pyplot as plt<load_from_csv>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
image_sub = pd.read_csv('.. /input/melanomapreds/submissionImage.csv')
 tabular_sub = pd.read_csv('.. /input/melanomapreds/submissionTabular.csv')
 tabular_sub.head()<save_to_csv>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
sub = image_sub.copy()
sub.target = 0.9 * image_sub.target.values + 0.1 * tabular_sub.target.values
sub.to_csv('submission.csv',index=False )<set_options>
%%time
cnt = 0
EPOCHS = 1120
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
rcParams['figure.figsize'] = 12, 4
pd.set_option('display.max_columns',100)
 for p in [np, pd, plt, sklearn, xgboost,sns,plt,re]:
print(p.__name__, p.__version__ )<set_options>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
rcParams['figure.figsize'] = 12, 4
pd.set_option('display.max_columns',100)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type=='object':
df[col] = df[col].astype('category')
 if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
def display_status(message):
display(message)
 clear_output(wait=True)
 print('Available datasets:')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
start = time.time()
display_status('Load aggregated data(1/3)')
 agg_data = pd.read_csv('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/agg_data.csv')
 agg_data = reduce_mem_usage(agg_data)
 X_test = agg_data[agg_data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 X_test.fillna(0,inplace=True)
 del agg_data
gc.collect()
basepath= '.. /input/competitive-data-science-predict-future-sales/'
test = pd.read_csv(basepath+"test.csv")
 display_status('Load serialized models(2/3)--- Time elapsed:{0} secs'.format(time.time() -start))
rf = pickle.load(open('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/rf_pred_30_10.model', 'rb'))
xgb = pickle.load(open('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/xgb_pred_100_10.model', 'rb'))
meta_model = pickle.load(open('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/metamodel.model', 'rb'))
display_status('Final prediction(3/3)--- Time elapsed:{0} secs'.format(time.time() -start))
meta_test = pd.DataFrame({'ID':test.index,'pred_1':rf.predict(X_test),'pred_2':xgb.predict(X_test)})
 X_meta_test = meta_test[['pred_1','pred_2']]
submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": meta_model.predict(X_meta_test ).clip(0, 20)
 })
 submission.to_csv('meta_model_pred.csv', index=False)
 submission.to_csv('submission.csv', index=False)
 print('Final prediction submission.csv file has been generated.Thanks for your patience')
 del rf, xgb, meta_model, meta_test, X_meta_test, submission, X_test
gc.collect()<load_from_csv>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
rcParams['figure.figsize'] = 12, 4
pd.set_option('display.max_columns',100)
 basepath= '.. /input/competitive-data-science-predict-future-sales/'
items = pd.read_csv(basepath+"items.csv")
 item_categories = pd.read_csv(basepath+"item_categories.csv")
 shops = pd.read_csv(basepath+"shops.csv")
 train = pd.read_csv(basepath+"sales_train.csv")
 test = pd.read_csv(basepath+"test.csv")
 print('All data has been loaded.' )<load_from_csv>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
agg_data = pd.read_csv('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/agg_data.csv')
 cols = ['holiday_num','item_cnt_month','next_month_holiday_num','vacation_period']
agg_data[cols].groupby(["holiday_num"], as_index=False
).agg({"item_cnt_month": ["sum"] }
).plot(kind='bar',x='holiday_num',y='item_cnt_month',
title='Current Month Holidays __VS__ Sales', subplots=True, rot=90)
 agg_data[cols].groupby(["next_month_holiday_num"] , as_index=False
).agg({"item_cnt_month": ["sum"] }
).plot(kind='bar',x='next_month_holiday_num',y='item_cnt_month',
title='Anticipation of num holidays next month __VS__ Sales', subplots=True, rot=90)
 days = pd.Series(['winter','etc','spring','etc','may', 'summer','summer','summer','etc','winter','winter','winter'])
 agg_data["vacation_period"] = agg_data["month"].map(days ).astype('category')
 agg_data[cols].groupby(["vacation_period"] , as_index=False
).agg({"item_cnt_month": ["sum"] }
).plot(kind='bar',x='vacation_period',y='item_cnt_month',
title='Vacation period __VS__ Sales.May is special and deserves its own category', subplots=True, rot=90)
 del agg_data, days
gc.collect()<merge>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
alldata = None
alldata = train.merge(items, how='inner', on='item_id')
 alldata= alldata.merge(item_categories, how='inner', on='item_category_id')
 alldata = alldata.merge(shops,how='inner', on='shop_id')
 alldata['year'] = pd.to_datetime(alldata['date'] ).dt.year
alldata['month'] = pd.to_datetime(alldata['date'] ).dt.month
alldata['year_month'] = alldata['year'].astype(str)+ '-' + alldata['month'].astype(str)
 popular_game_oct2015 = alldata.groupby(['year_month','item_category_name','item_name'],as_index=False ).sum()
cols=['date',	'date_block_num',	'item_name', 	'item_cnt_day', 'shop_id']
top_100 = popular_game_oct2015[popular_game_oct2015['year_month']=='2015-10'].sort_values(by=['item_cnt_day'], ascending=False ).iloc[:100,:]
top_100[['item_name','item_cnt_day']].head(15 ).plot.bar(x='item_name', rot=-90)
 del alldata,popular_game_oct2015,top_100
gc.collect()<define_variables>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
num_debut = len(set(test['item_id'])- set(train['item_id']))
total_test = len(set(test['item_id']))
pctg =(num_debut/total_test)*100
print('num of debut items: {0}({1}%)out of {2} items'.format(num_debut,pctg,total_test))<prepare_x_and_y>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
gc.collect()
agg_data = pd.read_csv('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/agg_data.csv')
 X_train = agg_data[agg_data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 y_train = agg_data[agg_data.date_block_num < 33]['item_cnt_month']
X_val = agg_data[agg_data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 y_val = agg_data[agg_data.date_block_num == 33]['item_cnt_month']
X_test = agg_data[agg_data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 X_train.fillna(0,inplace=True)
 X_val.fillna(0,inplace=True)
 X_test.fillna(0,inplace=True)
 rf_filename = 'rf_pred_30_10.model'
rf = pickle.load(open('/kaggle/input/create-model-simple-e-to-e-eda-to-ensemble/'+rf_filename, 'rb'))
del agg_data
gc.collect()
predictions = []
for tree in rf.estimators_:
predictions.append(tree.predict(X_val)[None, :])
 predictions = np.vstack(predictions)
 scores = []
acc = []
temp = [0] * len(predictions[0])
 for i in range(len(predictions)) :
temp += predictions[i]
acc.append(temp/(i+1))
scores=[]
for pred in acc:
scores.append(np.sqrt(mean_squared_error(y_val, pred)))
 scores
plt.figure(figsize=(10, 6))
plt.plot(scores, linewidth=3)
 plt.xlabel('num_trees')
 plt.ylabel('RMSE')
 del scores,predictions, rf
gc.collect()<install_modules>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
!cp.. /input/rapids/rapids.0.13.0 /opt/conda/envs/rapids.tar.gz
!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null
sys.path = ["/opt/conda/envs/rapids/lib/python3.6/site-packages"] + sys.path
sys.path = ["/opt/conda/envs/rapids/lib/python3.6"] + sys.path
sys.path = ["/opt/conda/envs/rapids/lib"] + sys.path
!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/
print('rapids ai installed' )<set_options>
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.26*df1['Confidence'] + 0.74*df2['Confidence']
df.head()
<load_from_csv><EOS> 
df.to_csv('submission.csv', index=False )