Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<predict_on_test>
plt.style.use('ggplot')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
Ytest = gbm.predict(Xte )<save_to_csv>
def seed_all(seed=42):
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_all()
df_test['fare_amount'] = pd.Series(Ytest)* df_test['min_fare']
df_test[['key','fare_amount']].to_csv('submission.csv',index=False )<define_variables>
def calc_metric_loss(pred_fvc,sigma,true_fvc):
true_fvc=np.reshape(true_fvc,pred_fvc.shape)
 sigma[sigma<70]=70
delta=np.abs(pred_fvc-true_fvc)
 delta[delta>1000]=1000
metric=-(np.sqrt(2)*delta/sigma)-np.log(np.sqrt(2)*sigma)
 return -metric
print("FARE MULTIPLE, LGBM")
 print(parameters)
 print(ll)
 print("df size: {}".format(len(df)))
 print("chunks: {}".format(chunks))
print("max bin: {}".format(max_bin))
print("min,max_fare_multiple {} {}".format(min_fare_multiple,max_fare_multiple))<predict_on_test>
class MLP2(nn.Module):
def __init__(self, input_dim, output_dim,hidden_layers=[1024, 1024], droprate=0.5, activation='relu'):
super(MLP2, self ).__init__()
self.model = nn.Sequential()
self.model.add_module('input', nn.Linear(input_dim, hidden_layers[0]))
if activation == 'relu':
self.model.add_module('relu0', nn.ReLU())
 elif activation == 'tanh':
self.model.add_module('tanh0', nn.Tanh())
 for i in range(len(hidden_layers)-1):
self.model.add_module('dropout'+str(i+1), nn.Dropout(p=droprate))
self.model.add_module('hidden'+str(i+1), nn.Linear(hidden_layers[i], hidden_layers[i+1]))
if activation == 'relu':
self.model.add_module('relu'+str(i+1), nn.ReLU())
 elif activation == 'tanh':
self.model.add_module('tanh'+str(i+1), nn.Tanh())
 self.model.add_module('dropout'+str(i+2), nn.Dropout(p=droprate))
self.model.add_module('final', nn.Linear(hidden_layers[i+1], output_dim))
def forward(self, x):
return self.model(x)
 def predict(self,x, samples):
self.train()
y_samples = [self.forward(x)for _ in range(samples)]
y_samples = torch.cat(y_samples, dim=1)
 std = torch.std(y_samples,dim=1)
 self.eval()
mean = self.forward(x)
 self.train()
return mean, std
def metric_loss(self,pred_fvc,true_fvc,pred_sigma):
true_fvc=torch.reshape(true_fvc,pred_fvc.shape)
 sigma_clipped=torch.clamp(pred_sigma,min=1e-3,max=70)
 delta=torch.clamp(torch.abs(pred_fvc-true_fvc),max=1000)
 metric=torch.div(-torch.sqrt(torch.tensor([2.0] ).to(device)) *delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0] ).to(device)) *sigma_clipped)
 return -metric
def fvc_loss(self,pred_fvc,true_fvc):
true_fvc=torch.reshape(true_fvc,pred_fvc.shape)
 fvc_err=torch.abs(pred_fvc-true_fvc)
 return fvc_err
Ytr_hat = gbm.predict(Xtr)
 Yval_hat = gbm.predict(Xval)
<feature_engineering>
class MC_Dropout_Model(nn.Module):
def __init__(self, input_dim, output_dim, drop_prob=0.2):
super(MC_Dropout_Model, self ).__init__()
self.input_dim = input_dim
self.output_dim = output_dim
self.drop_prob = drop_prob
self.fc1 = nn.Linear(input_dim, 100)
 self.drop1=nn.Dropout(p=drop_prob)
 self.fc2 = nn.Linear(100, 100)
 self.drop2=nn.Dropout(p=drop_prob)
 self.fc3 = nn.Linear(100, output_dim)
 def forward(self, x):
out=x
out = F.relu(self.fc1(out))
out= self.drop1(out)
 out = F.relu(self.fc2(out))
out= self.drop2(out)
 out = self.fc3(out)
 return out
def predict(self,x, samples):
self.train()
y_samples = [self.forward(x)for _ in range(samples)]
y_samples = torch.cat(y_samples, dim=1)
 std = torch.std(y_samples,dim=1)
 self.eval()
mean = self.forward(x)
 self.train()
return mean, std
def metric_loss(self,pred_fvc,true_fvc,pred_sigma):
true_fvc=torch.reshape(true_fvc,pred_fvc.shape)
 sigma_clipped=torch.clamp(pred_sigma,min=70)
 delta=torch.clamp(torch.abs(pred_fvc-true_fvc),max=1000)
 metric=torch.div(-torch.sqrt(torch.tensor([2.0] ).to(device)) *delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0] ).to(device)) *sigma_clipped)
 return -metric
def fvc_loss(self,pred_fvc,true_fvc):
true_fvc=torch.reshape(true_fvc,pred_fvc.shape)
 fvc_err=torch.abs(pred_fvc-true_fvc)
 return fvc_err
dftr['fare_predicted'] = pd.Series(Ytr)* dftr['min_fare']
dfval['fare_predicted'] = pd.Series(Yval)* dfval['min_fare']
dftr['fare_err'] =(dftr['fare_amount'] - dftr['fare_predicted'] ).abs()
dfval['fare_err'] =(dfval['fare_amount'] - dfval['fare_predicted'] ).abs()<train_model>
train=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 submission=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )
rms_tr =(dftr['fare_err']**2 ).mean() **.5
print("Accuracy training: ", rms_tr )<compute_test_metric>
train['base_Weeks']=train.groupby(['Patient'])['Weeks'].transform('min')
 base=train[train.Weeks==train.base_Weeks]
base = base.rename(columns={'FVC': 'base_FVC','Percent': 'base_Percent'})
 base.drop_duplicates(subset=['Patient', 'Weeks'], keep='first',inplace=True)
 train=train.merge(base[['Patient','base_FVC','base_Percent']],on='Patient',how='left')
 train['Week_passed'] = train['Weeks'] - train['base_Weeks']
rms_val =(dfval['fare_err']**2 ).mean() **.5
print("Accuracy2: ", rms_val )<save_to_csv>
test = test.rename(columns={'Weeks': 'base_Weeks', 'FVC': 'base_FVC','Percent': 'base_Percent'})
 submission = pd.read_csv(".. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv")
 submission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission['Weeks'] = submission['Patient_Week'].apply(lambda x:x.split('_')[1] ).astype(int)
 test = submission.drop(columns = ["FVC", "Confidence"] ).merge(test, on = 'Patient')
 test['Week_passed'] = test['Weeks'] - test['base_Weeks']
test=test[train.columns.drop(['FVC','Percent'])]
err100 = dfval.nlargest(100,'fare_err')
 err100.to_csv('100errval.csv' )<set_options>
COLS = ['Sex','SmokingStatus']
for col in COLS:
for mod in train[col].unique() :
train[mod] =(train[col] == mod ).astype(int)
 test[mod] =(test[col] == mod ).astype(int)
 train.drop(col,axis=1,inplace=True)
 test.drop(col,axis=1,inplace=True )
% matplotlib inline
plt.style.use('seaborn-whitegrid' )<load_from_csv>
robust_scaler = preprocessing.RobustScaler()
train[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])]=robust_scaler.fit_transform(train[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])] )
df_test = pd.read_csv(".. /input/test.csv")
 df_train = pd.read_csv('.. /input/train.csv', nrows = 500_000,)
 df_train.dtypes<count_missing_values>
with open('.. /input/file-dictionary/good_files.pickle', 'rb')as handle:
good_file_dict = pickle.load(handle)
 with open('.. /input/file-dictionary/bad_files.pickle', 'rb')as handle:
bad_file_dict = pickle.load(handle )
df_train.isnull().sum()<count_missing_values>
class OSIC(Dataset):
def __init__(self,patient_ids,df,file_dict,train=True, transform=None,nims=10):
self.df=df[df.Patient.isin(patient_ids)]
self.train=train
if self.train:
self.fvc=self.df['FVC'].values
else:
self.df[self.df.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])]=robust_scaler.transform(self.df[self.df.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])])
 self.data=self.df[self.df.columns.difference(['FVC','Patient','Percent'])].values
self.patients=self.df['Patient'].values
self.file_dict=file_dict
self.nims=nims
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if self.train:
data = {'fvc': self.fvc[idx],
'data': self.data[idx]}
else:
data = {'data': self.data[idx]}
return data
df_train=df_train.dropna()
df_test=df_test.dropna()
print(df_train.isnull().sum())
 print(df_test.isnull().sum() )<count_values>
epochs=200
batch_size=32
num_workers=3
df_train.query("fare_amount <= 0" ).count() 
<compute_test_metric>
ids=train.Patient.unique()
index = np.argwhere(ids=='ID00011637202177653955184')
 ids = list(np.delete(ids, index))
random.shuffle(ids)
 ids=np.array(ids)
 train_ids,val_ids=np.split(ids, [int(round(0.9 * len(ids), 0)) ])
 train_dataset = OSIC(train_ids,train,good_file_dict)
 train_dataloader = DataLoader(train_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers)
 val_dataset = OSIC(val_ids,train,good_file_dict)
 val_dataloader = DataLoader(val_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers )
def haversine_distance(lat1, long1, lat2, long2):
R = 6371
phi1 = np.radians(lat1)
 phi2 = np.radians(lat2)
 delta_phi = np.radians(lat2-lat1)
 delta_lambda = np.radians(long2-long1)
 a = np.sin(delta_phi / 2.0)** 2 + np.cos(phi1)* np.cos(phi2)* np.sin(delta_lambda / 2.0)** 2
c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
d =(R * c)
 return d<train_model>
model = MC_Dropout_Model(train_dataset.data.shape[1], 1 ).to(device)
 print('Number of parameters:')
 print(sum(p.numel() for p in model.parameters() if p.requires_grad))
df_train = df_train.reset_index()
df_test=df_test.reset_index()<feature_engineering>
optimizer = optim.Adam(model.parameters())
 epoch_train_metric=[]
epoch_val_metric=[]
epoch_train_loss=[]
epoch_val_loss=[]
epoch=0
min_val_loss = 1e+100
min_val_metric = 1e+100
early_stop = False
patience=10
while epoch<epochs and not early_stop:
epoch+=1
train_loss=0
train_metric=0
model.train()
for batch_idx, data in enumerate(train_dataloader):
optimizer.zero_grad()
fvc,sigma = model.predict(data['data'].float().to(device),100)
 fvc_loss = model.fvc_loss(fvc,data['fvc'].to(device)).mean()
metric_loss = model.metric_loss(fvc,data['fvc'].to(device),sigma ).mean()
loss=fvc_loss
loss.backward()
train_loss += loss.item()
train_metric += metric_loss.item()
optimizer.step()
print('====> Epoch: {} Average train loss FVC: {:.4f}'.format(
 epoch, train_loss / len(train_dataloader)))
 print('====> Epoch: {} Average train metric: {:.4f}'.format(
 epoch, train_metric / len(train_dataloader)))
 val_loss=0
val_metric=0
model.eval()
with torch.no_grad() :
for batch_idx, data in enumerate(val_dataloader):
fvc,sigma = model.predict(data['data'].float().to(device),100)
 fvc_loss = model.fvc_loss(fvc,data['fvc'].to(device)).mean()
metric_loss = model.metric_loss(fvc,data['fvc'].to(device),sigma ).mean()
loss=fvc_loss
val_loss += loss.item()
val_metric += metric_loss.item()
print('====> Epoch: {} Average val loss FVC: {:.4f}'.format(
 epoch, val_loss / len(val_dataloader)))
 print('====> Epoch: {} Average val metric: {:.4f}'.format(
 epoch, val_metric / len(val_dataloader)))
 epoch_train_loss.append(train_loss/ len(train_dataloader))
epoch_val_loss.append(val_loss / len(val_dataloader))
epoch_train_metric.append(train_metric/ len(train_dataloader))
epoch_val_metric.append(val_metric / len(val_dataloader))
if val_metric/ len(val_dataloader)< min_val_metric:
min_val_metric = val_metric/ len(val_dataloader)
 best_model = copy.deepcopy(model.state_dict())
 print('Min metric %0.2f' % min_val_metric)
 epochs_no_improve = 0
else:
epochs_no_improve += 1
if epochs_no_improve == patience:
print('Early stopping!')
 early_stop = True
model.load_state_dict(best_model )
for i in range(len(df_test)) :
df_test.loc[[i], 'distance'] = haversine_distance(df_test.loc[i]['pickup_latitude'], df_test.loc[i]['pickup_longitude'], df_test.loc[i]['dropoff_latitude'], df_test.loc[i]['dropoff_longitude'] )<feature_engineering>
submission=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )
for i in range(len(df_train)) :
df_train.loc[[i], 'distance'] = haversine_distance(df_train.loc[i]['pickup_latitude'], df_train.loc[i]['pickup_longitude'], df_train.loc[i]['dropoff_latitude'], df_train.loc[i]['dropoff_longitude'] )<count_values>
test_ids=test.Patient.unique()
test_dataset = OSIC(test_ids,test,good_file_dict,train=False)
 test_dataloader = DataLoader(test_dataset, batch_size=batch_size,shuffle=False,num_workers=num_workers)
 model.eval()
fvc_pred = []
sigma_pred = []
with torch.no_grad() :
for batch_idx, data in enumerate(test_dataloader):
fvc,sigma = model.predict(data['data'].float().to(device),100)
 fvc_pred.append(fvc)
 sigma_pred.append(sigma)
 fvc_pred=torch.cat(fvc_pred, dim=0)
 sigma_pred=torch.cat(sigma_pred, dim=0)
 test['FVC']=fvc_pred.cpu().numpy()
test['Confidence']=sigma_pred.cpu().numpy()
df_train.query('distance == 0' ).count()<data_type_conversions>
test['Patient_Week']=test["Patient"] + '_' + test['Weeks'].apply(str )
df_train['distance'] = pd.to_numeric(df_train['distance'])
 df_train = df_train.query('distance > 0.0')
 df_test['distance'] = pd.to_numeric(df_test['distance'] )<concatenate>
submission=submission[['Patient_Week']].merge(test[['Patient_Week','FVC','Confidence']],on='Patient_Week' )
<count_values><EOS> 
submission.to_csv('submission.csv', index=False, float_format='%.1f' )