Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<load_pretrained>
warnings.filterwarnings("ignore" )
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=10.0, max_zoom=1.1, max_lighting=0.2, max_warp=0.2, p_affine=0.75, p_lighting=0.75)
 train_data = ImageDataBunch.from_df(path/'train'/'train', train_df, ds_tfms=tfms, size=128 )<define_variables>
import torch.nn as nn
import torch
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from scipy.ndimage import zoom
from scipy.stats import kurtosis
from scipy.stats import skew
train_data.show_batch(rows=3, figsize=(5,6))<choose_model_class>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.enabled = True
seed_everything(42 )
learn = cnn_learner(train_data, models.resnet50, metrics=[accuracy],model_dir="/tmp/model/" )<find_best_params>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
device = torch.device('cuda')
 tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 feature_ct = pd.read_csv('.. /input/2020osic/CT_21feature.csv')
 tr = tr.merge(feature_ct, on='Patient')
 chunk = pd.read_csv(f"{ROOT}/test.csv" )
learn.lr_find()<train_model>
def load_scan(path,resize_type='no'):

 slices = [pydicom.read_file(path + '/' + s)for s in os.listdir(path)]
slices.sort(key = lambda x: int(x.InstanceNumber))
try:
slice_thickness = abs(slices[-1].ImagePositionPatient[2] - slices[0].ImagePositionPatient[2])/(len(slices))
except:
try:
slice_thickness = abs(slices[-1].SliceLocation - slices[0].SliceLocation)/(len(slices))
except:
slice_thickness = slices[0].SliceThickness
for s in slices:
s.SliceThickness = slice_thickness
if resize_type == 'resize':
s.PixelSpacing = s.PixelSpacing*(s.Rows/512)
 return slices
def transform_to_hu(slices):

 images = np.stack([file.pixel_array for file in slices])
 images = images.astype(np.int16)
 images[images <= -1000] = 0
for n in range(len(slices)) :
intercept = slices[n].RescaleIntercept
slope = slices[n].RescaleSlope
if slope != 1:
images[n] = slope * images[n].astype(np.float64)
 images[n] = images[n].astype(np.int16)
 images[n] += np.int16(intercept)
 return np.array(images, dtype=np.int16 )
lr =1.0e-2
learn.fit_one_cycle(7,slice(lr))<train_model>
class Test_Generate(Dataset):
def __init__(self,imgs_dicom):
self.imgs_dicom = imgs_dicom
def __getitem__(self,index):
metainf = self.imgs_dicom[index]
slice_img = metainf.pixel_array
slice_img =(slice_img-slice_img.min())/(slice_img.max() -slice_img.min())
 slice_img =(slice_img*255 ).astype(np.uint8)
 if metainf.Rows!=512 or metainf.Columns!=512:
slice_img = cv2.resize(slice_img,(512,512))
slice_img = slice_img[None,:,:]
slice_img =(slice_img/255 ).astype(np.float32)
 return slice_img
def __len__(self):
return len(self.imgs_dicom )
learn.fit_one_cycle(5, slice(1e-4, lr/5))<create_dataframe>
device = torch.device('cuda:0')
 checkpoint = '.. /input/2020osic/best_lung_Unet_densenet121_my.pth'
Model_M = smp.Unet('densenet121', classes=1, in_channels=1,activation='sigmoid',encoder_weights=None ).to(device)
 Model_M.load_state_dict(torch.load(checkpoint))
def Unet_mask(model_m: nn.Module,input_data: DataLoader):
model_m.eval()
outs = []
for idx, sample in enumerate(test_loader):
image = sample
image = image.to(device)
 with torch.no_grad() :
out = model_m(image)
 out = out.cpu().data.numpy()
out = np.where(out>0.5,1,0)
 out = np.squeeze(out,axis=1)
 outs.append(out)
 outs = np.concatenate(outs)
 return outs
solution = pd.DataFrame(columns=test_df.columns)
 solution<predict_on_test>
ct_root_path = '.. /input/osic-pulmonary-fibrosis-progression/test/'
lung_stat_pd = pd.DataFrame(columns=['Patient','Volume','Mean','Median','Skew','Kurthosis','HAA',
'midMean','midMedian','midSkew','midKurthosis','midHAA',
'pro_0','pro_100','pro_200','pro_300','pro_400','pro_500',
'pro_600','pro_700','pro_800','pro_900'])
 haa_thresh = [-600,-250]
for i,p in enumerate(tqdm(pd.unique(chunk['Patient']))):
lung_stat_pd.loc[i,'Patient'] = p
patient_scans = load_scan(ct_root_path + p)
 test_db = Test_Generate(patient_scans)
 test_loader = DataLoader(test_db, batch_size=8, shuffle=False, num_workers=4)
 masks = Unet_mask(Model_M,test_loader)
 patient_images = transform_to_hu(patient_scans)
 if patient_images[0].shape!=(512,512):
patient_images = preprocess_img(patient_images,'resize')
 lung_stat_pd.loc[i,'Volume'] = caculate_lung_volume(patient_scans,masks)
 statistical_characteristic = caculate_histgram_statistical(patient_images,masks,haa_thresh)
 lung_stat_pd.loc[i,'Mean'] = statistical_characteristic['Mean']
lung_stat_pd.loc[i,'Median'] = statistical_characteristic['Median']
lung_stat_pd.loc[i,'Skew'] = statistical_characteristic['Skew']
lung_stat_pd.loc[i,'Kurthosis'] = statistical_characteristic['Kurthosis']
lung_stat_pd.loc[i,'HAA'] = statistical_characteristic['HAA']
lung_stat_pd.loc[i,'midMean'] = statistical_characteristic['midMean']
lung_stat_pd.loc[i,'midMedian'] = statistical_characteristic['midMedian']
lung_stat_pd.loc[i,'midSkew'] = statistical_characteristic['midSkew']
lung_stat_pd.loc[i,'midKurthosis'] = statistical_characteristic['midKurthosis']
lung_stat_pd.loc[i,'midHAA'] = statistical_characteristic['midHAA']
lung_stat_pd.loc[i,'pro_0'] = statistical_characteristic['pro_0']
lung_stat_pd.loc[i,'pro_100'] = statistical_characteristic['pro_100']
lung_stat_pd.loc[i,'pro_200'] = statistical_characteristic['pro_200']
lung_stat_pd.loc[i,'pro_300'] = statistical_characteristic['pro_300']
lung_stat_pd.loc[i,'pro_400'] = statistical_characteristic['pro_400']
lung_stat_pd.loc[i,'pro_500'] = statistical_characteristic['pro_500']
lung_stat_pd.loc[i,'pro_600'] = statistical_characteristic['pro_600']
lung_stat_pd.loc[i,'pro_700'] = statistical_characteristic['pro_700']
lung_stat_pd.loc[i,'pro_800'] = statistical_characteristic['pro_800']
lung_stat_pd.loc[i,'pro_900'] = statistical_characteristic['pro_900']
lung_stat_pd.head()
for index,row in test_df.iterrows() :
img_name = row['id']
img = open_image(path/'test'/'test'/img_name)
 pred_class,pred_idx,outputs = learn.predict(img)
 solution.loc[len(solution)] = [img_name,outputs.numpy() [1]]<save_to_csv>
chunk = chunk.merge(lung_stat_pd)
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 sub.head()
solution.to_csv('submission.csv', index=False )<save_to_csv>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
def create_download_link(df, title = "Download CSV file", filename = "submission.csv"):
csv = df.to_csv()
b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 df = pd.DataFrame(np.random.randn(50, 4), columns=list('ABCD'))
create_download_link(df)
<load_from_csv>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
train_df = pd.read_csv('.. /input/train.csv', nrows = 1_000_000)
 train_df.dtypes<count_missing_values>
base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','FVC']] 
.rename({'FVC': 'min_FVC'}, axis=1)
.groupby('Patient')
.first() 
.reset_index()
)
print(train_df.isnull().sum() )<drop_column>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','Percent']] 
.rename({'Percent': 'min_Percent'}, axis=1)
.groupby('Patient')
.first() 
.reset_index() 
)
 data = data.merge(base, on='Patient', how='left')
 del base
data.head()
train_df = train_df.dropna(how = 'any', axis = 'rows' )<drop_column>
def clean_df(df):
return df[(df.fare_amount > 0)& 
(df.pickup_longitude > -80)&(df.pickup_longitude < -70)& 
(df.pickup_latitude > 35)&(df.pickup_latitude < 45)& 
(df.dropoff_longitude > -80)&(df.dropoff_longitude < -70)& 
(df.dropoff_latitude > 35)&(df.dropoff_latitude < 45)& 
(df.passenger_count > 0)&(df.passenger_count < 10)]
train_df = clean_df(train_df)
 print(len(train_df))<concatenate>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['week'] = data['base_week']
data['percent'] =(data['min_Percent'] - data['min_Percent'].min())/(data['min_Percent'].max() - data['min_Percent'].min())
 data['volume'] =(data['Volume'] - data['Volume'].min())/(data['Volume'].max() - data['Volume'].min())
 data['mean'] =(data['Mean'] - data['Mean'].min())/(data['Mean'].max() - data['Mean'].min())
 data['skew'] =(data['Skew'] - data['Skew'].min())/(data['Skew'].max() - data['Skew'].min())
 data['median'] =(data['Median'] - data['Median'].min())/(data['Median'].max() - data['Median'].min())
 data['kurthosis'] =(data['Kurthosis'] - data['Kurthosis'].min())/(data['Kurthosis'].max() - data['Kurthosis'].min())
 data['haa'] =(data['HAA'] - data['HAA'].min())/(data['HAA'].max() - data['HAA'].min())
 data['midmedian'] =(data['midMedian'] - data['midMedian'].min())/(data['midMedian'].max() - data['midMedian'].min())
 data['midmean'] =(data['midMean'] - data['midMean'].min())/(data['midMean'].max() - data['midMean'].min())
 data['midskew'] =(data['midSkew'] - data['midSkew'].min())/(data['midSkew'].max() - data['midSkew'].min())
 data['midkurthosis'] =(data['midKurthosis'] - data['midKurthosis'].min())/(data['midKurthosis'].max() - data['midKurthosis'].min())
 data['midhaa'] =(data['midHAA'] - data['midHAA'].min())/(data['midHAA'].max() - data['midHAA'].min())
 data['pro_0'] =(data['pro_0']-data['pro_0'].min())/(data['pro_0'].max() - data['pro_0'].min())
 data['pro_100'] =(data['pro_100']-data['pro_100'].min())/(data['pro_100'].max() - data['pro_100'].min())
 data['pro_200'] =(data['pro_200']-data['pro_200'].min())/(data['pro_200'].max() - data['pro_200'].min())
 data['pro_300'] =(data['pro_300']-data['pro_300'].min())/(data['pro_300'].max() - data['pro_300'].min())
 data['pro_400'] =(data['pro_400']-data['pro_400'].min())/(data['pro_400'].max() - data['pro_400'].min())
 data['pro_500'] =(data['pro_500']-data['pro_500'].min())/(data['pro_500'].max() - data['pro_500'].min())
 data['pro_600'] =(data['pro_600']-data['pro_600'].min())/(data['pro_600'].max() - data['pro_600'].min())
 data['pro_700'] =(data['pro_700']-data['pro_700'].min())/(data['pro_700'].max() - data['pro_700'].min())
 data['pro_800'] =(data['pro_800']-data['pro_800'].min())/(data['pro_800'].max() - data['pro_800'].min())
 data['pro_900'] =(data['pro_900']-data['pro_900'].min())/(data['pro_900'].max() - data['pro_900'].min())
 data['res_fvc'] = data['min_FVC']-data['FVC']
data.loc[:,"Sex"] = pd.factorize(data.Sex)[0]
data.loc[:,"SmokingStatus"] = pd.factorize(data.SmokingStatus)[0]
data['Sex'] =(data['Sex'] - data['Sex'].min())/(data['Sex'].max() - data['Sex'].min())
 data['SmokingStatus'] =(data['SmokingStatus'] - data['SmokingStatus'].min())/(data['SmokingStatus'].max() - data['SmokingStatus'].min())
def sphere_dist(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):
R_earth = 6371
pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,
[pickup_lat, pickup_lon,
dropoff_lat, dropoff_lon])
 dlat = dropoff_lat - pickup_lat
dlon = dropoff_lon - pickup_lon
a = np.sin(dlat/2.0)**2 + np.cos(pickup_lat)* np.cos(dropoff_lat)* np.sin(dlon/2.0)**2
return 2 * R_earth * np.arcsin(np.sqrt(a))
def add_datetime_info(dataset):
dataset['pickup_datetime'] = pd.to_datetime(dataset['pickup_datetime'])
 dataset['hour'] = dataset.pickup_datetime.dt.hour
dataset['day'] = dataset.pickup_datetime.dt.day
dataset['month'] = dataset.pickup_datetime.dt.month
dataset['weekday'] = dataset.pickup_datetime.dt.weekday
return dataset
train_df['distance'] = sphere_dist(train_df['pickup_latitude'], train_df['pickup_longitude'],
train_df['dropoff_latitude'] , train_df['dropoff_longitude'])
 train_df = add_datetime_info(train_df)
 train_df.head()<drop_column>
FE = ['percent','SmokingStatus','Sex','age','week','volume','mean','skew','kurthosis','haa','median',
'midmean','midmedian','midskew','midkurthosis','midhaa',
'pro_0','pro_100','pro_200','pro_300','pro_400','pro_500',
'pro_600','pro_700','pro_800','pro_900']
train_df.drop(columns=['key', 'pickup_datetime'], inplace=True)
 train_df.head()<feature_engineering>
tr = data[data.WHERE=='train']
test = data[data.WHERE=='test']
sub = data.loc[data.WHERE=='test']
del data
tr.head()
train_df['pickup_long_15'] = train_df['pickup_longitude']*np.cos(15* np.pi / 180)- train_df['pickup_latitude']*np.sin(15* np.pi/180)
 train_df['pickup_long_30'] = train_df['pickup_longitude']*np.cos(30* np.pi / 180)- train_df['pickup_latitude']*np.sin(30* np.pi/180)
 train_df['pickup_long_45'] = train_df['pickup_longitude']*np.cos(45* np.pi / 180)- train_df['pickup_latitude']*np.sin(45* np.pi/180)
 train_df['pickup_long_60'] = train_df['pickup_longitude']*np.cos(60* np.pi / 180)- train_df['pickup_latitude']*np.sin(60* np.pi/180)
 train_df['pickup_long_75'] = train_df['pickup_longitude']*np.cos(75* np.pi / 180)- train_df['pickup_latitude']*np.sin(75* np.pi/180)
 train_df['pickup_lat_15'] = train_df['pickup_longitude']*np.sin(15* np.pi / 180)+ train_df['pickup_latitude']*np.cos(15* np.pi/180)
 train_df['pickup_lat_30'] = train_df['pickup_longitude']*np.sin(30* np.pi / 180)+ train_df['pickup_latitude']*np.cos(30* np.pi/180)
 train_df['pickup_lat_45'] = train_df['pickup_longitude']*np.sin(45* np.pi / 180)+ train_df['pickup_latitude']*np.cos(45* np.pi/180)
 train_df['pickup_lat_60'] = train_df['pickup_longitude']*np.sin(60* np.pi / 180)+ train_df['pickup_latitude']*np.cos(60* np.pi/180)
 train_df['pickup_lat_75'] = train_df['pickup_longitude']*np.sin(75* np.pi / 180)+ train_df['pickup_latitude']*np.cos(75* np.pi/180)
 train_df['dropoff_long_15'] = train_df['dropoff_longitude']*np.cos(15* np.pi / 180)- train_df['dropoff_latitude']*np.sin(15* np.pi/180)
 train_df['dropoff_long_30'] = train_df['dropoff_longitude']*np.cos(30* np.pi / 180)- train_df['dropoff_latitude']*np.sin(30* np.pi/180)
 train_df['dropoff_long_45'] = train_df['dropoff_longitude']*np.cos(45* np.pi / 180)- train_df['dropoff_latitude']*np.sin(45* np.pi/180)
 train_df['dropoff_long_60'] = train_df['dropoff_longitude']*np.cos(60* np.pi / 180)- train_df['dropoff_latitude']*np.sin(60* np.pi/180)
 train_df['dropoff_long_75'] = train_df['dropoff_longitude']*np.cos(75* np.pi / 180)- train_df['dropoff_latitude']*np.sin(75* np.pi/180)
 train_df['dropoff_lat_15'] = train_df['dropoff_longitude']*np.sin(15* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(15* np.pi/180)
 train_df['dropoff_lat_30'] = train_df['dropoff_longitude']*np.sin(30* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(30* np.pi/180)
 train_df['dropoff_lat_45'] = train_df['dropoff_longitude']*np.sin(45* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(45* np.pi/180)
 train_df['dropoff_lat_60'] = train_df['dropoff_longitude']*np.sin(60* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(60* np.pi/180)
 train_df['dropoff_lat_75'] = train_df['dropoff_longitude']*np.sin(75* np.pi / 180)+ train_df['dropoff_latitude']*np.cos(75* np.pi/180 )<split>
class MishFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, x):
ctx.save_for_backward(x)
 return x * torch.tanh(F.softplus(x))
@staticmethod
def backward(ctx, grad_output):
x = ctx.saved_variables[0]
sigmoid = torch.sigmoid(x)
 tanh_sp = torch.tanh(F.softplus(x))
return grad_output *(tanh_sp + x * sigmoid *(1 - tanh_sp * tanh_sp))
class Mish(nn.Module):
def forward(self, x):
return MishFunction.apply(x)
 def to_Mish(model):
for child_name, child in model.named_children() :
if isinstance(child, nn.ReLU):
setattr(model, child_name, Mish())
 else:
to_Mish(child )
y = train_df['fare_amount']
train = train_df.drop(columns=['fare_amount'])
 x_train,x_test,y_train,y_test = train_test_split(train,y,random_state=0,test_size=0.2 )<train_model>
C1, C2 = torch.tensor(70,dtype=torch.float),torch.tensor(1000,dtype=torch.float)
 C1, C2 = C1.to(device),C2.to(device)
 def score_np(fvc_true, y_pred):
sigma = y_pred[:,2] - y_pred[:,0]
fvc_pred = y_pred[:,1]
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 def score(y_true, y_pred):
y_true = y_true.to(torch.float)
 y_pred = y_pred.to(torch.float)
 sigma = y_pred[:,2] - y_pred[:,0]
fvc_pred = y_pred[:,1]
sigma_clip = torch.max(sigma, C1)
 delta = torch.abs(y_true[:,0] - fvc_pred)
 delta = torch.min(delta, C2)
 sq2 = torch.sqrt(torch.tensor(2, dtype=torch.float))
metric =(delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)
 return torch.mean(metric)
 def qloss(y_true, y_pred):
device = y_true.device
qs = [0.25, 0.50, 0.75]
q = torch.tensor(np.array([qs]), dtype=torch.float32)
 q = q.to(device)
 e = y_true - y_pred
v = torch.max(q*e,(q-1)*e)
 return torch.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss

 class GaussianNoise(nn.Module):

 def __init__(self, sigma=0.1, is_relative_detach=True):
super().__init__()
self.sigma = sigma
self.is_relative_detach = is_relative_detach
self.register_buffer('noise', torch.tensor(0))
def forward(self, x):
if self.training and self.sigma != 0:
scale = self.sigma * x.detach() if self.is_relative_detach else self.sigma * x
sampled_noise = self.noise.expand(*x.size() ).float().normal_() * scale
x = x + sampled_noise
return x
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=512):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.5)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.5)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class make_big_model(nn.Module):
def __init__(self, in_ch, out_ch=3):
super(make_big_model, self ).__init__()
self.gasus_nosie = GaussianNoise()
self.fc1 = nn.Sequential(
 nn.Linear(in_ch, 256),
Mish() 
)
 self.fc2 = nn.Sequential(
 nn.Linear(256, 256),
Mish() ,
nn.Dropout(0.5)
)
 self.fc3_p1 = nn.Sequential(
 nn.Linear(256, 128),
Mish() ,
nn.Dropout(0.5)
)
 self.fc3_p2 = nn.Linear(128, out_ch)
 def forward(self, x):
x = self.gasus_nosie(x)
 x = self.fc1(x)
 x = self.fc2(x)
 x = self.fc3_p1(x)
 x = self.fc3_p2(x)
 return x
class make_model(nn.Module):
def __init__(self, in_ch, out_ch=3):
super(make_model, self ).__init__()
self.fc1 = nn.Sequential(
 nn.Linear(in_ch, 160),
Mish() 
)
 self.fc2 = nn.Sequential(
 nn.Linear(160, 128),
Mish() 
)
 self.fc3_p1 = nn.Sequential(
 nn.Linear(128, 64),
Mish() 
)
 self.fc3_p2 = nn.Linear(128, out_ch)
 def forward(self, x):
x = self.fc1(x)
 x = self.fc2(x)
 x = self.fc3_p2(x)
 return x
def XGBmodel(x_train,x_test,y_train,y_test):
matrix_train = xgb.DMatrix(x_train,label=y_train)
 matrix_test = xgb.DMatrix(x_test,label=y_test)
 model=xgb.train(params={'objective':'reg:linear','eval_metric':'rmse'},
dtrain=matrix_train,num_boost_round=100,
early_stopping_rounds=100,evals=[(matrix_test,'test')])
 return model
model = XGBmodel(x_train,x_test,y_train,y_test )<feature_engineering>
NFOLD = 5
kf = KFold(n_splits=NFOLD)
 pd_patient = pd.DataFrame({"Patient":tr["Patient"].unique() })
 for idx,(tr_idx, val_idx)in enumerate(kf.split(pd_patient)) :
pd_patient.loc[val_idx,"fold"] = idx
pd_patient.head()
test_df = pd.read_csv('.. /input/test.csv')
 test_df['distance'] = sphere_dist(test_df['pickup_latitude'], test_df['pickup_longitude'],
test_df['dropoff_latitude'] , test_df['dropoff_longitude'])
 test_df = add_datetime_info(test_df)
 test_df['pickup_long_15'] = test_df['pickup_longitude']*np.cos(15* np.pi / 180)- test_df['pickup_latitude']*np.sin(15* np.pi/180)
 test_df['pickup_long_30'] = test_df['pickup_longitude']*np.cos(30* np.pi / 180)- test_df['pickup_latitude']*np.sin(30* np.pi/180)
 test_df['pickup_long_45'] = test_df['pickup_longitude']*np.cos(45* np.pi / 180)- test_df['pickup_latitude']*np.sin(45* np.pi/180)
 test_df['pickup_long_60'] = test_df['pickup_longitude']*np.cos(60* np.pi / 180)- test_df['pickup_latitude']*np.sin(60* np.pi/180)
 test_df['pickup_long_75'] = test_df['pickup_longitude']*np.cos(75* np.pi / 180)- test_df['pickup_latitude']*np.sin(75* np.pi/180)
 test_df['pickup_lat_15'] = test_df['pickup_longitude']*np.sin(15* np.pi / 180)+ test_df['pickup_latitude']*np.cos(15* np.pi/180)
 test_df['pickup_lat_30'] = test_df['pickup_longitude']*np.sin(30* np.pi / 180)+ test_df['pickup_latitude']*np.cos(30* np.pi/180)
 test_df['pickup_lat_45'] = test_df['pickup_longitude']*np.sin(45* np.pi / 180)+ test_df['pickup_latitude']*np.cos(45* np.pi/180)
 test_df['pickup_lat_60'] = test_df['pickup_longitude']*np.sin(60* np.pi / 180)+ test_df['pickup_latitude']*np.cos(60* np.pi/180)
 test_df['pickup_lat_75'] = test_df['pickup_longitude']*np.sin(75* np.pi / 180)+ test_df['pickup_latitude']*np.cos(75* np.pi/180)
 test_df['dropoff_long_15'] = test_df['dropoff_longitude']*np.cos(15* np.pi / 180)- test_df['dropoff_latitude']*np.sin(15* np.pi/180)
 test_df['dropoff_long_30'] = test_df['dropoff_longitude']*np.cos(30* np.pi / 180)- test_df['dropoff_latitude']*np.sin(30* np.pi/180)
 test_df['dropoff_long_45'] = test_df['dropoff_longitude']*np.cos(45* np.pi / 180)- test_df['dropoff_latitude']*np.sin(45* np.pi/180)
 test_df['dropoff_long_60'] = test_df['dropoff_longitude']*np.cos(60* np.pi / 180)- test_df['dropoff_latitude']*np.sin(60* np.pi/180)
 test_df['dropoff_long_75'] = test_df['dropoff_longitude']*np.cos(75* np.pi / 180)- test_df['dropoff_latitude']*np.sin(75* np.pi/180)
 test_df['dropoff_lat_15'] = test_df['dropoff_longitude']*np.sin(15* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(15* np.pi/180)
 test_df['dropoff_lat_30'] = test_df['dropoff_longitude']*np.sin(30* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(30* np.pi/180)
 test_df['dropoff_lat_45'] = test_df['dropoff_longitude']*np.sin(45* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(45* np.pi/180)
 test_df['dropoff_lat_60'] = test_df['dropoff_longitude']*np.sin(60* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(60* np.pi/180)
 test_df['dropoff_lat_75'] = test_df['dropoff_longitude']*np.sin(75* np.pi / 180)+ test_df['dropoff_latitude']*np.cos(75* np.pi/180)
 test_key = test_df['key']
x_pred = test_df.drop(columns=['key', 'pickup_datetime'])
 prediction = model.predict(xgb.DMatrix(x_pred), ntree_limit = model.best_ntree_limit )<save_to_csv>
tr = tr.merge(pd_patient, on='Patient', how='left')
 tr.head()
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission<import_modules>
batch = 128
grph = True
kaggle = True
if grph: 
<set_options>
class Data_Generate(Dataset):
def __init__(self,data,label=None):
self.data = data
self.label = label
def __getitem__(self,index):
z_ = self.data[index]
if self.label is not None:
y_ = self.label[index]
y_ = y_[None,]
return z_,y_
else:
return z_
def __len__(self):
return len(self.data )
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 pd.set_option('display.max_colwidth', -1)
 pd.set_option('mode.chained_assignment','raise' )<load_from_csv>
class EarlyStopping:

 def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt'):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
self.delta = delta
self.path = path
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_loss, model)
 elif score < self.best_score + self.delta:
self.counter += 1
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , self.path)
 self.val_loss_min = val_loss
def read_files(chunks = 1100, chunk_no = None):
random.seed(42)
 nrows = int(55423857 / chunks)
 def skiprows(x):
if x == 0:
return False
return chunk_no != random.randrange(chunks)
 if kaggle:
df = pd.read_csv('.. /input/train.csv', nrows = nrows, parse_dates=["pickup_datetime"])
 else:
with zipfile.ZipFile('all.zip')as zipf:
with zipf.open('train.csv')as myZip:
if chunk_no == None:
df = pd.read_csv(myZip,nrows=nrows,parse_dates=["pickup_datetime"])
 else:
df = pd.read_csv(myZip,skiprows=skiprows,parse_dates=["pickup_datetime"])
 random.seed(42)
 df = df.sample(frac=1 ).reset_index(drop=True)
 if kaggle:
df_test = pd.read_csv('.. /input/test.csv', nrows = nrows, parse_dates=["pickup_datetime"])
 else:
with zipfile.ZipFile('all.zip')as zipf:
with zipf.open('test.csv')as myZip:
df_test = pd.read_csv(myZip,parse_dates=["pickup_datetime"])
 return df, df_test<define_variables>
%%time
cnt = 0
EPOCHS = 1000
criterion = mloss(0.3)
 for fold in range(NFOLD):
list_train_loss,list_val_loss,list_train_score,list_val_score = [],[],[],[]
val_out = []
print(f"FOLD {fold+1}")
 tr_z = tr[FE][tr.fold!=fold].values.astype(np.float32)
 tr_y = tr.res_fvc[tr.fold!=fold].values.astype(np.float32)
 val_z = tr[FE][tr.fold==fold].values.astype(np.float32)
 val_y = tr.res_fvc[tr.fold==fold].values.astype(np.float32)
 base_tr = tr['min_FVC'][tr.fold!=fold].values.astype(np.float32)
 base_val = tr['min_FVC'][tr.fold!=fold].values.astype(np.float32)
 train_db = Data_Generate(tr_z,tr_y)
 train_loader = DataLoader(train_db, batch_size=batch, shuffle=True, num_workers=4)
 val_db = Data_Generate(val_z,val_y)
 val_loader = DataLoader(val_db, batch_size=batch, shuffle=False, num_workers=4)
 tr_num_batch = len(train_loader)
 val_num_batch = len(val_loader)
 net = make_big_model(len(FE),3 ).to(device)
 optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=0.001 ,weight_decay=5e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=20, min_lr=1e-8, verbose=False)
 early_stopping = EarlyStopping(patience=80,path=f'Osic-NN-fold_{fold}.pth',verbose=False)
 for epoch in tqdm(range(EPOCHS)) :
train_loss,train_score,val_loss,val_score = 0,0,0,0
net.train()
for idx, sample in enumerate(train_loader):
feature, label = sample
feature, label = feature.to(device), label.to(device)
 out = net(feature)
 loss = criterion(label, out)
 score_ = score(label ,out)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss += loss.item() /tr_num_batch
train_score += score_.item() /tr_num_batch
list_train_loss.append(train_loss)
 list_train_score.append(train_score)
 net.eval()
for idx, sample in enumerate(val_loader):
feature, label = sample
feature, label = feature.to(device), label.to(device)
 with torch.no_grad() :
out = net(feature)
 loss = criterion(label, out)
 score_ = score(label, out)
 val_loss += loss.item() /val_num_batch
val_score += score_.item() /val_num_batch
list_val_loss.append(val_loss)
 list_val_score.append(val_score)
 early_stopping(val_score, net)
 if early_stopping.early_stop:
print("Early stopping")
 break
scheduler.step(val_loss)
 print(f"train loss: {min(list_train_loss)} train score: {min(list_train_score)}
\
val loss: {min(list_val_loss)} val score: {min(list_val_score)}
\
final lr: {optimizer.param_groups[0]['lr']}" 
)
chunks = 8<load_from_csv>
def score_np(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 scores,sigma,mean_FVC,true_y = [],[],[],[]
tr['pred_FVC']=-1
tr['Confidence']=-1
preds = []
for fold in range(NFOLD):
pred = []
net = make_big_model(in_ch=len(FE)).to(device)
 net.load_state_dict(torch.load(f"Osic-NN-fold_{fold}.pth"))
net.eval()
tr_z = tr[FE][tr.fold==fold].values.astype(np.float32)
 tr_y = tr.FVC[tr.fold==fold].values.astype(np.float32)
 base_val = tr['min_FVC'][tr.fold==fold].values.astype(np.float32)
 valid_db = Data_Generate(tr_z,tr_y)
 valid_loader = DataLoader(valid_db, batch_size=batch, shuffle=False, num_workers=4)
 for idx, sample in enumerate(valid_loader):
feature, label = sample
feature, label = feature.to(device), label.to(device)
 with torch.no_grad() :
out = net(feature)
 out = out.cpu().numpy()
pred.append(out)
 pred = np.concatenate(pred)
 preds.append(pred)
 tr.loc[tr.fold==fold,'pred_FVC'] = base_val - pred[:,1]
tr.loc[tr.fold==fold,'pred_Confidence'] = abs(pred[:,2]-pred[:,0])
 order_tr = tr.groupby(tr['Patient'] ).apply(lambda x:x.sort_values("Weeks",ascending=False))
p_id = pd.unique(tr.Patient)
 for i in range(len(tr)) :
sigma.append(tr.iloc[i].pred_Confidence)
 mean_FVC.append(tr.iloc[i].pred_FVC)
 true_y.append(tr.iloc[i].FVC)
 score_np(np.array(true_y),np.array(mean_FVC),np.array(sigma))
df,df_test = read_files(chunks )<feature_engineering>
all_mean,last_mean = [],[]
for i in tr.Patient.unique() :
sigma = tr[tr.Patient==i].pred_Confidence
mean_FVC = tr[tr.Patient==i].pred_FVC
true_y = tr[tr.Patient==i].FVC
all_mean.append(score_np(np.array(true_y),np.array(mean_FVC),np.array(sigma)))
 last_mean.append(score_np(np.array(true_y[-3:]),np.array(mean_FVC[-3:]),np.array(sigma[-3:])))
 print(np.mean(all_mean),np.mean(last_mean))
def calculate_distance(data):
for df in data:
R=6371.0
pi=math.pi
df_tmp = df.eval(,inplace=False)
 df['dist'] = df_tmp['distance']
df['dist2'] = df_tmp['distance'] * df_tmp['distance']
return data<feature_engineering>
tr_1 = tr.copy()
long_km =.1 / 11.160876733592401
lat_km =.1 / 3.077915051090274
def is_airport(data):
for df in data:
df["is_jfk_pickup"] = False
df.loc[df.eval(
 '(-73.778889 - 2*@long_km < pickup_longitude < -73.778889 + 2*@long_km)\
&\ 
(40.639722 - 1*@lat_km < pickup_latitude < 40.639722 +.5*@lat_km)')
 ,"is_jfk_pickup"] = True
df["is_ewr_pickup"] = False
df.loc[df.eval(
 '(-74.168611 - 2*@long_km < pickup_longitude < -74.168611 + 2*@long_km)\
&\ 
(40.6925 - 1*@lat_km < pickup_latitude < 40.6925 + 1*@lat_km)')
 ,"is_ewr_pickup"] = True
df["is_lga_pickup"] = False
df.loc[df.eval(
 '(-73.872611 - 1.4*@long_km < pickup_longitude < -73.872611 + 1.5*@long_km)\
&\ 
(40.77725 -.3*@lat_km < pickup_latitude < 40.77725 +.3*@lat_km)')
 ,"is_lga_pickup"] = True
df["is_jfk_dropoff"] = False
df.loc[df.eval(
 '(-73.778889 - 2*@long_km < dropoff_longitude < -73.778889 + 2*@long_km)\
&\ 
(40.639722 - 1*@lat_km < dropoff_latitude < 40.639722 +.5*@lat_km)')
 ,"is_jfk_dropoff"] = True
df["is_ewr_dropoff"] = False
df.loc[df.eval(
 '(-74.168611 - 2*@long_km < dropoff_longitude < -74.168611 + 2*@long_km)\
&\ 
(40.6925 - 1*@lat_km < dropoff_latitude < 40.6925 + 1*@lat_km)')
 ,"is_ewr_dropoff"] = True
df["is_lga_dropoff"] = False
df.loc[df.eval(
 '(-73.872611 - 1.4*@long_km < dropoff_longitude < -73.872611 + 1.5*@long_km)\
&\ 
(40.77725 -.3*@lat_km < dropoff_latitude < 40.77725 +.3*@lat_km)')
 ,"is_lga_dropoff"] = True
return data 
<compute_test_metric>
df,df_test = calculate_distance([df,df_test] )<compute_test_metric>
test = pd.read_csv(f'{ROOT}/test.csv')
 for i in test.Patient:
sub.loc[sub.Patient==i,'week'] = sub[sub.Patient==i].Weeks.values-test[test.Patient==i].Weeks.values
sub
df,df_test = is_airport([df,df_test] )<feature_engineering>
ze =(sub[FE].values ).astype(np.float32)
 base_eval =(sub['min_FVC'].values ).astype(np.float32)
 pe = np.zeros(( ze.shape[0], 3))
test_db = Data_Generate(ze)
 test_loader = DataLoader(test_db, batch_size=batch, shuffle=False, num_workers=4 )
def is_business_day(date):
return bool(len(pd.bdate_range(date, date)))
 def time_related_vars(data):
for df in data:
df['weekday'] = df['pickup_datetime'].dt.weekday
df['day'] = df['pickup_datetime'].dt.day
df['hour'] = df['pickup_datetime'].dt.hour
df['month'] = df['pickup_datetime'].dt.month
df['year'] = df['pickup_datetime'].dt.year
df['daytime'] = False
df.loc[(df.hour >= 8)&(df.hour < 20),'daytime'] = True
df['time_stamp'] = df['year'].astype('str')+ '-' + df['month'].astype('str')+ '-' + df['day'].astype('str')+ ':' + df['hour'].astype('str')
 return data<compute_test_metric>
print("predict test...")
 pe = 0
for k in range(NFOLD):
if k == 2:
continue
pred = []
base_fvc = sub.FVC.values.astype(np.float32)
 net = make_big_model(len(FE)).to(device)
 net.load_state_dict(torch.load(f"Osic-NN-fold_{k}.pth"))
net.eval()
for idx, sample in enumerate(test_loader):
data = sample
data = data.to(device)
 with torch.no_grad() :
out = net(data)
 out = out.cpu().numpy()
pred.append(out)
 pred = base_fvc[:,None] - np.concatenate(pred)
 pe += pred / 4
df,df_test = time_related_vars(( df,df_test))<merge>
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = abs(pe[:, 2] - pe[:, 0] )
grp = df.groupby('time_stamp',as_index=False)['fare_amount'].agg(['sum','mean','std','skew','count'] ).reset_index()
df = df.merge(grp, how='left', on=['time_stamp'])
 df_test = df_test.merge(grp, how='left', on=['time_stamp'] )<feature_engineering>
subm = sub[['Patient_Week','FVC1','Confidence1']].copy()
subm.rename(columns={'FVC1':'FVC','Confidence1':'Confidence'},inplace=True)
 subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )
def basic_fare() :
df['min_fare'] = 2.5 + 0.5 * 5 * df.dist / 1.6
df_test['min_fare'] = 2.5 + 0.5 * 5 * df_test.dist / 1.6
df['min_multiple'] = df['fare_amount'] / df['min_fare']<categorify>
from sklearn.model_selection import KFold
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor
from sklearn.multioutput import MultiOutputRegressor
import joblib
if not kaggle:
df.to_pickle('df.pkl')
 df_test.to_pickle('df_test.pkl' )<load_pretrained>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 feature_ct = pd.read_csv('.. /input/2020osic/CT_21feature.csv')
 tr = tr.merge(feature_ct, on='Patient')
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','FVC']] 
.rename({'FVC': 'min_FVC'}, axis=1)
.groupby('Patient')
.first() 
.reset_index() 
)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','Percent']] 
.rename({'Percent': 'min_Percent'}, axis=1)
.groupby('Patient')
.first() 
.reset_index() 
)
 data = data.merge(base, on='Patient', how='left')
 del base
data.loc[:,"Sex"] = pd.factorize(data.Sex)[0]
data.loc[:,"SmokingStatus"] = pd.factorize(data.SmokingStatus)[0]
train_df = data[data.WHERE=='train']
train_df.head()
if not kaggle:
df = pd.read_pickle('df.pkl')
 df_test = pd.read_pickle('df_test.pkl' )<drop_column>
train_reg = pd.DataFrame(columns=['Patient', 'q2_Slope', 'q5_Slope','q8_Slope','slp'])
 def model_slope(x):
b = x[0][0]
slope_1 = [(t[0]-b)/t[1] for t in x[1:]]
sort_slope = sorted(slope_1,reverse=True)
 length = len(slope_1)
 return np.mean(slope_1),np.mean(sort_slope[:length//2]),np.mean(sort_slope[length//2:])
 for idx,i in enumerate(pd.unique(train_df['Patient'])) :
FVC_Weeks = train_df[train_df.Patient==i][['FVC','base_week']].values
slope_m,slope_f,slope_l = model_slope(FVC_Weeks)
 sub = train_df.loc[train_df.Patient == i, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
train_reg.loc[idx] = [i,slope_m,slope_f,slope_l,a]
train_reg.head()
df.dropna(inplace=True)
 df.drop(df.index[(df.pickup_longitude < -75)| 
(df.pickup_longitude > -72)| 
(df.pickup_latitude < 40)| 
(df.pickup_latitude > 42)],inplace=True)
 df.drop(df.index[(df.dropoff_longitude < -75)| 
(df.dropoff_longitude > -72)| 
(df.dropoff_latitude < 40)| 
(df.dropoff_latitude > 42)],inplace=True )<init_hyperparams>
final_df = pd.DataFrame(columns=train_df.columns)
 for idx,i in enumerate(pd.unique(train_df['Patient'])) :
temp_df = train_df[train_df.Patient==i]
final_df = final_df.append(temp_df[temp_df.Weeks==temp_df.min_week].iloc[0,:])
 train_reg = train_reg.merge(final_df)
 train_reg = train_reg.drop(['min_week','base_week'],axis=1)
 train_reg.head()
max_fare_multiple = 20
min_fare_multiple = 0.05<drop_column>
train_reg.loc[:,"Sex"] = pd.factorize(train_reg.Sex)[0]
train_reg.loc[:,"SmokingStatus"] = pd.factorize(train_reg.SmokingStatus)[0]
train_reg['SmokingStatus'] =(train_reg['SmokingStatus'] - train_reg['SmokingStatus'].min())/(train_reg['SmokingStatus'].max() - train_reg['SmokingStatus'].min())
 train_reg.head()
drop_too_expensive = df.index[(df.fare_amount >(max_fare_multiple *(2.5 + 0.5 * 5 * df.dist / 1.6)))]
print("n_dropped={}, {}%".format(len(drop_too_expensive),len(drop_too_expensive)/len(df)))
 df.drop(drop_too_expensive,inplace=True )<drop_column>
NFOLD = 5
kf = KFold(n_splits=NFOLD)
 train_reg['fold']=-1
for idx,(tr_idx, val_idx)in enumerate(kf.split(train_reg)) :
train_reg.loc[val_idx,"fold"] = idx
train_reg.head()
drop_too_cheap = df.index[df.fare_amount <(min_fare_multiple *(2.5 + 0.5 * 5 * df.dist / 1.6)) ]
print("n_dropped={}, {}%".format(len(drop_too_cheap),len(drop_too_cheap)/len(df)))
 df.drop(drop_too_cheap,inplace=True )<data_type_conversions>
train_reg = train_reg.merge(feature_ct)
 train_reg_bp = train_reg.copy()
train_reg.head()
df['float_pickup_datetime'] = df['pickup_datetime'].values.astype('float')
 df_test['float_pickup_datetime'] = df_test['pickup_datetime'].values.astype('float' )<split>
train_reg['age'] =(train_reg['Age'] - train_reg['Age'].min())/(train_reg['Age'].max() - train_reg['Age'].min())
 train_reg['BASE'] =(train_reg['min_FVC'] - train_reg['min_FVC'].min())/(train_reg['min_FVC'].max() - train_reg['min_FVC'].min())
 train_reg['percent'] =(train_reg['min_Percent'] - train_reg['min_Percent'].min())/(train_reg['min_Percent'].max() - train_reg['min_Percent'].min())
 train_reg['volume'] =(train_reg['Volume'] - train_reg['Volume'].min())/(train_reg['Volume'].max() - train_reg['Volume'].min())
 train_reg['mean'] =(train_reg['Mean'] - train_reg['Mean'].min())/(train_reg['Mean'].max() - train_reg['Mean'].min())
 train_reg['skew'] =(train_reg['Skew'] - train_reg['Skew'].min())/(train_reg['Skew'].max() - train_reg['Skew'].min())
 train_reg['median'] =(train_reg['Median'] - train_reg['Median'].min())/(train_reg['Median'].max() - train_reg['Median'].min())
 train_reg['kurthosis'] =(train_reg['Kurthosis'] - train_reg['Kurthosis'].min())/(train_reg['Kurthosis'].max() - train_reg['Kurthosis'].min())
 train_reg['haa'] =(train_reg['HAA'] - train_reg['HAA'].min())/(train_reg['HAA'].max() - train_reg['HAA'].min())
 train_reg['midmedian'] =(train_reg['midMedian'] - train_reg['midMedian'].min())/(train_reg['midMedian'].max() - train_reg['midMedian'].min())
 train_reg['midmean'] =(train_reg['midMean'] - train_reg['midMean'].min())/(train_reg['midMean'].max() - train_reg['midMean'].min())
 train_reg['midskew'] =(train_reg['midSkew'] - train_reg['midSkew'].min())/(train_reg['midSkew'].max() - train_reg['midSkew'].min())
 train_reg['midkurthosis'] =(train_reg['midKurthosis'] - train_reg['midKurthosis'].min())/(train_reg['midKurthosis'].max() - train_reg['midKurthosis'].min())
 train_reg['midhaa'] =(train_reg['midHAA'] - train_reg['midHAA'].min())/(train_reg['midHAA'].max() - train_reg['midHAA'].min())
 train_reg['pro_0'] =(train_reg['pro_0']-train_reg['pro_0'].min())/(train_reg['pro_0'].max() - train_reg['pro_0'].min())
 train_reg['pro_100'] =(train_reg['pro_100']-train_reg['pro_100'].min())/(train_reg['pro_100'].max() - train_reg['pro_100'].min())
 train_reg['pro_200'] =(train_reg['pro_200']-train_reg['pro_200'].min())/(train_reg['pro_200'].max() - train_reg['pro_200'].min())
 train_reg['pro_300'] =(train_reg['pro_300']-train_reg['pro_300'].min())/(train_reg['pro_300'].max() - train_reg['pro_300'].min())
 train_reg['pro_400'] =(train_reg['pro_400']-train_reg['pro_400'].min())/(train_reg['pro_400'].max() - train_reg['pro_400'].min())
 train_reg['pro_500'] =(train_reg['pro_500']-train_reg['pro_500'].min())/(train_reg['pro_500'].max() - train_reg['pro_500'].min())
 train_reg['pro_600'] =(train_reg['pro_600']-train_reg['pro_600'].min())/(train_reg['pro_600'].max() - train_reg['pro_600'].min())
 train_reg['pro_700'] =(train_reg['pro_700']-train_reg['pro_700'].min())/(train_reg['pro_700'].max() - train_reg['pro_700'].min())
 train_reg['pro_800'] =(train_reg['pro_800']-train_reg['pro_800'].min())/(train_reg['pro_800'].max() - train_reg['pro_800'].min())
 train_reg['pro_900'] =(train_reg['pro_900']-train_reg['pro_900'].min())/(train_reg['pro_900'].max() - train_reg['pro_900'].min())
 train_reg['Sex'] =(train_reg['Sex'] - train_reg['Sex'].min())/(train_reg['Sex'].max() - train_reg['Sex'].min())
 train_reg['SmokingStatus'] =(train_reg['SmokingStatus'] - train_reg['SmokingStatus'].min())/(train_reg['SmokingStatus'].max() - train_reg['SmokingStatus'].min() )
dftr = df.sample(frac = 0.8, random_state=42)
 dfval = df.drop(dftr.index )<define_variables>
ll = ['passenger_count','dist','bday','weekday','hour','daytime','businesstime','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude','lat_diff','long_diff']
ll = ['dist']
ll = ['passenger_count','dist']
ll = ['dist','bday']
ll = ['dist','bday','weekday']
ll = ['dist','bday','weekday','hour']
ll = ['dist','bday','weekday','hour','daytime']
ll = ['dist','bday','weekday','hour','businesstime']
ll = ['dist','bday','weekday','hour','daytime','businesstime']
ll = ['dist','month']
ll = ['dist','bday','weekday','hour','daytime','month']
ll = ['dist','bday','weekday','hour','daytime','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude']
ll = ['dist','bday','weekday','hour','daytime','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude','lat_diff','long_diff']
ll = ['dist','bday','weekday','hour','daytime','month','lat_avg','long_avg','lat_diff','long_diff']
ll = ['dist','bday','weekday','hour','daytime','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude']
ll = ['dist','dist2','weekday','hour','daytime','year','month','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude',
'is_jfk_pickup', 'is_ewr_pickup', 'is_lga_pickup', 'is_jfk_dropoff', 'is_ewr_dropoff', 'is_lga_dropoff']
ll = ['dist','dist2','float_pickup_datetime', 'weekday','hour','daytime','year','month','pickup_longitude',
'pickup_latitude','dropoff_longitude','dropoff_latitude','is_jfk_pickup', 'is_ewr_pickup',
'is_lga_pickup', 'is_jfk_dropoff', 'is_ewr_dropoff', 'is_lga_dropoff',
'sum','mean','std','skew','count'] 
<prepare_x_and_y>
FE = ['percent','SmokingStatus','Sex','age','BASE','volume','mean','skew','kurthosis','haa','median',
'midmean','midmedian','midskew','midkurthosis','midhaa',
'pro_0','pro_100','pro_200','pro_300','pro_400','pro_500',
'pro_600','pro_700','pro_800','pro_900']
train_reg['pre_s'] = -1
m,last3 = [],[]
for n in range(NFOLD):
tr_x,val_x = train_reg[train_reg.fold!=n][FE].values,train_reg[train_reg.fold==n][FE].values
tr_y,val_y = train_reg[train_reg.fold!=n]['slp'].values,train_reg[train_reg.fold==n]['slp'].values
randomforest = RandomForestRegressor(n_estimators = 500, oob_score = True, n_jobs = -1,
max_features = "sqrt", min_samples_leaf = 5, max_samples = 0.8)
 randomforest.fit(tr_x, tr_y)
 joblib.dump(randomforest, f'./all_rf_{n}.pkl')
 index = train_reg[train_reg.fold==n].index
train_reg.loc[index,['pre_s']] = randomforest.predict(val_x)
 s = np.mean(abs(train_reg.loc[index,['pre_s']].values-train_reg.loc[index,['slp']].values))
print(f"{n}th model's out-of-bag score : ", s)
 for p in train_reg[train_reg.fold==n].Patient:
percent_true = train_df.Percent.values[train_df.Patient == p]
fvc_true = train_df.FVC.values[train_df.Patient == p]
weeks_true = train_df.Weeks.values[train_df.Patient == p]
a = train_reg[train_reg.Patient==p].pre_s.values
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 train_df.loc[train_df.Patient==p,'img_fvc'] = fvc
train_df.loc[train_df.Patient==p,'img_con'] = percent
m.append(score_np(fvc_true, fvc,percent))
last3.append(score_np(fvc_true[-3:], fvc[-3:],percent[-3:]))
print(f'all_mean:{(np.mean(m)) },last3_mean:{np.mean(last3)}' )
ntr = len(dftr['dist'])
 nval = len(dfval['dist'])
 nte = len(df_test['dist'])
 Xtr = pd.DataFrame(dftr[ll] ).values.reshape(ntr, len(ll))
Xval = pd.DataFrame(dfval[ll] ).values.reshape(nval, len(ll))
Xte = pd.DataFrame(df_test[ll] ).values.reshape(nte, len(ll))
Ytr = dftr['min_multiple'].values
Yval = dfval['min_multiple'].values<choose_model_class>
test = pd.read_csv(f"{ROOT}/test.csv")
 test_df = test.merge(lung_stat_pd,)
 test_df.head()
max_bin = 2*255
gbm = lgb.LGBMRegressor(n_estimators=10000, silent=False, max_bin=max_bin )<train_model>
train_reg = train_reg_bp
test_df['age'] =(test_df['Age'] - train_reg['Age'].min())/(train_reg['Age'].max() - train_reg['Age'].min())
 test_df['BASE'] =(test_df['FVC'] - train_reg['min_FVC'].min())/(train_reg['min_FVC'].max() - train_reg['min_FVC'].min())
 test_df['percent'] =(test_df['Percent'] - train_reg['min_Percent'].min())/(train_reg['min_Percent'].max() - train_reg['min_Percent'].min())
 test_df['volume'] =(test_df['Volume'] - train_reg['Volume'].min())/(train_reg['Volume'].max() - train_reg['Volume'].min())
 test_df['mean'] =(test_df['Mean'] - train_reg['Mean'].min())/(train_reg['Mean'].max() - train_reg['Mean'].min())
 test_df['skew'] =(test_df['Skew'] - train_reg['Skew'].min())/(train_reg['Skew'].max() - train_reg['Skew'].min())
 test_df['median'] =(test_df['Median'] - train_reg['Median'].min())/(train_reg['Median'].max() - train_reg['Median'].min())
 test_df['kurthosis'] =(test_df['Kurthosis'] - train_reg['Kurthosis'].min())/(train_reg['Kurthosis'].max() - train_reg['Kurthosis'].min())
 test_df['haa'] =(test_df['HAA'] - train_reg['HAA'].min())/(train_reg['HAA'].max() - train_reg['HAA'].min())
 test_df['midmedian'] =(test_df['midMedian'] - train_reg['midMedian'].min())/(train_reg['midMedian'].max() - train_reg['midMedian'].min())
 test_df['midmean'] =(test_df['midMean'] - train_reg['midMean'].min())/(train_reg['midMean'].max() - train_reg['midMean'].min())
 test_df['midskew'] =(test_df['midSkew'] - train_reg['midSkew'].min())/(train_reg['midSkew'].max() - train_reg['midSkew'].min())
 test_df['midkurthosis'] =(test_df['midKurthosis'] - train_reg['midKurthosis'].min())/(train_reg['midKurthosis'].max() - data['midKurthosis'].min())
 test_df['midhaa'] =(test_df['midHAA'] - train_reg['midHAA'].min())/(train_reg['midHAA'].max() - train_reg['midHAA'].min())
 test_df['pro_0'] =(test_df['pro_0']-train_reg['pro_0'].min())/(train_reg['pro_0'].max() - train_reg['pro_0'].min())
 test_df['pro_100'] =(test_df['pro_100']-train_reg['pro_100'].min())/(train_reg['pro_100'].max() - train_reg['pro_100'].min())
 test_df['pro_200'] =(test_df['pro_200']-train_reg['pro_200'].min())/(train_reg['pro_200'].max() - train_reg['pro_200'].min())
 test_df['pro_300'] =(test_df['pro_300']-train_reg['pro_300'].min())/(train_reg['pro_300'].max() - train_reg['pro_300'].min())
 test_df['pro_400'] =(test_df['pro_400']-train_reg['pro_400'].min())/(train_reg['pro_400'].max() - train_reg['pro_400'].min())
 test_df['pro_500'] =(test_df['pro_500']-train_reg['pro_500'].min())/(train_reg['pro_500'].max() - train_reg['pro_500'].min())
 test_df['pro_600'] =(test_df['pro_600']-train_reg['pro_600'].min())/(train_reg['pro_600'].max() - train_reg['pro_600'].min())
 test_df['pro_700'] =(test_df['pro_700']-train_reg['pro_700'].min())/(train_reg['pro_700'].max() - train_reg['pro_700'].min())
 test_df['pro_800'] =(test_df['pro_800']-train_reg['pro_800'].min())/(train_reg['pro_800'].max() - train_reg['pro_800'].min())
 test_df['pro_900'] =(test_df['pro_900']-train_reg['pro_900'].min())/(train_reg['pro_900'].max() - train_reg['pro_900'].min())
 test_df.loc[:,"Sex"] = pd.factorize(test_df.Sex)[0]
test_df.loc[:,"SmokingStatus"] = pd.factorize(test_df.SmokingStatus)[0]
test_df['Sex'] =(test_df['Sex'] - train_reg['Sex'].min())/(train_reg['Sex'].max() - train_reg['Sex'].min())
 test_df['SmokingStatus'] =(test_df['SmokingStatus'] - train_reg['SmokingStatus'].min())/(train_reg['SmokingStatus'].max() - train_reg['SmokingStatus'].min() )
parameters = gbm.fit(Xtr, Ytr )<load_pretrained>
sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 sub.head()
FE = ['percent','SmokingStatus','Sex','age','BASE','volume','mean','skew','kurthosis','haa','median',
'midmean','midmedian','midskew','midkurthosis','midhaa',
'pro_0','pro_100','pro_200','pro_300','pro_400','pro_500',
'pro_600','pro_700','pro_800','pro_900']
Y = ['slp']
pred_y = []
for n in range(NFOLD):
val_x = test_df[FE]
randomforest = joblib.load(f'./all_rf_{n}.pkl')
 pred_y.append(randomforest.predict(val_x))
pred_y = np.array(pred_y ).mean(0)
 test_df.loc[:,'preslp'] = pred_y
for p in test_df.Patient:
a = test_df[test_df.Patient==p].preslp.values
weeks_base = test_df[test_df.Patient==p].Weeks.values
weeks_true = sub[sub.Patient==p].Weeks.values
percent_true = sub[sub.Patient==p].Percent.values
fvc_true = sub[sub.Patient==p].FVC.values
fvc = a *(weeks_true - weeks_base[0])+ fvc_true[0]
percent = weeks_base[0] - a * abs(weeks_true - weeks_base[0])
 sub.loc[sub.Patient==p,'FVC'] = fvc
sub.loc[sub.Patient==p,'Confidence'] = percent
sub.head()
subs = sub[["Patient_Week","FVC","Confidence"]]
<load_pretrained><EOS> 
l = 0.
 m,last3 = [],[]
for i in train_df.Patient:
true_fvc = tr_1[tr_1.Patient==i].FVC
pred_fvc =(train_df[train_df.Patient==i].img_fvc)*l +(1-l)*tr_1[tr_1.Patient==i].pred_FVC
pred_sig =(train_df[train_df.Patient==i].img_con)*l +(1-l)*tr_1[tr_1.Patient==i].pred_Confidence
m.append(score_np(true_fvc, pred_fvc,pred_sig))
last3.append(score_np(true_fvc[-3:], pred_fvc[-3:],pred_sig[-3:]))
print(f'all_mean:{(np.mean(m)) },last3_mean:{np.mean(last3)}' )