Kaggle competition: House Prices - Advanced Regression Techniques
sub.to_csv("submission.csv", index=False )<set_options>
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
 train_ids=train['Id']
test_ids=test['Id']
train.drop('Id',axis=1, inplace=True)
 test.drop('Id',axis=1, inplace=True )
warnings.filterwarnings("ignore" )<import_modules>
show_missing_values(train )
import torch.nn as nn
import torch
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from scipy.ndimage import zoom
from scipy.stats import kurtosis
from scipy.stats import skew<set_options>
corr=train.corr().abs()
n_most_correlated=12
most_correlated_feature=corr['SalePrice'].sort_values(ascending=False)[:n_most_correlated].drop('SalePrice')
 most_correlated_feature_name=most_correlated_feature.index.values
def seed_everything(seed=2020):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.enabled = True
seed_everything(42 )<load_from_csv>
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
device = torch.device('cuda')
 tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 feature_ct = pd.read_csv('.. /input/2020osic/CT_21feature.csv')
 tr = tr.merge(feature_ct, on='Patient')
 chunk = pd.read_csv(f"{ROOT}/test.csv" )<normalization>
ntrain = train.shape[0]
ntest = test.shape[0]
def load_scan(path,resize_type='no'):

 slices = [pydicom.read_file(path + '/' + s)for s in os.listdir(path)]
slices.sort(key = lambda x: int(x.InstanceNumber))
try:
slice_thickness = abs(slices[-1].ImagePositionPatient[2] - slices[0].ImagePositionPatient[2])/(len(slices))
except:
try:
slice_thickness = abs(slices[-1].SliceLocation - slices[0].SliceLocation)/(len(slices))
except:
slice_thickness = slices[0].SliceThickness
for s in slices:
s.SliceThickness = slice_thickness
if resize_type == 'resize':
s.PixelSpacing = s.PixelSpacing*(s.Rows/512)
 return slices
def transform_to_hu(slices):

 images = np.stack([file.pixel_array for file in slices])
 images = images.astype(np.int16)
 images[images <= -1000] = 0
for n in range(len(slices)) :
intercept = slices[n].RescaleIntercept
slope = slices[n].RescaleSlope
if slope != 1:
images[n] = slope * images[n].astype(np.float64)
 images[n] = images[n].astype(np.int16)
 images[n] += np.int16(intercept)
 return np.array(images, dtype=np.int16 )<normalization>
y_train=train['SalePrice']
X_train=train.drop('SalePrice', axis='columns')
 datasets=pd.concat(( X_train, test),axis='index')
 print(datasets.shape )
class Test_Generate(Dataset):
def __init__(self,imgs_dicom):
self.imgs_dicom = imgs_dicom
def __getitem__(self,index):
metainf = self.imgs_dicom[index]
slice_img = metainf.pixel_array
slice_img =(slice_img-slice_img.min())/(slice_img.max() -slice_img.min())
 slice_img =(slice_img*255 ).astype(np.uint8)
 if metainf.Rows!=512 or metainf.Columns!=512:
slice_img = cv2.resize(slice_img,(512,512))
slice_img = slice_img[None,:,:]
slice_img =(slice_img/255 ).astype(np.float32)
 return slice_img
def __len__(self):
return len(self.imgs_dicom )<categorify>
show_missing_values(datasets )
device = torch.device('cuda:0')
 checkpoint = '.. /input/2020osic/best_lung_Unet_densenet121_my.pth'
Model_M = smp.Unet('densenet121', classes=1, in_channels=1,activation='sigmoid',encoder_weights=None ).to(device)
 Model_M.load_state_dict(torch.load(checkpoint))
def Unet_mask(model_m: nn.Module,input_data: DataLoader):
model_m.eval()
outs = []
for idx, sample in enumerate(test_loader):
image = sample
image = image.to(device)
 with torch.no_grad() :
out = model_m(image)
 out = out.cpu().data.numpy()
out = np.where(out>0.5,1,0)
 out = np.squeeze(out,axis=1)
 outs.append(out)
 outs = np.concatenate(outs)
 return outs<create_dataframe>
staretegies={}
staretegies['PoolQC']='None'
staretegies['MiscFeature']='None'
staretegies['Alley']='None'
staretegies['Fence']='None'
staretegies['FireplaceQu']='None'
staretegies['LotFrontage']='LotFrontage'
staretegies['GarageType']='None'
staretegies['GarageFinish']='None'
staretegies['GarageQual']='None'
staretegies['GarageCond']='None'
staretegies['GarageYrBlt']='Zero'
staretegies['GarageArea']='Zero'
staretegies['GarageCars']='Zero'
staretegies['BsmtFinSF1']='Zero'
staretegies['BsmtFinSF2']='Zero'
staretegies['BsmtUnfSF']='Zero'
staretegies['TotalBsmtSF']='Zero'
staretegies['BsmtFullBath']='Zero'
staretegies['BsmtHalfBath']='Zero'
staretegies['BsmtQual']='None'
staretegies['BsmtCond']='None'
staretegies['BsmtExposure']='None'
staretegies['BsmtFinType1']='None'
staretegies['BsmtFinType2']='None'
staretegies['MasVnrType']='None'
staretegies['MasVnrArea']='Zero'
staretegies['MSZoning']='Mode'
staretegies['Utilities']='Drop'
staretegies['Functional']='Functional'
staretegies['Electrical']='Mode'
staretegies['KitchenQual']='Mode'
staretegies['Exterior1st']='Mode'
staretegies['Exterior2nd']='Mode'
staretegies['SaleType']='Mode'
staretegies['MSSubClass']='None'
ct_root_path = '.. /input/osic-pulmonary-fibrosis-progression/test/'
lung_stat_pd = pd.DataFrame(columns=['Patient','Volume','Mean','Median','Skew','Kurthosis','HAA',
'midMean','midMedian','midSkew','midKurthosis','midHAA',
'pro_0','pro_100','pro_200','pro_300','pro_400','pro_500',
'pro_600','pro_700','pro_800','pro_900'])
 haa_thresh = [-600,-250]
for i,p in enumerate(tqdm(pd.unique(chunk['Patient']))):
lung_stat_pd.loc[i,'Patient'] = p
patient_scans = load_scan(ct_root_path + p)
 test_db = Test_Generate(patient_scans)
 test_loader = DataLoader(test_db, batch_size=8, shuffle=False, num_workers=4)
 masks = Unet_mask(Model_M,test_loader)
 patient_images = transform_to_hu(patient_scans)
 if patient_images[0].shape!=(512,512):
patient_images = preprocess_img(patient_images,'resize')
 lung_stat_pd.loc[i,'Volume'] = caculate_lung_volume(patient_scans,masks)
 statistical_characteristic = caculate_histgram_statistical(patient_images,masks,haa_thresh)
 lung_stat_pd.loc[i,'Mean'] = statistical_characteristic['Mean']
lung_stat_pd.loc[i,'Median'] = statistical_characteristic['Median']
lung_stat_pd.loc[i,'Skew'] = statistical_characteristic['Skew']
lung_stat_pd.loc[i,'Kurthosis'] = statistical_characteristic['Kurthosis']
lung_stat_pd.loc[i,'HAA'] = statistical_characteristic['HAA']
lung_stat_pd.loc[i,'midMean'] = statistical_characteristic['midMean']
lung_stat_pd.loc[i,'midMedian'] = statistical_characteristic['midMedian']
lung_stat_pd.loc[i,'midSkew'] = statistical_characteristic['midSkew']
lung_stat_pd.loc[i,'midKurthosis'] = statistical_characteristic['midKurthosis']
lung_stat_pd.loc[i,'midHAA'] = statistical_characteristic['midHAA']
lung_stat_pd.loc[i,'pro_0'] = statistical_characteristic['pro_0']
lung_stat_pd.loc[i,'pro_100'] = statistical_characteristic['pro_100']
lung_stat_pd.loc[i,'pro_200'] = statistical_characteristic['pro_200']
lung_stat_pd.loc[i,'pro_300'] = statistical_characteristic['pro_300']
lung_stat_pd.loc[i,'pro_400'] = statistical_characteristic['pro_400']
lung_stat_pd.loc[i,'pro_500'] = statistical_characteristic['pro_500']
lung_stat_pd.loc[i,'pro_600'] = statistical_characteristic['pro_600']
lung_stat_pd.loc[i,'pro_700'] = statistical_characteristic['pro_700']
lung_stat_pd.loc[i,'pro_800'] = statistical_characteristic['pro_800']
lung_stat_pd.loc[i,'pro_900'] = statistical_characteristic['pro_900']
lung_stat_pd.head()<feature_engineering>
def fill_missing_values(fill_data, mystaretegies):
for column, strategy in mystaretegies.items() :
if strategy=='None':
fill_data[column]=fill_data[column].fillna('None')
 elif strategy=='Zero':
fill_data[column]=fill_data[column].fillna(0)
 elif strategy=='Mode':
fill_data[column]=fill_data[column].fillna(fill_data[column].mode() [0])
 elif strategy=='LotFrontage':
fill_data[column]=fill_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
elif strategy=='Drop':
fill_data=fill_data.drop([column],axis=1)
 elif strategy=='Functional':
fill_data[column]=fill_data[column].fillna('Typ')
 return fill_data
chunk = chunk.merge(lung_stat_pd)
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 sub.head()<concatenate>
datasets_no_missing=fill_missing_values(datasets, staretegies )
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<feature_engineering>
for name in ['MSSubClass', 'OverallCond', 'YrSold', 'MoSold']:
datasets_no_missing[name]= datasets_no_missing[name].astype(str )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<groupby>
col_names=datasets_no_missing.columns.values
col_types=datasets_no_missing.dtypes
object_cols=[]
numeric_cols=[]
for col_name, col_type in zip(col_names, col_types):
if col_type=='object':
object_cols.append(col_name)
 else:
numeric_cols.append(col_name)
 print("String değerler içeren özellikler:")
 print(object_cols)
 print("
Sayıal değerler içeren özellikler:")
 print(numeric_cols )
base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','FVC']] 
.rename({'FVC': 'min_FVC'}, axis=1)
.groupby('Patient')
.first() 
.reset_index()
)<merge>
label_encoder_col_names =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','Percent']] 
.rename({'Percent': 'min_Percent'}, axis=1)
.groupby('Patient')
.first() 
.reset_index() 
)
 data = data.merge(base, on='Patient', how='left')
 del base
data.head()<feature_engineering>
for col_name in label_encoder_col_names:
labelEncoder=LabelEncoder()
labelEncoder.fit(datasets_no_missing[col_name].values)
 datasets_no_missing[col_name]=labelEncoder.transform(datasets_no_missing[col_name].values )
<feature_engineering>
datasets_no_missing['TotalSF'] = datasets_no_missing['TotalBsmtSF'] + datasets_no_missing['1stFlrSF'] + datasets_no_missing['2ndFlrSF']
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['week'] = data['base_week']
data['percent'] =(data['min_Percent'] - data['min_Percent'].min())/(data['min_Percent'].max() - data['min_Percent'].min())
 data['volume'] =(data['Volume'] - data['Volume'].min())/(data['Volume'].max() - data['Volume'].min())
 data['mean'] =(data['Mean'] - data['Mean'].min())/(data['Mean'].max() - data['Mean'].min())
 data['skew'] =(data['Skew'] - data['Skew'].min())/(data['Skew'].max() - data['Skew'].min())
 data['median'] =(data['Median'] - data['Median'].min())/(data['Median'].max() - data['Median'].min())
 data['kurthosis'] =(data['Kurthosis'] - data['Kurthosis'].min())/(data['Kurthosis'].max() - data['Kurthosis'].min())
 data['haa'] =(data['HAA'] - data['HAA'].min())/(data['HAA'].max() - data['HAA'].min())
 data['midmedian'] =(data['midMedian'] - data['midMedian'].min())/(data['midMedian'].max() - data['midMedian'].min())
 data['midmean'] =(data['midMean'] - data['midMean'].min())/(data['midMean'].max() - data['midMean'].min())
 data['midskew'] =(data['midSkew'] - data['midSkew'].min())/(data['midSkew'].max() - data['midSkew'].min())
 data['midkurthosis'] =(data['midKurthosis'] - data['midKurthosis'].min())/(data['midKurthosis'].max() - data['midKurthosis'].min())
 data['midhaa'] =(data['midHAA'] - data['midHAA'].min())/(data['midHAA'].max() - data['midHAA'].min())
 data['pro_0'] =(data['pro_0']-data['pro_0'].min())/(data['pro_0'].max() - data['pro_0'].min())
 data['pro_100'] =(data['pro_100']-data['pro_100'].min())/(data['pro_100'].max() - data['pro_100'].min())
 data['pro_200'] =(data['pro_200']-data['pro_200'].min())/(data['pro_200'].max() - data['pro_200'].min())
 data['pro_300'] =(data['pro_300']-data['pro_300'].min())/(data['pro_300'].max() - data['pro_300'].min())
 data['pro_400'] =(data['pro_400']-data['pro_400'].min())/(data['pro_400'].max() - data['pro_400'].min())
 data['pro_500'] =(data['pro_500']-data['pro_500'].min())/(data['pro_500'].max() - data['pro_500'].min())
 data['pro_600'] =(data['pro_600']-data['pro_600'].min())/(data['pro_600'].max() - data['pro_600'].min())
 data['pro_700'] =(data['pro_700']-data['pro_700'].min())/(data['pro_700'].max() - data['pro_700'].min())
 data['pro_800'] =(data['pro_800']-data['pro_800'].min())/(data['pro_800'].max() - data['pro_800'].min())
 data['pro_900'] =(data['pro_900']-data['pro_900'].min())/(data['pro_900'].max() - data['pro_900'].min())
 data['res_fvc'] = data['min_FVC']-data['FVC']
data.loc[:,"Sex"] = pd.factorize(data.Sex)[0]
data.loc[:,"SmokingStatus"] = pd.factorize(data.SmokingStatus)[0]
data['Sex'] =(data['Sex'] - data['Sex'].min())/(data['Sex'].max() - data['Sex'].min())
 data['SmokingStatus'] =(data['SmokingStatus'] - data['SmokingStatus'].min())/(data['SmokingStatus'].max() - data['SmokingStatus'].min())
<define_variables>
skewness = skewness[abs(skewness)> 0.75]
print("Box Cox transform uygulanan özelliklerin sayısı:{}".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
datasets_no_missing[feat] = boxcox1p(datasets_no_missing[feat], lam )
FE = ['percent','SmokingStatus','Sex','age','week','volume','mean','skew','kurthosis','haa','median',
'midmean','midmedian','midskew','midkurthosis','midhaa',
'pro_0','pro_100','pro_200','pro_300','pro_400','pro_500',
'pro_600','pro_700','pro_800','pro_900'] 
<merge>
datasets_no_missing_dummies=pd.get_dummies(datasets_no_missing)
 print(datasets_no_missing_dummies.shape )
tr = data[data.WHERE=='train']
test = data[data.WHERE=='test']
sub = data.loc[data.WHERE=='test']
del data
tr.head()<feature_engineering>
X_yellow=datasets_no_missing_dummies[:ntrain]
nfeature_name=datasets_no_missing_dummies.columns.values[:-1]
rank1D=Rank1D(features=nfeature_name, algorithm="shapiro")
 rank1D.fit(X_yellow[nfeature_name], y_train)
 rank1D.transform(X_yellow[nfeature_name])
 print("önem hesaplandı" )
class MishFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, x):
ctx.save_for_backward(x)
 return x * torch.tanh(F.softplus(x))
@staticmethod
def backward(ctx, grad_output):
x = ctx.saved_variables[0]
sigmoid = torch.sigmoid(x)
 tanh_sp = torch.tanh(F.softplus(x))
return grad_output *(tanh_sp + x * sigmoid *(1 - tanh_sp * tanh_sp))
class Mish(nn.Module):
def forward(self, x):
return MishFunction.apply(x)
 def to_Mish(model):
for child_name, child in model.named_children() :
if isinstance(child, nn.ReLU):
setattr(model, child_name, Mish())
 else:
to_Mish(child )<compute_test_metric>
n=30
n_most_important=df.index.values[:n]
C1, C2 = torch.tensor(70,dtype=torch.float),torch.tensor(1000,dtype=torch.float)
 C1, C2 = C1.to(device),C2.to(device)
 def score_np(fvc_true, y_pred):
sigma = y_pred[:,2] - y_pred[:,0]
fvc_pred = y_pred[:,1]
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 def score(y_true, y_pred):
y_true = y_true.to(torch.float)
 y_pred = y_pred.to(torch.float)
 sigma = y_pred[:,2] - y_pred[:,0]
fvc_pred = y_pred[:,1]
sigma_clip = torch.max(sigma, C1)
 delta = torch.abs(y_true[:,0] - fvc_pred)
 delta = torch.min(delta, C2)
 sq2 = torch.sqrt(torch.tensor(2, dtype=torch.float))
metric =(delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)
 return torch.mean(metric)
 def qloss(y_true, y_pred):
device = y_true.device
qs = [0.25, 0.50, 0.75]
q = torch.tensor(np.array([qs]), dtype=torch.float32)
 q = q.to(device)
 e = y_true - y_pred
v = torch.max(q*e,(q-1)*e)
 return torch.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss

 class GaussianNoise(nn.Module):

 def __init__(self, sigma=0.1, is_relative_detach=True):
super().__init__()
self.sigma = sigma
self.is_relative_detach = is_relative_detach
self.register_buffer('noise', torch.tensor(0))
def forward(self, x):
if self.training and self.sigma != 0:
scale = self.sigma * x.detach() if self.is_relative_detach else self.sigma * x
sampled_noise = self.noise.expand(*x.size() ).float().normal_() * scale
x = x + sampled_noise
return x
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=512):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.5)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.5)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class make_big_model(nn.Module):
def __init__(self, in_ch, out_ch=3):
super(make_big_model, self ).__init__()
self.gasus_nosie = GaussianNoise()
self.fc1 = nn.Sequential(
 nn.Linear(in_ch, 256),
Mish() 
)
 self.fc2 = nn.Sequential(
 nn.Linear(256, 256),
Mish() ,
nn.Dropout(0.5)
)
 self.fc3_p1 = nn.Sequential(
 nn.Linear(256, 128),
Mish() ,
nn.Dropout(0.5)
)
 self.fc3_p2 = nn.Linear(128, out_ch)
 def forward(self, x):
x = self.gasus_nosie(x)
 x = self.fc1(x)
 x = self.fc2(x)
 x = self.fc3_p1(x)
 x = self.fc3_p2(x)
 return x
class make_model(nn.Module):
def __init__(self, in_ch, out_ch=3):
super(make_model, self ).__init__()
self.fc1 = nn.Sequential(
 nn.Linear(in_ch, 160),
Mish() 
)
 self.fc2 = nn.Sequential(
 nn.Linear(160, 128),
Mish() 
)
 self.fc3_p1 = nn.Sequential(
 nn.Linear(128, 64),
Mish() 
)
 self.fc3_p2 = nn.Linear(128, out_ch)
 def forward(self, x):
x = self.fc1(x)
 x = self.fc2(x)
 x = self.fc3_p2(x)
 return x 
<feature_engineering>
scaler = MinMaxScaler()
scaler.fit(datasets_no_missing_dummies[n_most_important])
 datasets_no_missing_dummies_scaled=scaler.transform(datasets_no_missing_dummies[n_most_important] )
NFOLD = 5
kf = KFold(n_splits=NFOLD)
 pd_patient = pd.DataFrame({"Patient":tr["Patient"].unique() })
 for idx,(tr_idx, val_idx)in enumerate(kf.split(pd_patient)) :
pd_patient.loc[val_idx,"fold"] = idx
pd_patient.head()<merge>
preprocessed_train = datasets_no_missing_dummies_scaled[:ntrain]
preprocessed_test = datasets_no_missing_dummies_scaled[ntrain:]
tr = tr.merge(pd_patient, on='Patient', how='left')
 tr.head() 
<define_variables>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(preprocessed_train)
 rmse= np.sqrt(-cross_val_score(model, preprocessed_train, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
batch = 128<prepare_x_and_y>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )
class Data_Generate(Dataset):
def __init__(self,data,label=None):
self.data = data
self.label = label
def __getitem__(self,index):
z_ = self.data[index]
if self.label is not None:
y_ = self.label[index]
y_ = y_[None,]
return z_,y_
else:
return z_
def __len__(self):
return len(self.data )<train_model>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
class EarlyStopping:

 def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt'):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
self.delta = delta
self.path = path
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_loss, model)
 elif score < self.best_score + self.delta:
self.counter += 1
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , self.path)
 self.val_loss_min = val_loss<data_type_conversions>
score = rmsle_cv(model_lgb)
 print("LGMRboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
%%time
cnt = 0
EPOCHS = 1000
criterion = mloss(0.3)
 for fold in range(NFOLD):
list_train_loss,list_val_loss,list_train_score,list_val_score = [],[],[],[]
val_out = []
print(f"FOLD {fold+1}")
 tr_z = tr[FE][tr.fold!=fold].values.astype(np.float32)
 tr_y = tr.res_fvc[tr.fold!=fold].values.astype(np.float32)
 val_z = tr[FE][tr.fold==fold].values.astype(np.float32)
 val_y = tr.res_fvc[tr.fold==fold].values.astype(np.float32)
 base_tr = tr['min_FVC'][tr.fold!=fold].values.astype(np.float32)
 base_val = tr['min_FVC'][tr.fold!=fold].values.astype(np.float32)
 train_db = Data_Generate(tr_z,tr_y)
 train_loader = DataLoader(train_db, batch_size=batch, shuffle=True, num_workers=4)
 val_db = Data_Generate(val_z,val_y)
 val_loader = DataLoader(val_db, batch_size=batch, shuffle=False, num_workers=4)
 tr_num_batch = len(train_loader)
 val_num_batch = len(val_loader)
 net = make_big_model(len(FE),3 ).to(device)
 optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=0.001 ,weight_decay=5e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=20, min_lr=1e-8, verbose=False)
 early_stopping = EarlyStopping(patience=80,path=f'Osic-NN-fold_{fold}.pth',verbose=False)
 for epoch in tqdm(range(EPOCHS)) :
train_loss,train_score,val_loss,val_score = 0,0,0,0
net.train()
for idx, sample in enumerate(train_loader):
feature, label = sample
feature, label = feature.to(device), label.to(device)
 out = net(feature)
 loss = criterion(label, out)
 score_ = score(label ,out)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss += loss.item() /tr_num_batch
train_score += score_.item() /tr_num_batch
list_train_loss.append(train_loss)
 list_train_score.append(train_score)
 net.eval()
for idx, sample in enumerate(val_loader):
feature, label = sample
feature, label = feature.to(device), label.to(device)
 with torch.no_grad() :
out = net(feature)
 loss = criterion(label, out)
 score_ = score(label, out)
 val_loss += loss.item() /val_num_batch
val_score += score_.item() /val_num_batch
list_val_loss.append(val_loss)
 list_val_score.append(val_score)
 early_stopping(val_score, net)
 if early_stopping.early_stop:
print("Early stopping")
 break
scheduler.step(val_loss)
 print(f"train loss: {min(list_train_loss)} train score: {min(list_train_score)}
\
val loss: {min(list_val_loss)} val score: {min(list_val_score)}
\
final lr: {optimizer.param_groups[0]['lr']}" 
)
<compute_test_metric>
score = rmsle_cv(model_xgb)
 print("XGBRboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def score_np(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 scores,sigma,mean_FVC,true_y = [],[],[],[]
tr['pred_FVC']=-1
tr['Confidence']=-1
preds = []
for fold in range(NFOLD):
pred = []
net = make_big_model(in_ch=len(FE)).to(device)
 net.load_state_dict(torch.load(f"Osic-NN-fold_{fold}.pth"))
net.eval()
tr_z = tr[FE][tr.fold==fold].values.astype(np.float32)
 tr_y = tr.FVC[tr.fold==fold].values.astype(np.float32)
 base_val = tr['min_FVC'][tr.fold==fold].values.astype(np.float32)
 valid_db = Data_Generate(tr_z,tr_y)
 valid_loader = DataLoader(valid_db, batch_size=batch, shuffle=False, num_workers=4)
 for idx, sample in enumerate(valid_loader):
feature, label = sample
feature, label = feature.to(device), label.to(device)
 with torch.no_grad() :
out = net(feature)
 out = out.cpu().numpy()
pred.append(out)
 pred = np.concatenate(pred)
 preds.append(pred)
 tr.loc[tr.fold==fold,'pred_FVC'] = base_val - pred[:,1]
tr.loc[tr.fold==fold,'pred_Confidence'] = abs(pred[:,2]-pred[:,0])
 order_tr = tr.groupby(tr['Patient'] ).apply(lambda x:x.sort_values("Weeks",ascending=False))
p_id = pd.unique(tr.Patient)
 for i in range(len(tr)) :
sigma.append(tr.iloc[i].pred_Confidence)
 mean_FVC.append(tr.iloc[i].pred_FVC)
 true_y.append(tr.iloc[i].FVC)
 score_np(np.array(true_y),np.array(mean_FVC),np.array(sigma)) 
<compute_test_metric>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
all_mean,last_mean = [],[]
for i in tr.Patient.unique() :
sigma = tr[tr.Patient==i].pred_Confidence
mean_FVC = tr[tr.Patient==i].pred_FVC
true_y = tr[tr.Patient==i].FVC
all_mean.append(score_np(np.array(true_y),np.array(mean_FVC),np.array(sigma)))
 last_mean.append(score_np(np.array(true_y[-3:]),np.array(mean_FVC[-3:]),np.array(sigma[-3:])))
 print(np.mean(all_mean),np.mean(last_mean))<create_dataframe>
model_xgb.fit(preprocessed_train, y_train)
 xgb_train_pred = model_xgb.predict(preprocessed_train)
 xgb_pred = model_xgb.predict(preprocessed_test)
 print(rmsle(y_train, xgb_train_pred))
<compute_test_metric><EOS> 
print("Gönderi hazırlanıyor")
 submision = pd.DataFrame()
submision['Id'] = test_ids
submision['SalePrice'] = xgb_pred
submision.to_csv('n_most_xgb_submission.csv',index=False)
 print("Gönderi kaydedildi")
 submision.head(20)
 print(xgb_train_pred[:20])
 print(y_train.values[:20])
 print(submision.values[:20] )