Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<filter>
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts<correct_missing_values>
%matplotlib inline
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts<load_pretrained>
df_train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 df_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 print("There are %0.0f" %df_train.shape[0] + " houses and %0.0f" %df_train.shape[1] +" variables in the training dataset")
 print("There are %0.0f" %df_test.shape[0] + " houses and %0.0f" %df_test.shape[1] +" explanatory variables in the test dataset")
 df_all = df_train.append(df_test)
 df_all.reset_index(inplace=True, drop=True)
 df_all.head()
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
df_all_1 = df_all.copy()
df_all_1 = df_all.drop(['Id','SalePrice'], axis = 1 )
data = pd.read_pickle('data.pkl' )<drop_column>
df_all_1['MissingCount'] = df_all_1.isnull().sum(axis =1)
 df_all_1['MiscFeature'] = np.where(( df_all.MiscFeature.isnull() == True), 1,0)
 df_all_1['Alley'] = np.where(( df_all.Alley.isnull() == True), 1,0)
 df_all_1['Fence'] = np.where(( df_all.Fence.isnull() == True), 1,0)
 print("There are %0.0f" %df_all_1.shape[1] +" explanatory variables left after removing varibles with more than 50 % missing data" )
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
remainMiss = missingDesc.drop(['MiscFeature','Alley','Fence','SalePrice'], axis =0)
 df_all_1[remainMiss.index].dtypes
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
PoolIndex = df_all_1[df_all_1['PoolArea']!=0].index
df_all_1.loc[np.logical_not(df_all_1.index.isin(PoolIndex)) ,'PoolQC'] = 'NA'
df_all_1.loc[PoolIndex,'PoolQC'].isnull().sum()
df_all_1['PoolQC'] = df_all_1['PoolQC'].fillna(df_all_1.PoolQC[PoolIndex].mode() [0])
 df_all_1.PoolQC.value_counts()
del data
gc.collect() ;<train_model>
FireplaceIndex = df_all_1[df_all_1['Fireplaces']==0].index
df_all_1.loc[df_all_1.index.isin(FireplaceIndex),'FireplaceQu'] = 'NA'
df_all_1.loc[FireplaceIndex,'PoolQC'].isnull().sum()
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts<save_to_csv>
cat_length = 30
print('Imputing non-numeric missing variables...')
 df_all_1 = df_all_1.apply(lambda x: x.fillna(x.mode() [0])if(x.dtype == "object" and len(x.unique())<= cat_length)else x, axis=0 )
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<set_options>
pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )
plt.style.use('ggplot')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<set_options>
for col in remainMiss.index:
if(df_all_1[col].dtypes != 'object' and len(np.unique(df_all_1[col])) < cat_length):
print(df_all_1[col].value_counts() )
def seed_all(seed=42):
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_all()<compute_test_metric>
print('Imputing 3 missing numeric variables as categorical varaibles...')
 for col in ['BsmtHalfBath', 'BsmtFullBath','GarageCars']:
df_all_1[col] = df_all_1[col].fillna(df_all_1[col].mode() [0])
 pd.isnull(df_all_1 ).sum() [pd.isnull(df_all_1 ).sum(axis = 0)>0].sort_values(ascending=False )
def calc_metric_loss(pred_fvc,sigma,true_fvc):
true_fvc=np.reshape(true_fvc,pred_fvc.shape)
 sigma[sigma<70]=70
delta=np.abs(pred_fvc-true_fvc)
 delta[delta>1000]=1000
metric=-(np.sqrt(2)*delta/sigma)-np.log(np.sqrt(2)*sigma)
 return -metric<choose_model_class>
print("Imputing GarageYrBlt...")
 temp = df_all_1[df_all_1.GarageYrBlt.isnull() == True].index
df_all_1["GarageYrBlt"]= np.where(( df_all_1.GarageYrBlt.isnull() == True), df_all_1.YearBuilt,df_all_1.GarageYrBlt ).astype('int64')
 print('Chekcing if GarageYrBlt is corretly imputed...')
(df_all_1.loc[temp,"GarageYrBlt"] == df_all_1.loc[temp,"YearBuilt"] ).unique()
class MLP2(nn.Module):
def __init__(self, input_dim, output_dim,hidden_layers=[1024, 1024], droprate=0.5, activation='relu'):
super(MLP2, self ).__init__()
self.model = nn.Sequential()
self.model.add_module('input', nn.Linear(input_dim, hidden_layers[0]))
if activation == 'relu':
self.model.add_module('relu0', nn.ReLU())
 elif activation == 'tanh':
self.model.add_module('tanh0', nn.Tanh())
 for i in range(len(hidden_layers)-1):
self.model.add_module('dropout'+str(i+1), nn.Dropout(p=droprate))
self.model.add_module('hidden'+str(i+1), nn.Linear(hidden_layers[i], hidden_layers[i+1]))
if activation == 'relu':
self.model.add_module('relu'+str(i+1), nn.ReLU())
 elif activation == 'tanh':
self.model.add_module('tanh'+str(i+1), nn.Tanh())
 self.model.add_module('dropout'+str(i+2), nn.Dropout(p=droprate))
self.model.add_module('final', nn.Linear(hidden_layers[i+1], output_dim))
def forward(self, x):
return self.model(x)
 def predict(self,x, samples):
self.train()
y_samples = [self.forward(x)for _ in range(samples)]
y_samples = torch.cat(y_samples, dim=1)
 std = torch.std(y_samples,dim=1)
 self.eval()
mean = self.forward(x)
 self.train()
return mean, std
def metric_loss(self,pred_fvc,true_fvc,pred_sigma):
true_fvc=torch.reshape(true_fvc,pred_fvc.shape)
 sigma_clipped=torch.clamp(pred_sigma,min=1e-3,max=70)
 delta=torch.clamp(torch.abs(pred_fvc-true_fvc),max=1000)
 metric=torch.div(-torch.sqrt(torch.tensor([2.0] ).to(device)) *delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0] ).to(device)) *sigma_clipped)
 return -metric
def fvc_loss(self,pred_fvc,true_fvc):
true_fvc=torch.reshape(true_fvc,pred_fvc.shape)
 fvc_err=torch.abs(pred_fvc-true_fvc)
 return fvc_err<choose_model_class>
imputeCol = []
for col in df_all_1.columns:
if(df_all_1[col].dtypes != 'object' and len(df_all_1[col].unique())>= cat_length):
imputeCol.append(col)
 print("Columns used for KNN imputation are:
" + str(imputeCol))
imputer = KNNImputer(n_neighbors=5, weights='uniform', metric='nan_euclidean')
 X = df_all_1[imputeCol]
print('Imputing remaining numerical variables with KNN...')
 Xtrans = imputer.fit_transform(X)
 XT = pd.DataFrame(data=Xtrans)
 XT.columns = imputeCol
df_all_1[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']] \
= XT[['LotFrontage','MasVnrArea','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2', 'BsmtFinSF1']]
print('There are now ' + str(df_all_1.isnull().any().sum())+ ' missing values in our data set.' )
class MC_Dropout_Model(nn.Module):
def __init__(self, input_dim, output_dim, drop_prob=0.2):
super(MC_Dropout_Model, self ).__init__()
self.input_dim = input_dim
self.output_dim = output_dim
self.drop_prob = drop_prob
self.fc1 = nn.Linear(input_dim, 100)
 self.drop1=nn.Dropout(p=drop_prob)
 self.fc2 = nn.Linear(100, 100)
 self.drop2=nn.Dropout(p=drop_prob)
 self.fc3 = nn.Linear(100, output_dim)
 def forward(self, x):
out=x
out = F.relu(self.fc1(out))
out= self.drop1(out)
 out = F.relu(self.fc2(out))
out= self.drop2(out)
 out = self.fc3(out)
 return out
def predict(self,x, samples):
self.train()
y_samples = [self.forward(x)for _ in range(samples)]
y_samples = torch.cat(y_samples, dim=1)
 std = torch.std(y_samples,dim=1)
 self.eval()
mean = self.forward(x)
 self.train()
return mean, std
def metric_loss(self,pred_fvc,true_fvc,pred_sigma):
true_fvc=torch.reshape(true_fvc,pred_fvc.shape)
 sigma_clipped=torch.clamp(pred_sigma,min=70)
 delta=torch.clamp(torch.abs(pred_fvc-true_fvc),max=1000)
 metric=torch.div(-torch.sqrt(torch.tensor([2.0] ).to(device)) *delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0] ).to(device)) *sigma_clipped)
 return -metric
def fvc_loss(self,pred_fvc,true_fvc):
true_fvc=torch.reshape(true_fvc,pred_fvc.shape)
 fvc_err=torch.abs(pred_fvc-true_fvc)
 return fvc_err<load_from_csv>
numVal =(( df_all_1.dtypes != 'object' ).index)[df_all_1.dtypes != 'object']
length = []
for col in numVal:
length.append(len(df_all_1[col].unique()))
numVal_df = pd.DataFrame({'Variables':numVal,'UniqueValuesLength':length})
 num_cat_Val = numVal_df[numVal_df['UniqueValuesLength'] < cat_length].sort_values(by='UniqueValuesLength', ascending=False)
 num_cat_Val.head()
train=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 submission=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )<merge>
print('There are %i' %numVal_df[numVal_df['UniqueValuesLength'] < cat_length].shape[0] + ' numerical variables and %i' \
%(df_all_1.dtypes == 'object' ).sum() + ' non-numerical variables can potentially be treated as categorcial variables with some exceptions.' )
train['base_Weeks']=train.groupby(['Patient'])['Weeks'].transform('min')
 base=train[train.Weeks==train.base_Weeks]
base = base.rename(columns={'FVC': 'base_FVC','Percent': 'base_Percent'})
 base.drop_duplicates(subset=['Patient', 'Weeks'], keep='first',inplace=True)
 train=train.merge(base[['Patient','base_FVC','base_Percent']],on='Patient',how='left')
 train['Week_passed'] = train['Weeks'] - train['base_Weeks']<load_from_csv>
cont_val = numVal_df[numVal_df['UniqueValuesLength'] >= cat_length].sort_values(by='UniqueValuesLength')
 cont_val.head()
test = test.rename(columns={'Weeks': 'base_Weeks', 'FVC': 'base_FVC','Percent': 'base_Percent'})
 submission = pd.read_csv(".. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv")
 submission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission['Weeks'] = submission['Patient_Week'].apply(lambda x:x.split('_')[1] ).astype(int)
 test = submission.drop(columns = ["FVC", "Confidence"] ).merge(test, on = 'Patient')
 test['Week_passed'] = test['Weeks'] - test['base_Weeks']
test=test[train.columns.drop(['FVC','Percent'])]<data_type_conversions>
yCor = cor['SalePrice'].sort_values(ascending = False ).reset_index().drop(index=0)
 yCor.columns = ["Variables", "Cor"]
yCor[(yCor["Cor"] > 0.5)|(yCor["Cor"] < -0.5)]
COLS = ['Sex','SmokingStatus']
for col in COLS:
for mod in train[col].unique() :
train[mod] =(train[col] == mod ).astype(int)
 test[mod] =(test[col] == mod ).astype(int)
 train.drop(col,axis=1,inplace=True)
 test.drop(col,axis=1,inplace=True )<normalization>
df_all_red = df_all_1.copy()
robust_scaler = preprocessing.RobustScaler()
train[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])]=robust_scaler.fit_transform(train[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])] )<load_pretrained>
df_all_red.YearRemodAdd = np.where(df_all_1.YearBuilt >df_all_1.YearRemodAdd, df_all_1.YearBuilt, df_all_1.YearRemodAdd)
 df_all_red.YrSold = np.where(df_all_1.YearBuilt >df_all_1.YrSold , df_all_1.YearBuilt, df_all_1.YrSold)
 df_all_red.YrSold = np.where(df_all_red.YearRemodAdd > df_all_red.YrSold, df_all_1.YearRemodAdd, df_all_red.YrSold)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt < df_all_red.YearBuilt, df_all_red.YearBuilt, df_all_red.GarageYrBlt)
 df_all_red.GarageYrBlt = np.where(df_all_red.GarageYrBlt >2010, df_all_red.YearBuilt,df_all_red.GarageYrBlt)
 df_all_red['RestorePeriod'] = df_all_red.YearRemodAdd - df_all_red.YearBuilt
df_all_red['HouseAge'] = df_all_red.YrSold - df_all_red.YearBuilt
df_all_red["RemodAge"] = df_all_red.YrSold - df_all_red.YearRemodAdd
df_all_red['GarageAge'] = df_all_red.YrSold - df_all_red.GarageYrBlt
df_all_red["IsNewHouse"] = np.where(df_all_red.YearBuilt == df_all_red.YrSold, 1 ,0)
 df_all_red["Remodeled"] = np.where(df_all_red.RestorePeriod != 0, 1 ,0)
 df_all_red["GreatDepression"] = np.where(( df_all_1.YearBuilt >=1929)&(df_all_1.YearBuilt <= 1939), 1,0)
 df_all_red["Pre-GreatDepression"] = np.where(( df_all_1.YearBuilt <1929), 1,0)
 df_all_red["MortgageCrisis"] = np.where(( df_all_1.YearBuilt >=2007)&(df_all_1.YearBuilt <= 2010), 1,0)
 df_all_red["MortgageCrisisRestoration"] = np.where(( df_all_1.YearRemodAdd >=2007)&(df_all_1.YearRemodAdd <= 2010), 1,0)
with open('.. /input/file-dictionary/good_files.pickle', 'rb')as handle:
good_file_dict = pickle.load(handle)
 with open('.. /input/file-dictionary/bad_files.pickle', 'rb')as handle:
bad_file_dict = pickle.load(handle )<normalization>
df_all_red.drop(["YearBuilt","GarageYrBlt", "YearRemodAdd"], axis = 1, inplace = True )
class OSIC(Dataset):
def __init__(self,patient_ids,df,file_dict,train=True, transform=None,nims=10):
self.df=df[df.Patient.isin(patient_ids)]
self.train=train
if self.train:
self.fvc=self.df['FVC'].values
else:
self.df[self.df.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])]=robust_scaler.transform(self.df[self.df.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])])
 self.data=self.df[self.df.columns.difference(['FVC','Patient','Percent'])].values
self.patients=self.df['Patient'].values
self.file_dict=file_dict
self.nims=nims
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if self.train:
data = {'fvc': self.fvc[idx],
'data': self.data[idx]}
else:
data = {'data': self.data[idx]}
return data<define_variables>
df_all_red["TotalFlrSF"] = df_all_red["1stFlrSF"] + df_all_red["2ndFlrSF"]
df_all_red["TotalBsmtFin"] = df_all_red['BsmtFinSF1']+df_all_red['BsmtFinSF2']
df_all_red["BsmtFinRatio"] =(df_all_red['TotalBsmtFin'] / df_all_red['TotalBsmtSF'] ).fillna(0)
 df_all_red["IsBasement"] = np.where(df_all_red['TotalBsmtSF'] < 1 , 0, 1)
 df_all_red["PorchArea"] = df_all_red['OpenPorchSF'] + df_all_red['EnclosedPorch'] + df_all_red['ScreenPorch'] + \
df_all_red["3SsnPorch"] + df_all_red['WoodDeckSF']
df_all_red["TotalHouseArea"] = df_all_red['TotalFlrSF'] + df_all_red['TotalBsmtSF']
df_all_red["TotalLivArea"] = df_all_red['GrLivArea'] + df_all_red['TotalBsmtSF']
df_all_red["2ndFloor"] = np.where(( df_all_red['2ndFlrSF'] < 1), 0, 1)
 df_all_red["GarageAreaPerCar"] =(df_all_red.GarageArea / df_all_red.GarageCars ).fillna(0)
 df_all_red["HouseLotRatio"] = df_all_red.TotalHouseArea /df_all_red.LotArea
df_all_red["LivLotRatio"] = df_all_red.GrLivArea / df_all_red.LotArea
df_all_red["GarageLotRatio"] = df_all_red.GarageArea / df_all_red.LotArea
df_all_red["MasVnrRatio"] = df_all_red.MasVnrArea / df_all_red.TotalHouseArea
df_all_red["DifArea"] =(df_all_red.LotArea - df_all_red["1stFlrSF"] - df_all_red.GarageArea - df_all_red.PorchArea)
 df_all_red["LowQualFinSFRatio"] = df_all_red.LowQualFinSF/df_all_red.TotalHouseArea
epochs=200
batch_size=32
num_workers=3<split>
for col in ['TotRmsAbvGrd', 'OverallQual', 'OverallCond','BedroomAbvGr','GarageCars','KitchenAbvGr','FullBath','MissingCount']:
print(col + ' has ' + str(len(df_all_red[col].value_counts())) + ' unique numerical values.' )
ids=train.Patient.unique()
index = np.argwhere(ids=='ID00011637202177653955184')
 ids = list(np.delete(ids, index))
random.shuffle(ids)
 ids=np.array(ids)
 train_ids,val_ids=np.split(ids, [int(round(0.9 * len(ids), 0)) ])
 train_dataset = OSIC(train_ids,train,good_file_dict)
 train_dataloader = DataLoader(train_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers)
 val_dataset = OSIC(val_ids,train,good_file_dict)
 val_dataloader = DataLoader(val_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers )<choose_model_class>
for col in ['OverallQual','OverallCond']:
df_all_red[col].replace(np.linspace(1,10,10 ).astype(int), list(np.linspace(10,100,10)) , inplace=True)
 df_all_red['OverallScore'] = df_all_red[['OverallQual', 'OverallCond']].sum(axis = 1)/2
df_all_red["OverallGrade"] = df_all_red["OverallQual"] * df_all_red["OverallCond"]
model = MC_Dropout_Model(train_dataset.data.shape[1], 1 ).to(device)
 print('Number of parameters:')
 print(sum(p.numel() for p in model.parameters() if p.requires_grad))<choose_model_class>
for col in ['BsmtFinType1','BsmtFinType2']:
df_all_red[col].replace(to_replace=['Unf', 'LwQ','Rec','BLQ','ALQ','GLQ'], value=[1/6,2/6,3/6,4/6,5/6,1], inplace=True)
 for col in ['BsmtCond', 'BsmtQual', 'GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual','HeatingQC','FireplaceQu','PoolQC']:
df_all_red[col].replace(to_replace=['NA','Po','Fa', 'TA','Gd','Ex'], value=[0,1/5,2/5,3/5,4/5,1], inplace=True)
 df_all_red["Functional"].replace(['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],[1/7,2/7,3/7,4/7,5/7,6/7,1] , inplace = True)
 df_all_red["BsmtExposure"].replace(['No','Mn','Av','Gd'],[1/4,2/4,3/4,1] , inplace = True)
 ratingCols= ['BsmtFinType1','BsmtFinType2','BsmtQual','BsmtCond','BsmtExposure','GarageQual','GarageCond','ExterQual','ExterCond','KitchenQual', \
"HeatingQC","Functional",'FireplaceQu','PoolQC']
df_all_red[ratingCols] = df_all_red[ratingCols]*100
df_all_red['BsmtScore']= df_all_red[["BsmtCond", "BsmtQual","BsmtExposure","BsmtFinType1", "BsmtFinType2"]].sum(axis = 1)/5
df_all_red['GarageScore']=df_all_red[["GarageQual", "GarageCond"]].sum(axis = 1)/2
df_all_red['ExterScore'] = df_all_red[["ExterQual", "ExterCond"]].sum(axis = 1)/2
df_all_red['QualScore'] = df_all_red[["OverallQual", "ExterQual", "GarageQual", "BsmtQual","BsmtFinType1", "BsmtFinType2", "KitchenQual","PoolQC", \
"FireplaceQu"]].sum(axis = 1)/9
df_all_red['CondScore'] = df_all_red[["OverallCond", "ExterCond", "GarageCond", "BsmtCond", "HeatingQC", "Functional"]].sum(axis = 1)/6
df_all_red['GarageGrade'] = df_all_red["GarageQual"] * df_all_red["GarageCond"]
df_all_red['ExterGrade'] = df_all_red["ExterQual"] * df_all_red["ExterCond"]
df_all_red["KitchenGrade"] = df_all_red["KitchenAbvGr"] * df_all_red["KitchenQual"]
df_all_red['BsmtGrade'] = df_all_red["BsmtCond"] * df_all_red["BsmtQual"]
df_all_red['FireplaceGrade'] = df_all_red["Fireplaces"] * df_all_red["FireplaceQu"]
df_all_red['PoolGrade'] = df_all_red["PoolArea"] * df_all_red["PoolQC"]
optimizer = optim.Adam(model.parameters())
 epoch_train_metric=[]
epoch_val_metric=[]
epoch_train_loss=[]
epoch_val_loss=[]
epoch=0
min_val_loss = 1e+100
min_val_metric = 1e+100
early_stop = False
patience=10
while epoch<epochs and not early_stop:
epoch+=1
train_loss=0
train_metric=0
model.train()
for batch_idx, data in enumerate(train_dataloader):
optimizer.zero_grad()
fvc,sigma = model.predict(data['data'].float().to(device),100)
 fvc_loss = model.fvc_loss(fvc,data['fvc'].to(device)).mean()
metric_loss = model.metric_loss(fvc,data['fvc'].to(device),sigma ).mean()
loss=fvc_loss
loss.backward()
train_loss += loss.item()
train_metric += metric_loss.item()
optimizer.step()
print('====> Epoch: {} Average train loss FVC: {:.4f}'.format(
 epoch, train_loss / len(train_dataloader)))
 print('====> Epoch: {} Average train metric: {:.4f}'.format(
 epoch, train_metric / len(train_dataloader)))
 val_loss=0
val_metric=0
model.eval()
with torch.no_grad() :
for batch_idx, data in enumerate(val_dataloader):
fvc,sigma = model.predict(data['data'].float().to(device),100)
 fvc_loss = model.fvc_loss(fvc,data['fvc'].to(device)).mean()
metric_loss = model.metric_loss(fvc,data['fvc'].to(device),sigma ).mean()
loss=fvc_loss
val_loss += loss.item()
val_metric += metric_loss.item()
print('====> Epoch: {} Average val loss FVC: {:.4f}'.format(
 epoch, val_loss / len(val_dataloader)))
 print('====> Epoch: {} Average val metric: {:.4f}'.format(
 epoch, val_metric / len(val_dataloader)))
 epoch_train_loss.append(train_loss/ len(train_dataloader))
epoch_val_loss.append(val_loss / len(val_dataloader))
epoch_train_metric.append(train_metric/ len(train_dataloader))
epoch_val_metric.append(val_metric / len(val_dataloader))
if val_metric/ len(val_dataloader)< min_val_metric:
min_val_metric = val_metric/ len(val_dataloader)
 best_model = copy.deepcopy(model.state_dict())
 print('Min metric %0.2f' % min_val_metric)
 epochs_no_improve = 0
else:
epochs_no_improve += 1
if epochs_no_improve == patience:
print('Early stopping!')
 early_stop = True
model.load_state_dict(best_model )<load_from_csv>
for col in num_cat_Val['Variables']:
print(df_all_red[col].value_counts() )
submission=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )<predict_on_test>
df_all_red["MSSubClass_1-Story"] = np.where(df_all_red.MSSubClass.isin([20,30,40,120]), 1, 0)
 df_all_red["MSSubClass_1-1/2Story"] = np.where(df_all_red.MSSubClass.isin([45,50,150]), 1, 0)
 df_all_red["MSSubClass_2-Story"] =np.where(df_all_red.MSSubClass.isin([60,70,160]), 1, 0)
 df_all_red["MSSubClass_2-1/2Story"] = np.where(df_all_red.MSSubClass.isin([75]), 1, 0)
 df_all_red["MSSubClass_Split"] = np.where(df_all_red.MSSubClass.isin([80,85]), 1, 0)
 df_all_red["MSSubClass_AllAges"] = np.where(df_all_red.MSSubClass.isin([40,45,50,75,90,150, 190]), 1, 0)
 df_all_red["MSSubClass_Newer"] = np.where(df_all_red.MSSubClass.isin([20, 60, 120, 160]), 1, 0)
 df_all_red["MSSubClass_Older"] = np.where(df_all_red.MSSubClass.isin([30, 70]), 1, 0)
 df_all_red["MSSubClass_AllStyles"] = np.where(df_all_red.MSSubClass.isin([20, 90, 190]), 1, 0)
 df_all_red["MSSubClass_Pud"] = np.where(df_all_red.MSSubClass.isin([150, 160, 180]), 1, 0)
 df_all_red["MSSubClass_SplitMultiLevel"] = np.where(df_all_red.MSSubClass.isin([80,85, 180]), 1, 0)
test_ids=test.Patient.unique()
test_dataset = OSIC(test_ids,test,good_file_dict,train=False)
 test_dataloader = DataLoader(test_dataset, batch_size=batch_size,shuffle=False,num_workers=num_workers)
 model.eval()
fvc_pred = []
sigma_pred = []
with torch.no_grad() :
for batch_idx, data in enumerate(test_dataloader):
fvc,sigma = model.predict(data['data'].float().to(device),100)
 fvc_pred.append(fvc)
 sigma_pred.append(sigma)
 fvc_pred=torch.cat(fvc_pred, dim=0)
 sigma_pred=torch.cat(sigma_pred, dim=0)
 test['FVC']=fvc_pred.cpu().numpy()
test['Confidence']=sigma_pred.cpu().numpy()<feature_engineering>
df_all_red.drop(["MSSubClass"], axis = 1, inplace = True )
test['Patient_Week']=test["Patient"] + '_' + test['Weeks'].apply(str )<merge>
print("Collapsing categories...")
 df_all_red['LotShape'] = np.where(df_all_1.LotShape != 'Reg', 'IR', df_all_1['LotShape'])
 df_all_red['LandContour'] = np.where(df_all_1.LandContour != 'Lvl', 'IR', df_all_1['LandContour'])
 df_all_red['RoofMatl'] = np.where(df_all_1.RoofMatl != 'CompShg', 'Non-Standard', df_all_1['RoofMatl'])
 df_all_red['Heating'] = np.where(df_all_1.Heating.str.contains("Gas"), 1, 0)
 df_all_red["MSZoning"] = np.where(df_all_red.MSZoning.isin(["RH", "RM", "RL"]), 1, 0)
 df_all_red['Electrical'] = np.where(df_all_1.Electrical != 'SBrkr', 'non-Standard', df_all_1['Electrical'])
 df_all_red['SaleCondition'] = np.where(df_all_1.SaleCondition != 'Normal', 'non-Normal', df_all_1['SaleCondition'])
 df_all_red["SaleType"] = np.where(df_all_1.SaleType.isin(["Con", "ConLD", "ConLI", "ConLw"]), "Contract", df_all_1.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["WD", "CWD"]), "Warranty", df_all_red.SaleType)
 df_all_red["SaleType"] = np.where(df_all_red.SaleType.isin(["COD"]), "Oth", df_all_red.SaleType)
 df_all_red["Foundation"] = np.where(df_all_1["Foundation"] == "Stone", "BrkTil", df_all_1["Foundation"])
 df_all_red["Foundation"] = np.where(df_all_red["Foundation"] == "Wood", "PConc", df_all_red["Foundation"])
 df_all_red["BldgType"] = np.where(df_all_1.BldgType.isin(['1Fam', '2fmCon']), "Family", df_all_1.BldgType)
 df_all_red["BldgType"] = np.where(df_all_red.BldgType.isin(['TwnhsE', 'Twnhs']), "Townhouse", df_all_red.BldgType )
submission=submission[['Patient_Week']].merge(test[['Patient_Week','FVC','Confidence']],on='Patient_Week' )<save_to_csv>
df_all_red["CNormal"] = np.where(( df_all_1.Condition1.isin(["Norm"])) |(df_all_1.Condition2.isin(["Norm"])) , 1, 0)
 df_all_red["CRailroad"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn", "RRNe", "RRAe"])) , 1, 0)
 df_all_red["CPark"] = np.where(( df_all_1.Condition1.isin(["PosN", "PosA"])) |(df_all_1.Condition2.isin(["PosN", "PosA"])) , 1, 0)
 df_all_red["CAdjacent"] = np.where(( df_all_1.Condition1.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) |(df_all_1.Condition2.isin(["PosA", "Artery", "Feedr", "RRAn", "RRAe"])) , 1, 0)
 df_all_red["CWithin"] = np.where(( df_all_1.Condition1.isin(["RRNn", "PosN", "RRNe"])) |(df_all_1.Condition2.isin(["RRNn", "PosN", "RRNe"])) , 1, 0)
 df_all_red["CNorthS"] = np.where(( df_all_1.Condition1.isin(["RRNn", "RRAn"])) |(df_all_1.Condition2.isin(["RRNn", "RRAn"])) , 1, 0)
 df_all_red["CEastW"] = np.where(( df_all_1.Condition1.isin(["RRNe", "RRAe"])) |(df_all_1.Condition2.isin(["RRNe", "RRAe"])) , 1, 0)
 df_all_red["CStreet"] = np.where(( df_all_1.Condition1.isin(["Artery", "Feedr"])) |(df_all_1.Condition2.isin(["Artery", "Feedr"])) , 1, 0)
 df_all_red["HouseStyle_Floor"] = np.where(df_all_1.HouseStyle.isin(["1Story"]), "1", "2")
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["1.5Fin", "1.5Unf"]), "1.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["2.5Fin", "2.5Unf"]), "2.5", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_Floor"] = np.where(df_all_red.HouseStyle.isin(["SLvl", "SFoyer"]), "Split", df_all_red["HouseStyle_Floor"])
 df_all_red["HouseStyle_IsSplit"] = np.where(df_all_red.HouseStyle_Floor == 'Split',1,0)
 df_all_red['HasMasVnrType'] = np.where(df_all_1['MasVnrType'] == "None",0,1 )
submission.to_csv('submission.csv', index=False, float_format='%.1f' )<import_modules>
df_all_red.drop(["Condition1", "Condition2", "YrSold"], axis = 1, inplace = True )
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<define_variables>
df_all_2 = df_all_red.copy()
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42)
 ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128<load_from_csv>
labelencoder = preprocessing.LabelEncoder()
label_cols = [col for col in df_all_red.columns if len(df_all_red[col].value_counts())== 2]
for col in label_cols:
df_all_2[col] = labelencoder.fit_transform(df_all_red[col] )
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
<count_unique_values>
cat_cols = df_all_2.columns[(df_all_2.dtypes== 'object')|(df_all_2.dtypes =='category')]
print("There are " + str(len(cat_cols)) +" categorical variables require one hot encoding.")
 cat_cols
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
print("One hot encoding...")
 original_columns = list(df_all_2.columns)
 df_final= pd.get_dummies(df_all_2, columns=cat_cols, dummy_na=False, drop_first=True)
 new_columns = [c for c in df_final.columns if c not in original_columns]
print("There are " + str(len(new_columns)) + " variables have been created after one hot encoding" )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE'] 
<filter>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
tr.shape, chunk.shape, sub.shape<compute_test_metric>
df_final['SalePrice'] = df_all['SalePrice']
df_final['ID'] = df_all['Id']
print("The final dataset has " + str(df_final.shape[0])+ " observations with " + str(df_final.shape[1]-2)+ " explanatory variables and 1 response variable")
 df_final.head(5 )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="relu", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.005, amsgrad=False), metrics=[score])
 return model
net = make_model()
print(net.summary())
 print(net.count_params())
<prepare_x_and_y>
train = df_final[df_final.ID.isin(df_train.Id)]
test = df_final[df_final.ID.isin(df_test.Id)]
X = train.drop(['SalePrice', "ID"], axis=1)
 y = np.array(train['SalePrice'])
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=79 )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3)) 
<choose_model_class>
k_fold = KFold(n_splits=5, shuffle=True, random_state=79)
 param_grid = {
'feature_fraction': np.arange(0.1,0.3,0.02),
'subsample': np.arange(0.70,0.8,0.01),
'learning_rate': [0.01],
'max_depth': [3,5,7,9],
'num_leaves' : np.arange(30,60,10),
'n_estimators' : [3000,4000,5000,7000],
'max_bin': [30,40,50,60],
"lambda_l1":np.arange(0.0, 1.1, 0.2),
"lambda_l2":np.arange(0.0, 1.1, 0.2),
"min_gain_to_split": np.arange(0, 36, 2),
}
print('Parameter tuning...')
 lgb_est = lgb.LGBMRegressor(random_state = 79, objective = "regression",
metric = "rmse",boosting_type ='gbdt')
 lgb_model = RandomizedSearchCV(lgb_est,param_grid, random_state = 79, n_jobs=-1,
cv = k_fold, scoring = "neg_mean_squared_error")
 lgb_model.fit(X_train, y_train,
eval_set = [(X_train, y_train),(X_test, y_test)],
eval_metric = ['rmse'],
eval_names = ["Train","Validation"],
early_stopping_rounds = 10,
verbose = 100 )
NFOLD = 5
kf = KFold(n_splits=NFOLD )<find_best_model_class>
print(lgb_model.best_params_)
 final_model = lgb_model.best_estimator_
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
i = 0
train_pred = np.zeros(X.shape[0])
 test_pred = []
fold = ['1st', '2nd', '3rd', '4th', '5th']
for train_index, test_index in k_fold.split(X):
print("TRAIN:", train_index[:10], "TEST:", test_index[:10])
 X_train, X_test = X.loc[train_index,:], X.loc[test_index,:]
y_train, y_test = y[train_index], y[test_index]
cv_model = lgb.LGBMRegressor(random_state=79, objective = "regression",
metric = "rmse",**lgb_model.best_params_)
 cv_model.fit(X_train,y_train)
 print("Making prediction for the " + fold[i] + " fold...")
 train_pred[test_index] = cv_model.predict(X_test)
 test_pred.append(cv_model.predict(test.drop(["ID","SalePrice"], axis = 1)))
 i += 1
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<feature_engineering>
cv_score = np.mean(cross_val_score(final_model,X,y,scoring = 'neg_root_mean_squared_error', cv=k_fold,n_jobs = -1,verbose =0))
print('RMSE for training set :', round(np.sqrt(mean_squared_error(y, train_pred)) ,2))
print('Cross Validation RMSE for training set :', round(-cv_score,2))
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10)
 subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<save_to_csv>
final_pred = sum(test_pred)/5
<import_modules><EOS> 
submission = pd.DataFrame({"Id": test.ID, "SalePrice": final_pred})
 submission.to_csv("submission.csv", index = None)
 submission