Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print(train.shape)
 display(train.head())
 train.fillna(value=-999.0,inplace=True)
 test.fillna(value=-999.0,inplace=True)
 print('Type numeric:')
 list_feature_Nan = []
for i in train.select_dtypes(exclude=['object'] ).columns:
if(train[i] == -999.0 ).astype(int ).sum() > 0:
print("Feature: ", i, "has", round(((train[i] == -999.0 ).astype(int ).sum() /1460)*100), "% of NaN")
 list_feature_Nan.append(i)
 print('Type object:')
 for i in train.select_dtypes(include=['object'] ).columns:
if(train[i] == -999.0 ).astype(int ).sum() > 0:
print("Feature: ", i, "has", round(((train[i] == -999.0 ).astype(int ).sum() /1460)*100), "% of NaN")
test = pd.read_csv(f'{ROOT}/test.csv')
 for i in test.Patient:
sub.loc[sub.Patient==i,'week'] = sub[sub.Patient==i].Weeks.values-test[test.Patient==i].Weeks.values
sub<categorify>
for i in list_feature_Nan:
train[i].replace(-999.0,train[i].mean() ,inplace=True)
 test[i].replace(-999.0,train[i].mean() ,inplace=True)
 y = np.array(pd.DataFrame(np.log(train.SalePrice)))
 y2 = np.array(pd.DataFrame(np.log(train.SalePrice)))
 mean_saleprice = pd.DataFrame(train.groupby(['GrLivArea'])['LotArea'].mean())
 mean_saleprice.columns = ['LotArea_bis']
train.drop('SalePrice',axis=1,inplace=True)
 all_data = all_data = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'],test.loc[:,'MSSubClass':'SaleCondition']))
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
print(skewed_feats)
 train[skewed_feats] = np.log1p(train[skewed_feats])
 test[skewed_feats] = np.log1p(test[skewed_feats])
 train[skewed_feats] = train[skewed_feats].fillna(all_data[skewed_feats].mean())
 test[skewed_feats] = test[skewed_feats].fillna(all_data[skewed_feats].mean() )
ze =(sub[FE].values ).astype(np.float32)
 base_eval =(sub['min_FVC'].values ).astype(np.float32)
 pe = np.zeros(( ze.shape[0], 3))
test_db = Data_Generate(ze)
 test_loader = DataLoader(test_db, batch_size=batch, shuffle=False, num_workers=4 )<categorify>
le = preprocessing.LabelEncoder()
train_str = train.select_dtypes(include=['object'])
 test_str = test.select_dtypes(include=['object'])
 display(train_str.head())
 print(train_str.columns.values)
 train.drop(train_str.columns.values,axis=1,inplace=True)
 test.drop(train_str.columns.values,axis=1,inplace=True )
print("predict test...")
 pe = 0
for k in range(NFOLD):
if k == 2:
continue
pred = []
base_fvc = sub.FVC.values.astype(np.float32)
 net = make_big_model(len(FE)).to(device)
 net.load_state_dict(torch.load(f"Osic-NN-fold_{k}.pth"))
net.eval()
for idx, sample in enumerate(test_loader):
data = sample
data = data.to(device)
 with torch.no_grad() :
out = net(data)
 out = out.cpu().numpy()
pred.append(out)
 pred = base_fvc[:,None] - np.concatenate(pred)
 pe += pred / 4<feature_engineering>
train_str_dum = pd.get_dummies(train_str)
 test_str_dum = pd.get_dummies(test_str)
 columns_dum = list(set(train_str_dum)& set(test_str_dum))
train_str_dum = train_str_dum[columns_dum]
test_str_dum = test_str_dum[columns_dum]
train_flo = train.select_dtypes(exclude=['object'])
 test_flo = test.select_dtypes(exclude=['object'])
 new_train = pd.merge(train_flo,train_str_dum,left_index=True,right_index=True)
 new_test = pd.merge(test_flo,test_str_dum,left_index=True,right_index=True)
 display(new_train.head())
 print(new_train.columns.values )
sub['FVC1'] = pe[:, 1]
sub['Confidence1'] = abs(pe[:, 2] - pe[:, 0] )<save_to_csv>
train_clf = new_train.copy()
train_clf.drop('Id',axis=1,inplace=True)
 index = pd.DataFrame(test.Id,columns = ['Id'])
 test_clf = new_test.drop('Id',axis=1)
 train_clf2 = train_clf.drop(['LotFrontage','MasVnrArea','GarageYrBlt'],axis=1)
 train_clf2 = pd.merge(train_clf2,train[['LotFrontage','MasVnrArea','GarageYrBlt']],left_index=True,right_index=True)
 train_clf2['tot_sf'] = train_clf2['TotalBsmtSF'] + train_clf2['GrLivArea']
train_clf2['ratio_fl'] = train_clf2['2ndFlrSF'] / train_clf2['1stFlrSF']
train_clf2['garage_ex'] =(train_clf2['GarageQual_Gd'] + train_clf2['GarageQual_TA'] + train_clf2['GarageQual_Fa'] + train_clf2['GarageQual_Po'])*(train_clf2['GarageCond_Ex'])
 clus = KernelPCA(n_components = 25)
 train_clf2_pca = clus.fit_transform(train_clf2)
 train_clf3 = pd.merge(train_clf2,pd.DataFrame(train_clf2_pca),left_index=True,right_index=True)
 test_clf2 = test_clf.drop(['LotFrontage','MasVnrArea','GarageYrBlt'],axis=1)
 test_clf2 = pd.merge(test_clf2,test[['LotFrontage','MasVnrArea','GarageYrBlt']],left_index=True,right_index=True)
 test_clf2['tot_sf'] = test_clf2['TotalBsmtSF'] + test_clf2['GrLivArea']
test_clf2['ratio_fl'] = test_clf2['2ndFlrSF'] / test_clf2['1stFlrSF']
test_clf2['garage_ex'] =(test_clf2['GarageQual_Gd'] + test_clf2['GarageQual_TA'] + test_clf2['GarageQual_Fa'] + test_clf2['GarageQual_Po'])*(test_clf2['GarageCond_Ex'])
 test_clf3 = pd.merge(test_clf2,pd.DataFrame(clus.transform(test_clf2)) ,left_index=True,right_index=True)
 index = pd.DataFrame(test.Id,columns = ['Id'])
 def rmse(predictions, targets):
return np.sqrt(((predictions - targets)** 2 ).mean())
 def score(clf, train_np, random_state, folds, target, length):
kf = KFold(n = length , n_folds=folds, shuffle = True, random_state = random_state)
 for itrain, itest in kf:
Xtr, Xte = train_np[itrain], train_np[itest]
ytr, yte = target[itrain], target[itest]
clf.fit(Xtr, ytr.ravel())
 pred = pd.DataFrame(clf.predict(Xte))
return rmse(yte, pred)
 return rmse(y, pred )
subm = sub[['Patient_Week','FVC1','Confidence1']].copy()
subm.rename(columns={'FVC1':'FVC','Confidence1':'Confidence'},inplace=True)
 subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<import_modules>
train_clf4 = train_clf3.copy()
test_clf4 = test_clf3.copy()
from sklearn.model_selection import KFold
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor
from sklearn.multioutput import MultiOutputRegressor
import joblib<load_from_csv>
for i in train_clf4.select_dtypes(include=['float64'] ).columns:
if train_clf4[i].min() > test_clf4[i].min() :
train_clf4[i][train_clf4[i]==train_clf4[i].min() ] = test_clf4[i].min()
for i in train_clf4.select_dtypes(include=['float64'] ).columns:
if train_clf4[i].min() < test_clf4[i].min() :
test_clf4[i][test_clf4[i]==test_clf4[i].min() ] = train_clf4[i].min()
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 feature_ct = pd.read_csv('.. /input/2020osic/CT_21feature.csv')
 tr = tr.merge(feature_ct, on='Patient')
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','FVC']] 
.rename({'FVC': 'min_FVC'}, axis=1)
.groupby('Patient')
.first() 
.reset_index() 
)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
base =(
 data 
.loc[data.Weeks == data.min_week][['Patient','Percent']] 
.rename({'Percent': 'min_Percent'}, axis=1)
.groupby('Patient')
.first() 
.reset_index() 
)
 data = data.merge(base, on='Patient', how='left')
 del base
data.loc[:,"Sex"] = pd.factorize(data.Sex)[0]
data.loc[:,"SmokingStatus"] = pd.factorize(data.SmokingStatus)[0]
train_df = data[data.WHERE=='train']
train_df.head()<normalization>
for i in train_clf4.select_dtypes(include=['float64'] ).columns:
if train_clf4[i].max() > test_clf4[i].max() :
test_clf4[i][test_clf4[i]==test_clf4[i].max() ] = train_clf4[i].max()
for i in train_clf4.select_dtypes(include=['float64'] ).columns:
if train_clf4[i].max() < test_clf4[i].max() :
train_clf4[i][train_clf4[i]==train_clf4[i].max() ] = test_clf4[i].max()
train_reg = pd.DataFrame(columns=['Patient', 'q2_Slope', 'q5_Slope','q8_Slope','slp'])
 def model_slope(x):
b = x[0][0]
slope_1 = [(t[0]-b)/t[1] for t in x[1:]]
sort_slope = sorted(slope_1,reverse=True)
 length = len(slope_1)
 return np.mean(slope_1),np.mean(sort_slope[:length//2]),np.mean(sort_slope[length//2:])
 for idx,i in enumerate(pd.unique(train_df['Patient'])) :
FVC_Weeks = train_df[train_df.Patient==i][['FVC','base_week']].values
slope_m,slope_f,slope_l = model_slope(FVC_Weeks)
 sub = train_df.loc[train_df.Patient == i, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
train_reg.loc[idx] = [i,slope_m,slope_f,slope_l,a]
train_reg.head()<merge>
def put_corr_feat(data,test,target):
data_new = data.copy()
test_new = test.copy()
for i in data.columns:
corr1=stats.pearsonr(np.array(pd.DataFrame(data[i])) ,target)[0]
corr2=stats.pearsonr(np.array(pd.DataFrame(data[i]**3)) ,target)[0]
if abs(corr2)>abs(corr1)*1.2:
print(i)
 data_new[i]=np.array(data_new[i])**3
test_new[i]=np.array(test_new[i])**3
return data_new, test_new
data1, test1 = put_corr_feat(train_clf3,test_clf3,y )
final_df = pd.DataFrame(columns=train_df.columns)
 for idx,i in enumerate(pd.unique(train_df['Patient'])) :
temp_df = train_df[train_df.Patient==i]
final_df = final_df.append(temp_df[temp_df.Weeks==temp_df.min_week].iloc[0,:])
 train_reg = train_reg.merge(final_df)
 train_reg = train_reg.drop(['min_week','base_week'],axis=1)
 train_reg.head()<feature_engineering>
def frange(start, stop, step):
i = start
a = []
while i < stop:
yield i
a.append(i)
 i += step
return a
Cs1 = [5000,6000,7000,8000,9000,10000]
res = []
res1 = []
res2 = []
res3 = []
res4 = []
Cs3=[]
train_np = np.array(train_clf3)
 robust_scaler = RobustScaler()
train_reg.loc[:,"Sex"] = pd.factorize(train_reg.Sex)[0]
train_reg.loc[:,"SmokingStatus"] = pd.factorize(train_reg.SmokingStatus)[0]
train_reg['SmokingStatus'] =(train_reg['SmokingStatus'] - train_reg['SmokingStatus'].min())/(train_reg['SmokingStatus'].max() - train_reg['SmokingStatus'].min())
 train_reg.head()<feature_engineering>
index = pd.DataFrame(test.Id,columns = ['Id'] )
NFOLD = 5
kf = KFold(n_splits=NFOLD)
 train_reg['fold']=-1
for idx,(tr_idx, val_idx)in enumerate(kf.split(train_reg)) :
train_reg.loc[val_idx,"fold"] = idx
train_reg.head()<merge>
np.random.seed(123)
 ntrain = train_clf4.shape[0]
sparse_data = []
tr_te = pd.concat(( train_clf4, test_clf4), axis = 0)
 tmp = csr_matrix(tr_te)
 sparse_data.append(tmp)
 xtr_te = hstack(sparse_data, format = 'csr')
 xtrain = xtr_te[:ntrain, :]
xtest = xtr_te[ntrain:, :]
def batch_generator(X, y, batch_size, shuffle):
number_of_batches = np.ceil(X.shape[0]/batch_size)
 counter = 0
sample_index = np.arange(X.shape[0])
 if shuffle:
np.random.shuffle(sample_index)
 while True:
batch_index = sample_index[batch_size*counter:batch_size*(counter+1)]
X_batch = X[batch_index,:].toarray()
y_batch = y[batch_index]
counter += 1
yield X_batch, y_batch
if(counter == number_of_batches):
if shuffle:
np.random.shuffle(sample_index)
 counter = 0
def batch_generatorp(X, batch_size, shuffle):
number_of_batches = X.shape[0] / np.ceil(X.shape[0]/batch_size)
 counter = 0
sample_index = np.arange(X.shape[0])
 while True:
batch_index = sample_index[batch_size * counter:batch_size *(counter + 1)]
X_batch = X[batch_index, :].toarray()
counter += 1
yield X_batch
if(counter == number_of_batches):
counter = 0
def nn_model() :
model = Sequential()
model.add(Dense(10, input_dim = xtrain.shape[1], init = 'he_normal'))
model.add(PReLU())
 model.add(Dropout(0.4))
model.add(Dense(5, init = 'he_normal'))
model.add(PReLU())
 model.add(Dropout(0.2))
model.add(Dense(1, init = 'he_normal'))
model.compile(loss = 'mse', optimizer = 'adadelta')
 return(model)
 nfolds = 2
folds = KFold(len(y), n_folds = nfolds, shuffle = True, random_state = 111)
 i = 0
nbags = 2
nepochs = 1
pred_oob = np.zeros(xtrain.shape[0])
 pred_test = np.zeros(xtest.shape[0])
 def model() :
for(inTr, inTe)in folds:
xtr = xtrain[inTr]
ytr = y[inTr]
xte = xtrain[inTe]
yte = y[inTe]
pred = np.zeros(xte.shape[0])
 for j in range(nbags):
model = nn_model()
fit = model.fit_generator(generator = batch_generator(xtr, ytr, 64, True),
nb_epoch = nepochs,
samples_per_epoch = xtr.shape[0],
verbose = 0)
 pred += model.predict_generator(generator = batch_generatorp(xte, 50, False), val_samples = xte.shape[0])[:,0]
score = rmse(np.exp(yte), np.exp(pred))
print(score)
 pred_test += model.predict_generator(generator = batch_generatorp(xtest, 25, False), val_samples = xtest.shape[0])[:,0]
pred /= nbags
pred_oob[inTe] = pred
i += 1
train_reg = train_reg.merge(feature_ct)
 train_reg_bp = train_reg.copy()
train_reg.head()<feature_engineering>
clf = xgb.XGBRegressor(n_estimators = 6000, seed = 0, learning_rate = 0.01, max_depth = 3, subsample = 0.8, colsample_bytree = 0.8, colsample_bylevel = 0.8)
 clf2 = Ridge(alpha=21)
 clf3 = LassoCV(alphas = [1, 0.1, 0.001, 0.0005])
 Select = SelectKBest(f_regression, k = 268)
 train_fin = np.array(Select.fit_transform(train_clf4,y))
test_fin = np.array(Select.transform(test_clf4))
train_fin_ridge = np.array(robust_scaler.fit_transform(train_clf3))
test_fin_ridge = np.array(robust_scaler.transform(test_clf3))
kf1 = KFold(n = 1450 , n_folds=7, random_state=0, shuffle = True)
 i = 0
res1 = []
res4 = []
for itrain, itest in kf1:
i = i + 1
Xtr, Xte = train_fin[itrain], train_fin[itest]
ytr, yte = y[itrain], y[itest]
clf.fit(Xtr, ytr.ravel())
 if i == 1:
pred1 = pd.DataFrame(clf.predict(test_fin))
print("Fold 1 :", rmse(yte, pd.DataFrame(clf.predict(Xte))))
res1.append(rmse(yte, pd.DataFrame(clf.predict(Xte))))
if i > 1 :
pred1 = pred1 + pd.DataFrame(clf.predict(test_fin))
print("Fold ",i, " :", rmse(yte, pd.DataFrame(clf.predict(Xte))))
res1.append(rmse(yte, pd.DataFrame(clf.predict(Xte))))
clf2.fit(train_fin_ridge, y)
 pred2 = pd.DataFrame(clf2.predict(test_fin_ridge))
clf3.fit(train_fin_ridge, y)
 pred3 = pd.DataFrame(clf3.predict(test_fin_ridge))
pred_1 = pred1/7
for i in pred2[np.exp(pred2 ).values<1].index.values:
pred2[pred2.index == i] = pred_1[pred_1.index == i]
for i in pred2[pred2.values>12].index.values:
pred2[pred2.index == i] = pred_1[pred_1.index == i]
print(np.mean(res1))
pred_1 =(pred_1+2*pred2+pred3)/4
pred_2 =(pred_1+pred3)/2
pred_xg = pred_1
pred_1.columns = ['SalePrice']
pred_2.columns = ['SalePrice']
pred_xg.columns = ['SalePrice']
pred3.columns = ['SalePrice']
pred_final_1 = pd.DataFrame(np.exp(pred_1), index = new_test.index, columns = ['SalePrice'])
 pred_final_2 = pd.DataFrame(np.exp(pred_2), index = new_test.index, columns = ['SalePrice'])
 pred_Lasso = pd.DataFrame(np.exp(pred3), index = new_test.index, columns = ['SalePrice'])
 pred_xg = pd.DataFrame(np.exp(pred_xg), index = new_test.index, columns = ['SalePrice'])
 for i in pred_final_1[pred_final_1.values<100].index.values:
pred_final_1[pred_final_1.index == i] = 180921.1959
pred_final = pred_final_1
pred_submit = pd.merge(index,pred_final,left_index=True,right_index=True)
 pred_Lasso_submit = pd.merge(index,pred_Lasso,left_index=True,right_index=True)
 pred_xg_submit = pd.merge(index,pred_xg,left_index=True,right_index=True)
 pred_xg_lasso_submit = pd.merge(index,pred_final_2,left_index=True,right_index=True)
 pred_submit.to_csv('XG_Ridge_Lasso.csv',index=False)
 pred_Lasso_submit.to_csv('Lasso.csv',index=False)
 pred_xg_submit.to_csv('XGB.csv',index=False)
 pred_xg_lasso_submit.to_csv('XGB_Lasso.csv',index=False )