Kaggle competition: House Prices - Advanced Regression Techniques
train_data = pd.DataFrame(ohe_set[:891])
 test_data = pd.DataFrame(ohe_set[891:])
 X = train_data
y = train_df["Survived"]<split>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print("Train set size:", train.shape)
 print("Test set size:", test.shape)
 print('START data processing', datetime.now() , )
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.1, random_state = 42 )<train_model>
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True )
xgb_classifier = xgboost.XGBRFClassifier(n_estimators=30,
learning_rate=1,
reg_lambda=0.15,
gamma=0.06,
max_depth=20)
 xgb_classifier.fit(X_train, y_train)
 y_test_pred = xgb_classifier.predict(X_test ).tolist()
accuracy_score(y_test_pred,y_test )<compute_test_metric>
train = train[train.GrLivArea < 4500]
train.reset_index(drop=True, inplace=True)
actual = y_test
predicted = y_test_pred
matrix = confusion_matrix(actual,predicted, labels=[1,0])
 print('Confusion matrix :
',matrix)
 tp, fn, fp, tn = confusion_matrix(actual,predicted,labels=[1,0] ).reshape(-1)
 print('Outcome values :
', tp, fn, fp, tn)
 matrix = classification_report(actual,predicted,labels=[1,0])
 print('Classification report :
',matrix )<predict_on_test>
train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
y_pred = xgb_classifier.predict(test_data ).tolist()<save_to_csv>
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 print(features.shape )
predictions = pd.DataFrame(y_pred)
 datasets = pd.concat([test_set["PassengerId"],predictions],axis=1)
 datasets.columns = ["PassengerId","Survived"]
datasets.to_csv("gender_submission.csv",index=False )<import_modules>
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features["PoolQC"] = features["PoolQC"].fillna("None" )
import seaborn as sns
import matplotlib.pyplot as plt
import time<load_from_csv>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 train_data.head()<load_from_csv>
skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data.head()
test_data.isna().sum()<feature_engineering>
features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
test_data["Fare"] = test_data["Fare"].fillna(0)
 test_data["Fare"].isna().sum()<count_values>
print(features.shape)
 final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape)
women = train_data.loc[train_data.Sex == "female"]["Survived"]
rate_women = sum(women)/ len(women)
 print("Rate of Women :",rate_women )<count_values>
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers])
men = train_data.loc[train_data.Sex == "male"]["Survived"]
rate_men = sum(men)/ len(men)
 print("Rate of Men :",rate_men )<save_to_csv>
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = X.drop(overfit, axis=1 ).copy()
X_sub = X_sub.drop(overfit, axis=1 ).copy()
print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape )
y = train_data["Survived"]
features = ["Sex","Pclass","SibSp","Parch","Fare"]
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 model = RandomForestClassifier(n_estimators = 20,max_depth = 5 ,random_state=42)
 model.fit(X,y)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
<save_to_csv>
print('START ML', datetime.now() ,)
 kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y,
scoring="neg_mean_squared_error",
cv=kfolds))
return(rmse)
model2 = LogisticRegression(penalty= 'none' ,random_state=42 ,max_iter=150 ).fit(X, y)
 predictions2 = model2.predict(X_test)
 output2 = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions2})
 output2.to_csv('my_submission2.csv', index=False)
<train_model>
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas=alphas_alt, cv=kfolds,))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2,
random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, random_state=42, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() ,
SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =42)
 lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01, n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006, random_state=42)
 stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True)
ss = StandardScaler()
def model(x,y):
x, testx, y, testy = train_test_split(x, y, test_size=0.1, random_state=4355)
 all_models = ['sgd','svc','knn','rf','dtc', 'lr', 'LDA', 'xgboost']
models = [sgd,SVC, knn,rf,dtc,LogisticRegression, LDA, xgboost ]
for i,j in zip(models,all_models):
try:
model = i(random_state=344,n_components=5)
 except:
try:
model = i(random_state=344)
 except:
model = i()
model.fit(x,y)
 trsc = model.score(x,y)
 sc = model.score(testx,testy)
 print(f'For {j} we have train score of {trsc} and test score of {sc}')
 def modelR(x,y):
x, testx, y, testy = train_test_split(x, y, test_size=0.1, random_state=4355)
 all_models = ['svr','rfr','dtr', 'lreg', 'xgboostR']
models = [SVR,rfr,dtr,LinearRegression, xgboostR ]
for i,j in zip(models,all_models):
try:
model = i(random_state=344,n_components=5)
 except:
try:
model = i(random_state=344)
 except:
model = i()
model.fit(x,y)
 trsc = model.score(x,y)
 sc = model.score(testx,testy)
 print(f'For {j} we have train score of {trsc} and test score of {sc}')
 def scaler(df):
df = df.apply(lambda x:(x-x.mean())/x.std())
 return df
def pca(xd,n):
pca = PCA(n_components=n)
 xd = pca.fit(xd ).transform(xd)
 print(pca.explained_variance_ratio_)
 return xd
def setfig(w=10,h=7):
return plt.figure(figsize=[w,h])
 def predS(model, test_data):
pred = model.predict(test_data)
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 td = {'PassengerId':test_data.PassengerId, 'Survived':pred}
pd.DataFrame(td ).to_csv('Submission.csv',index=False )<import_modules>
print('TEST score on CV')
 score = cv_rmse(ridge)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
print('START Fit')
 print(datetime.now() , 'StackingCVRegressor')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print(datetime.now() , 'elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print(datetime.now() , 'lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print(datetime.now() , 'ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print(datetime.now() , 'svr')
 svr_model_full_data = svr.fit(X, y)
 print(datetime.now() , 'GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print(datetime.now() , 'xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print(datetime.now() , 'lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y)
odata = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 odata.head(3 )<count_missing_values>
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.1 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.25 * stack_gen_model.predict(np.array(X))))
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X)))
 print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)) )
odata.isna().sum()<count_missing_values>
submission.to_csv("House_price_submission_v54.csv", index=False)
 print('Save submission', datetime.now() , )