Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
import numpy as np
import pandas as pd
import seaborn as sns
import sklearn
from sklearn import metrics
import tensorflow as tf
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split, StratifiedKFold
import numpy as np
encoder=LabelEncoder()
total['Sex']=encoder.fit_transform(total['Sex'])
 total['Embarked']=encoder.fit_transform(total['Embarked'])
 total['Age']=total['Age'].astype(int)
 total.head()<categorify>
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
total['Famile_no']=total['SibSp']+total['Parch']+1
def family_gp(size):
a=''
if(size<=1):
a='Alone'
elif(size<=3):
a='nuclear family'
elif(size<=5):
a='middle'
else:
a='large'
return a
total['Family_grp']=total['Famile_no'].map(family_gp)
<drop_column>
def evaluate(model, x_val, y_val):
y_pred = model.predict(x_val)
 r2 = metrics.r2_score(y_val, y_pred)
 mse = metrics.mean_squared_error(y_val, y_pred)
 mae = metrics.mean_absolute_error(y_val, y_pred)
 msle = metrics.mean_squared_log_error(y_val, y_pred)
 mape = np.mean(tf.keras.metrics.mean_absolute_percentage_error(y_val, y_pred ).numpy())
 rmse = np.sqrt(mse)
 rmlse_score = rmlse(y_val, y_pred ).numpy()
print("R2 Score:", r2)
 print("MSE:", mse)
 print("MAE:", mae)
 print("MSLE:", msle)
 print("MAPE", mape)
 print("RMSE:", rmse)
 print("RMLSE", rmlse_score)
 return {"r2": r2, "mse": mse, "mae": mae, "msle": msle, "mape": mape, "rmse": rmse, "rmlse": rmlse_score}
total.drop(['Name','Ticket','Cabin','SibSp','Parch'],axis=1,inplace=True)
 total.head()<normalization>
def rmlse(y_true, y_pred):
return tf.sqrt(tf.reduce_mean(tf.square(tf.math.log(y_pred + 1)- tf.math.log(y_true + 1))))
total=pd.get_dummies(total,columns=['Embarked','Family_grp','Pclass'])
 c=StandardScaler()
train=total[:len(train)]
train=c.fit_transform(train)
 test=total[len(train):]
test=c.fit_transform(test )<split>
def submit(model, X, ids, file_path):
SalePrice = model.predict(X)
 submission = pd.DataFrame({"Id": ids, "SalePrice": SalePrice.reshape(-1)})
 submission.to_csv(file_path, index=False )
xtrain,xvalid,ytrain,yvalid=train_test_split(train,y,test_size=0.3)
 models = {"KNN": KNeighborsClassifier() ,
"Logistic Regression": LogisticRegression(max_iter=10000),
"Random Forest": RandomForestClassifier() ,
"SVC" : SVC(probability=True),
"DecisionTreeClassifier" : DecisionTreeClassifier() ,
"AdaBoostClassifier" : AdaBoostClassifier() ,
"GradientBoostingClassifier" : GradientBoostingClassifier() ,
"GaussianNB" : GaussianNB() ,
"LinearDiscriminantAnalysis" : LinearDiscriminantAnalysis() ,
"QuadraticDiscriminantAnalysis" : QuadraticDiscriminantAnalysis() }
def fit_and_score(models, X_train, X_valid, y_train, y_valid):
np.random.seed(42)
 model_scores = {}
for name, model in models.items() :
model.fit(X_train, y_train)
 y_pred = model.predict(X_valid)
 model_scores[name] = roc_auc_score(y_pred, y_valid)
 return model_scores
fit_and_score(models,xtrain,xvalid,ytrain,yvalid)
<save_to_csv>
train.corr() ["SalePrice"].sort_values(key = lambda x: abs(x), ascending=False )
leaks = {
897:1,
899:1,
930:1,
932:1,
949:1,
987:1,
995:1,
998:1,
999:1,
1016:1,
1047:1,
1083:1,
1097:1,
1099:1,
1103:1,
1115:1,
}
model=GradientBoostingClassifier()
model.fit(train,target)
 a=model.predict(test)
 submission['Survived']=a
submission['Survived'] = sub['Survived'].apply(lambda x: 1 if x>0.8 else 0)
 submission['Survived'] = sub.apply(lambda r: leaks[int(r['PassengerId'])] if int(r['PassengerId'])in leaks else r['Survived'], axis=1)
 submission.to_csv('sub_tit.csv', index=False )<load_from_csv>
for data in [train, test]:
null_counts = data.isnull().sum()
null_counts[null_counts > 0]
null_columns = list(pd.DataFrame(null_counts[null_counts > 0] ).index)
 for column in null_columns:
if data[column].dtype == object:
data[column] = data[[column]].replace(np.NAN, "Unknown")
 else:
data[column] = data[column].replace(np.NAN, data[column].mean() )
df = pd.read_csv('.. /input/titanic/train.csv' )<count_values>
train_test = pd.get_dummies(pd.concat([train, test]))
df_cat['Cabin'].value_counts()<feature_engineering>
train_features = train_test.iloc[0: len(train)]
test_features = train_test.iloc[len(train):]
_ = train_features.pop("Id")
 _ = test_features.pop("SalePrice")
 test_ids = test_features.pop("Id" )
def splitter(cols):
return re.split('\, |\.',cols)[1]
def remove_digits(col):
for char in col:
if not char.isdigit() :
return col[0]
else:
return char
df_cat['title']= df_cat['Name'].apply(splitter)
 df_cat['deck'] = df_cat['Cabin'].apply(lambda x: str(x ).split(' ')[0] ).apply(remove_digits )<count_values>
train_features, val_features = train_test_split(train_features, test_size=0.2, random_state=np.random.randint(1000))
print(df_cat['title'].value_counts() ,'
',df_cat['deck'].value_counts() )<count_values>
thresold = 0.05
correlated_scores = train_features.corr() ["SalePrice"]
correlated_scores = correlated_scores[correlated_scores.abs() >= thresold]
correlated_columns = list(correlated_scores.index)
 correlated_columns.remove("SalePrice")
 print(correlated_columns )
df_cat['title'] = df_cat['title'].apply(lambda x: 'Other' if x not in ['Mr','Master','Miss','Mrs'] else x)
 print(df_cat['title'].value_counts() )<categorify>
train_targets = train_features.pop("SalePrice")
 val_targets = val_features.pop("SalePrice" )
df_cat['deck'] = df_cat['deck'].map({'A':'ABC','B':'ABC','C':'ABC','D':'DE','E':'DE','F':'FG','G':'FG','n':'U','T':'U'} )<feature_engineering>
categorical_columns = set(train.dtypes[train.dtypes==object].index )
df_num['alone'] =(df['SibSp'] + df['Parch'] ).apply(lambda x: 1 if x==0 else 0 )<concatenate>
scale_strategies = ["none", "standard_scale", "standard_scale_exclude_categorcial_features"]
scale_strategy = scale_strategies[2]
if scale_strategy == scale_strategies[1]:
train_features =(train_features - mean_value)/ std_value
val_features =(val_features - mean_value)/ std_value
test_features =(test_features - mean_value)/ std_value
if scale_strategy == scale_strategies[2]:
for column in train_features.columns:
is_categorical_feature = False
components = column.split("_")
 if len(components)== 2 and components[0] in categorical_columns:
is_categorical_feature = True
if is_categorical_feature == False:
for features in [train_features, val_features, test_features]:
features.loc[:, column] =(features.loc[:, column] - mean_value[column])/ std_value[column]
df_new = pd.concat([df_num,df_cat],axis=1 )<feature_engineering>
use_correlated_columns = True
if use_correlated_columns:
train_features = train_features[correlated_columns]
val_features = val_features[correlated_columns]
test_features = test_features[correlated_columns]
df_new['age_imp'] = df_new[['Age','title']].apply(imp,axis=1 )<filter>
begin = time.time()
parameters = {
"depth": [4, 5, 6, 7, 8, 9],
"learning_rate": [0.01, 0.05, 0.07, 0.08, 0.09, 0.10, 0.11, 0.12, 0.13, 0.14, 0.15],
"iterations": [500, 10000],
}
def train_with_catboost(hyperparameters, X_train, X_val, y_train, y_val):
keys = hyperparameters.keys()
best_index = {key:0 for key in keys}
best_cat = None
best_score = 10e8
for(index, key)in enumerate(keys):
print("Find best parameter for %s" %(key))
items = hyperparameters[key]
best_parameter = None
temp_best = 10e8
for(key_index, item)in enumerate(items):
iterations = hyperparameters["iterations"][best_index["iterations"]] if key != "iterations" else item
learning_rate = hyperparameters["learning_rate"][best_index["learning_rate"]] if key != "learning_rate" else item
depth = hyperparameters["depth"][best_index["depth"]] if key != "depth" else item
print("Training with iterations: %d learning_rate: %.2f depth:%d"%(iterations, learning_rate, depth))
cat = catboost.CatBoostRegressor(
 iterations = iterations,
learning_rate = learning_rate,
depth = depth,
verbose=500 
)
 cat.fit(X_train, y_train, verbose=False)
 result = evaluate(cat, X_val, y_val)
 score = result["rmlse"]
if score < temp_best:
temp_best = score
best_index[key] = key_index
best_parameter = item
if score < best_score:
best_score = score
best_cat = cat
print("Best Parameter for %s: "%(key), best_parameter)
 best_parameters = {
"iterations": hyperparameters["iterations"][best_index["iterations"]],
"learning_rate": hyperparameters["learning_rate"][best_index["learning_rate"]],
"depth": hyperparameters["depth"][best_index["depth"]]
}
return best_cat, best_score, best_parameters
best_cat, best_score, best_parameters = train_with_catboost(parameters, train_features, val_features, train_targets, val_targets)
 print("Best RMLSE: ", best_score)
 print("Best Parameters: ", best_parameters)
 elapsed = time.time() - begin
print("Elapsed time: ", elapsed)
 submit(best_cat, test_features, test_ids, "submission_cat.csv" )
df_new[df_new['Embarked'].isnull() ]<feature_engineering>
X = pd.concat([train_features, val_features])
 y = pd.concat([train_targets, val_targets])
 fold = 1
models = []
for train_indices, valid_indices in KFold(n_splits=5, shuffle=True ).split(X):
print("Training with Fold %d" %(fold))
X_train = X.iloc[train_indices]
X_val = X.iloc[valid_indices]
y_train = y.iloc[train_indices]
y_val = y.iloc[valid_indices]
cat = catboost.CatBoostRegressor(
 iterations = best_parameters["iterations"],
learning_rate = best_parameters["learning_rate"],
depth = best_parameters["depth"] 
)
 cat.fit(X_train, y_train, verbose=False)
 models.append(cat)
 evaluate(cat, X_val, y_val)
 submit(cat, test_features, test_ids, "submission_cat_fold%d.csv"%(fold))
fold += 1
df_new.loc[81,'Embarked'] = 'S'
df_new.loc[829,'Embarked'] = 'S'<categorify>
SalePrice = np.mean([model.predict(test_features)for model in models], axis=0)
 submission = pd.DataFrame({"Id": test_ids, "SalePrice": SalePrice})
 submission.to_csv("submission.csv", index=False )
<concatenate><EOS> 
cat = catboost.CatBoostRegressor(
 iterations = best_parameters["iterations"],
learning_rate = best_parameters["learning_rate"],
depth = best_parameters["depth"] 
)
 cat.fit(X, y, verbose=False)
 evaluate(cat, X, y)
 submit(cat, test_features, test_ids, "submission_cat_all_dataset.csv" )