Kaggle competition: House Prices - Advanced Regression Techniques
base_columns = ['Survived', 'Data']
data_columns = [col for col in df_full.columns if col not in base_columns]
scaler = StandardScaler()
df_full.loc[mask_train, data_columns] = scaler.fit_transform(df_full.loc[mask_train, data_columns])
 df_full.loc[mask_valid, data_columns] = scaler.transform(df_full.loc[mask_valid, data_columns] )<prepare_x_and_y>
os.chdir('/kaggle/input/house-prices-advanced-regression-techniques')
 raw_data = pd.read_csv('train.csv')
 test_data = pd.read_csv('test.csv')
 raw_data.head()
Xt = df_full.loc[mask_train].drop(columns=base_columns)
 Xv = df_full.loc[mask_valid].drop(columns=base_columns )<choose_model_class>
print(f"Amount of columns = {len(list(raw_data)) }
Amount of rows = {len(raw_data)}
So our dataset has dimensions 1460x81" )
models = {}
models['Logistic Regression'] = LogisticRegression(penalty='l2', C=1.0, random_state=0, solver='saga', max_iter=300)
 models['SVC_rbf'] = SVC(probability=True, kernel='rbf', gamma='scale', random_state=0)
 models['SVC_lin'] = SVC(probability=True, kernel='linear', random_state=0)
 models['KNN'] = KNeighborsClassifier(algorithm='auto', leaf_size=20, metric='minkowski', metric_params=None,
n_jobs=1, n_neighbors=10, p=3, weights='uniform')
 dtree_params = {'criterion': 'entropy', 'max_depth': 5, 'max_features': None, 'min_impurity_decrease': 0.0,
'min_samples_leaf': 0.01, 'min_samples_split': 0.01, 'min_weight_fraction_leaf': 0.0, 'splitter': 'best'}
models['Decision Tree'] = DecisionTreeClassifier(**dtree_params)
 models['Random Forest'] = RandomForestClassifier(criterion='entropy', n_estimators=200, oob_score=True)
 xgb_params = {'subsample': 0.5, 'reg_lambda': 5, 'reg_alpha': 0, 'n_estimators': 200, 'min_child_weight': 0, 'max_depth': 6,
'max_delta_step': 1, 'learning_rate': 1.0, 'gamma': 2, 'colsample_bytree': 0.5, 'colsample_bylevel': 0.5}
models['XGBoost'] = XGBClassifier(objective='binary:logistic', **xgb_params )<compute_train_metric>
target = raw_data['SalePrice']
test_Id = test_data['Id']
raw_len, test_len = len(raw_data), len(test_data)
 raw_data = raw_data.drop(['SalePrice'], axis=1)
 raw_data = pd.concat([raw_data, test_data], axis=0)
 raw_data = raw_data.drop(['Id'], axis=1 )
scoring = ['accuracy']
cv = StratifiedShuffleSplit(n_splits=10, test_size=0.3, random_state=0)
 for mname, model in models.items() :
result = cross_validate(model, Xt, yt, scoring=scoring, cv=cv, return_train_score=False)
 print("CV results for model {}: mean {:2.4f}, std {:2.4f}".format(mname, np.mean(result['test_accuracy']), np.std(result['test_accuracy'])) )<define_search_space>
messy_features = []
less_messy_features = []
for feature in list(raw_data):
nan_count = raw_data[feature].isna().sum()
if nan_count:
print(f"{feature} contains, {nan_count} NaN values")
 if nan_count > 0.40 * len(raw_data):
messy_features.append(feature)
 else:
less_messy_features.append(feature )
if model_tuning:
param_grid_XGBoost = {
'colsample_bytree': [0.1, 0.5, 1],
'colsample_bylevel': [0.1, 0.5, 1],
'subsample': [0.1, 0.5, 1],
'learning_rate': [0.05, 0.1, 0.3, 1.0],
'max_depth': [0, 3, 6, 10],
'reg_alpha': [0, 0.1, 1, 5],
'reg_lambda': [0, 0.1, 1, 5],
'gamma': [0, 1, 2, 5],
'n_estimators': [100, 200, 300, 500],
'min_child_weight': [0, 1, 2, 5],
'max_delta_step': [0, 1, 2, 5],
}
param_grid_DTC = {
'criterion': ['gini', 'entropy'],
'splitter': ['best', 'random'],
'max_depth': [None, 3, 5, 7, 10],
'min_samples_split': [2, 0.01, 0.05, 0.1],
'min_samples_leaf': [1, 0.01, 0.05, 0.1],
'min_weight_fraction_leaf': [0.0, 0.1, 0.2],
'max_features': [None, 'auto'],
'min_impurity_decrease': [0.0, 0.2, 0.4, 0.7],
}
tune_model = GridSearchCV(models['Decision Tree'], param_grid=param_grid_DTC, scoring='roc_auc', cv=cv)
 tune_model.fit(Xt, yt)
 print('Best parameters:
', tune_model.best_params_ )<save_to_csv>
print(f"Here are the extremely messy feature we collected:
{messy_features}
Let's remove them for now." )
list_submit = models.keys()
dict_submissions = {}
for mname in list_submit:
model = models[mname]
model.fit(Xt, yt)
 ytp = model.predict(Xt)
 acc = model.score(Xt, yt)
 yvp = model.predict(Xv)
 dict_submissions[mname] = yvp
submission = pd.DataFrame({"PassengerId": df_valid["PassengerId"], "Survived": yvp})
 submission.to_csv('submission_{}.csv'.format(mname), index=False )<set_options>
less_messy_data = raw_data.drop(messy_features, axis=1)
 n_features = len(list(less_messy_data))
print(f"We removed 5 features so our training data now has dimension:
1460x{n_features}" )
warnings.filterwarnings('ignore')
 print('Python: {}'.format(sys.version))
print('numpy: {}'.format(np.__version__))
print('pandas: {}'.format(pd.__version__))
print('scipy: {}'.format(sp.__version__))
print('matplotlib: {}'.format(matplotlib.__version__))
print('sklearn: {}'.format(sklearn.__version__))
print('-'*30)
 print(check_output(["ls", ".. /input"] ).decode("utf8"))<load_from_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 datasets = [train, test]<count_missing_values>
first_elements = [less_messy_data[i].iloc[0] for i in list(less_messy_data)]
categorical_features = []
numerical_features = []
for i in range(len(first_elements)) :
if type(first_elements[i])== str:
categorical_features.append(list(less_messy_data)[i])
 else:
numerical_features.append(list(less_messy_data)[i] )
print('Null training values:
', train.isnull().sum())
 print("-"*30)
 print('Test/Validation columns with null values:
', test.isnull().sum() )<feature_engineering>
for feature in categorical_features:
most_occurring_category = less_messy_data[feature].value_counts().index[0]
less_messy_data[feature] = less_messy_data[feature].fillna(most_occurring_category)
 less_messy_data.head(20 )
for d in datasets:
d['FamilySize'] = d['SibSp'] + d['Parch'] + 1
d['IsAlone'] = 1
d['IsAlone'].loc[d['FamilySize'] > 1] = 0
d['Title'] = d['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 d['Title'] = d['Title'].replace(['Lady', 'Countess','Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Other')
 d['Title'] = d['Title'].replace('Mlle', 'Miss')
 d['Title'] = d['Title'].replace('Ms', 'Miss')
 d['Title'] = d['Title'].replace('Mme', 'Mrs')
 d['FareBand'] = pd.qcut(d['Fare'], 4)
 d['AgeBand'] = pd.cut(d['Age'].astype(int), 5)
 train.info()
test.info()
train.head()<categorify>
for feature in numerical_features:
feature_mean = less_messy_data[feature].mean()
less_messy_data[feature] = less_messy_data[feature].fillna(np.floor(feature_mean))
less_messy_data.head(20 )
le = LabelEncoder()
titanic_train = train.copy(deep = 'True')
 titanic_test = test.copy(deep = 'True')
 titanic_train['Age'] = titanic_train['AgeBand']
titanic_train['Fare'] = titanic_train['FareBand']
titanic_test['Age'] = titanic_test['AgeBand']
titanic_test['Fare'] = titanic_test['FareBand']
column_transformed = ['Sex', 'Embarked', 'Title', 'AgeBand', 'FareBand']
column_transform = ['Sex', 'Embarked', 'Title', 'Age', 'Fare']
for d in datasets:
for i in range(len(column_transform)) :
d[column_transform[i]] = le.fit_transform(d[column_transformed[i]])
 datasets.append(titanic_train)
 datasets.append(titanic_test )<drop_column>
dummies = []
for cat in categorical_features:
dummies.append(pd.get_dummies(less_messy_data[cat]))
less_messy_data = less_messy_data.drop([cat], axis=1)
 n_dummies = len(dummies)
 dummies = [less_messy_data] + dummies
less_messy_data = pd.concat(dummies, axis=1)
 less_messy_data.head(5 )
drop_column = ['Name', 'SibSp', 'Parch', 'FareBand', 'AgeBand']
for d in datasets:
d.drop(drop_column, axis=1, inplace = True)
 train.drop('PassengerId', axis=1, inplace = True)
 titanic_train.drop('PassengerId', axis=1, inplace = True )<sort_values>
columns = list(less_messy_data)
 add_num = 1
for _ in range(10):
for i in range(1, len(columns)) :
if columns[i] in columns[:i]:
columns[i] = columns[i] + str(add_num)
 add_num += 1
less_messy_data.columns = columns
feature_names = ['Pclass', 'Sex', 'Age', 'Fare', 'Embarked', 'FamilySize', 'IsAlone', 'Title']
corr_matrix = train.corr()
print(corr_matrix["Survived"].sort_values(ascending=False))
print("-"*30)
 for feature in feature_names:
print('Correlation between Survived and', feature)
 print(titanic_train[[feature, 'Survived']].groupby([feature], as_index=False ).mean())
 print("-"*30 )<prepare_x_and_y>
remaining_numerical_features = list(less_messy_data)[:(n_features - n_dummies)]
scaler = MinMaxScaler()
scaled = scaler.fit_transform(less_messy_data[remaining_numerical_features])
 less_messy_data[remaining_numerical_features] = scaled
less_messy_data.head(5 )
X_train = train.drop("Survived", axis=1)
 Y_train = train["Survived"]
X_test = test.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape<create_dataframe>
test_data = less_messy_data.iloc[raw_len:, :]
less_messy_data = less_messy_data.iloc[:raw_len, :]
models = [
LinearRegression() ,
LogisticRegressionCV() ,
Perceptron() ,
GaussianNB() ,
KNeighborsClassifier() ,
SVC(probability=True),
DecisionTreeClassifier() ,
AdaBoostClassifier() ,
RandomForestClassifier() ,
XGBClassifier()
]
models_columns = ['Name', 'Parameters','Train Accuracy', 'Validation Accuracy', 'Execution Time']
models_df = pd.DataFrame(columns = models_columns)
 predictions = pd.DataFrame(columns = ['Survived'])
 cv_split = ShuffleSplit(n_splits = 10, test_size =.2, train_size =.8, random_state = 0)
 index = 0
for model in models:
models_df.loc[index, 'Name'] = model.__class__.__name__
models_df.loc[index, 'Parameters'] = str(model.get_params())
 scores = cross_validate(model, X_train, Y_train, cv= cv_split)
 models_df.loc[index, 'Execution Time'] = scores['fit_time'].mean()
models_df.loc[index, 'Train Accuracy'] = scores['train_score'].mean()
models_df.loc[index, 'Validation Accuracy'] = scores['test_score'].mean()
index += 1
models_df.sort_values(by = ['Validation Accuracy'], ascending = False, inplace = True)
 models_df<train_on_grid>
test_parameters = {'n_estimators': [150, 300, 450, 600, 750, 1000],
'learning_rate': [0.01, 0.07, 0.1, 0.25, 0.4, 0.6, 1],
'subsample': [0.3, 0.5, 0.7, 0.9],
'max_depth': [3, 4, 5, 6, 7, 8, 9],
'colsample_bytree': [0.5, 0.7, 0.9],
'min_child_weight': [1, 2, 3, 4]
}
xgb_randsearch = RandomizedSearchCV(XGBRegressor() ,
param_distributions = test_parameters,
n_iter = 100,
scoring='neg_mean_squared_error',
verbose = 3,
n_jobs = -1)
 xgb_randsearch.fit(less_messy_data, target )
<save_to_csv><EOS> 
prediction = xgb_randsearch.predict(test_data)
 prediction = pd.DataFrame(prediction, columns=['SalePrice'])
 submission = np.zeros(( len(prediction), 2))
submission[:, 0] = test_Id.values.flatten()
submission[:, 1] = prediction.values.flatten()
submission = pd.DataFrame(submission, columns=['Id', 'SalePrice'])
 submission.Id = submission.Id.astype(int)
 submission.to_csv('.. /.. /.. /kaggle/working/base_line_submission.csv', index=False )