Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<prepare_output>
warnings.filterwarnings('ignore' )
result = pd.DataFrame(train_Y[:5000], columns=['Y_dev'])
 result['Y_pred'] = pred_dev_labels
result['correct'] = result['Y_dev'] - result['Y_pred']
errors = result[result['correct'] != 0]
error_list = errors.index
print('Number of errors is ', len(errors))
print('The indices are ', error_list )<predict_on_test>
sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train.head()
predictions = model.predict(test_X_digit)
 print(predictions.shape )<prepare_output>
train = train.drop(train[(train['GarageCars']>3)&(train['SalePrice']<300000)].index ).reset_index(drop=True )
predicted_labels = np.argmax(predictions, axis=1 )<save_to_csv>
y_train = train.SalePrice
join_data = pd.concat(( train, test), sort=False ).reset_index(drop=True)
 train_size = len(train)
 test_size = len(test)
 print("Train set size: ", train_size)
 print("Test set size: ", test_size)
 print("Train+test set size: ", len(join_data))
result = pd.read_csv('.. /input/sample_submission.csv')
 result['Label'] = predicted_labels
result.to_csv('submission.csv', index=False )<set_options>
missings_count = {col:join_data[col].isnull().sum() for col in join_data.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))
%matplotlib inline 
<load_from_csv>
def fill_missings(data):
clean_data = data.copy()
fill_with_nones = ['PoolQC','MiscFeature','Alley','Fence','FireplaceQu','GarageType','GarageFinish',
'GarageQual','GarageCond','BsmtExposure','BsmtQual','BsmtCond', 'BsmtFinType1', 'BsmtFinType2',
'MasVnrType']
for col in fill_with_nones:
clean_data[col] = clean_data[col].fillna("None")
 fill_with_0 = ['MasVnrArea','BsmtFullBath','BsmtHalfBath','BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','GarageArea','GarageCars','TotalBsmtSF','GarageYrBlt']
for col in fill_with_0:
clean_data[col] = clean_data[col].fillna(0)
 clean_data["LotFrontage"] = clean_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
clean_data['MSZoning'].fillna(clean_data['MSZoning'].mode() [0])
 clean_data['Electrical'].fillna(clean_data['Electrical'].mode() [0])
 clean_data['Exterior1st'].fillna(clean_data['Exterior1st'].mode() [0])
 clean_data['Exterior2nd'].fillna(clean_data['Exterior2nd'].mode() [0])
 clean_data['KitchenQual'].fillna(clean_data['KitchenQual'].mode() [0])
 clean_data['SaleType'].fillna(clean_data['SaleType'].mode() [0])
 clean_data['Utilities'].fillna(clean_data['Utilities'].mode() [0])
 clean_data['Functional'].fillna('Typ')
 return clean_data
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv" )<count_missing_values>
def create_additional_features(all_data):
all_data['has_pool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has_2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has_garage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has_fireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has_bsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['Year_BuiltAndRemod']=all_data['YearBuilt']+all_data['YearRemodAdd']
all_data['Total_SF']=all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['Total_sqr_footage'] =(all_data['BsmtFinSF1'] + all_data['BsmtFinSF2'] + all_data['1stFlrSF'] + all_data['2ndFlrSF'])
 all_data['Total_Bath'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+ all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['Total_porch_SF'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] + all_data['EnclosedPorch'] + all_data['ScreenPorch'] + all_data['WoodDeckSF'])
 return all_data
print(train.isnull().sum() )<count_missing_values>
def categorize_data(data):
categorized_data = data.copy()
categorized_data['MSSubClass'] = categorized_data['MSSubClass'].apply(str)
 categorized_data['OverallCond'] = categorized_data['OverallCond'].astype(str)
 categorized_data['YrSold'] = categorized_data['YrSold'].astype(str)
 categorized_data['MoSold'] = categorized_data['MoSold'].astype(str)
 return categorized_data
print(test.isnull().sum() )<categorify>
def encode_categories(data):
all_data = data.copy()
cols_to_encode =('PoolQC', 'Alley', 'Fence', 'FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'KitchenQual', 'BsmtFinType1', 'BsmtFinType2', 'Functional',
'BsmtExposure', 'GarageFinish', 'LandSlope','LotShape', 'PavedDrive', 'Street', 'CentralAir', 'MSSubClass',
'OverallCond', 'YrSold', 'MoSold')
 for c in cols_to_encode:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
return all_data
train['Title'] = train.Name.str.extract('([A-Za-z]+ ).', expand=False)
 train['Title'] = train['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train['Title'] = train['Title'].replace('Mlle', 'Miss')
 train['Title'] = train['Title'].replace('Ms', 'Miss')
 train['Title'] = train['Title'].replace('Mme', 'Mrs')
 Title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
train['Title'] = train['Title'].map(Title_mapping)
 train['Title'] = train['Title'].fillna(0)
 del train['Name']
train["Age"] = train["Age"].fillna(train["Age"].median())
 train["Sex"] = train["Sex"].map({"male" : 0, "female" : 1})
 train["Embarked"] = train["Embarked"].fillna("S")
 train["Embarked"] = train["Embarked"].map({"S" : 0, "Q" : 1, "C" : 2})
 train['Ticket_Lett'] = train['Ticket'].apply(lambda x: str(x)[0])
 train['Ticket_Lett'] = train['Ticket_Lett'].apply(lambda x: str(x))
train['Ticket_Lett'] = np.where(( train['Ticket_Lett'] ).isin(['1', '2', '3', 'S', 'P', 'C', 'A']), train['Ticket_Lett'], np.where(( train['Ticket_Lett'] ).isin(['W', '4', '7', '6', 'L', '5', '8']), '0','0'))
train['Ticket_Lett']=train['Ticket_Lett'].replace("1",1 ).replace("2",2 ).replace("3",3 ).replace("0",0 ).replace("S",3 ).replace("P",0 ).replace("C",3 ).replace("A",3)
 del train['Ticket']
train['Cabin_Lett'] = train['Cabin'].apply(lambda x: str(x)[0])
 train['Cabin_Lett'] = train['Cabin_Lett'].apply(lambda x: str(x))
train['Cabin_Lett'] = np.where(( train['Cabin_Lett'] ).isin([ 'F', 'E', 'D', 'C', 'B', 'A']),train['Cabin_Lett'], np.where(( train['Cabin_Lett'] ).isin(['W', '4', '7', '6', 'L', '5', '8']), '0','0'))
train['Cabin_Lett']=train['Cabin_Lett'].replace("A",1 ).replace("B",2 ).replace("C",1 ).replace("0",0 ).replace("D",2 ).replace("E",2 ).replace("F",1)
 del train['Cabin']
train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
train<categorify>
all_data = fill_missings(join_data)
 all_data = create_additional_features(all_data)
 all_data = categorize_data(all_data)
 all_data = encode_categories(all_data)
 X = all_data[:train_size]
X_test_full = all_data[train_size:]
print(len(all_data), len(X), len(X_test_full))
X.dropna(axis=0, subset=['SalePrice'], inplace=True)
 y = X.SalePrice
X.drop(['SalePrice'], axis=1, inplace=True)
 outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers])
 overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 X = X.drop(overfit, axis=1)
 X_train_full, X_valid_full, y_train, y_valid = train_test_split(X, y, train_size=0.99, test_size=0.01,
random_state=0)
 low_cardinality_cols = [cname for cname in X_train_full.columns if X_train_full[cname].nunique() < 10 and
X_train_full[cname].dtype not in ['int64', 'float64']]
print("Low cardinality columns: ", low_cardinality_cols)
 numeric_cols = [cname for cname in X_train_full.columns if X_train_full[cname].dtype in ['int64', 'float64']]
print("Numeric columns: ", numeric_cols)
 my_cols = low_cardinality_cols + numeric_cols
X_train = X_train_full[my_cols].copy()
X_valid = X_valid_full[my_cols].copy()
X_test = X_test_full[my_cols].copy()
X_train = pd.get_dummies(X_train)
 X_valid = pd.get_dummies(X_valid)
 X_test = pd.get_dummies(X_test)
 X_train, X_valid = X_train.align(X_valid, join='left', axis=1)
 X_train, X_test = X_train.align(X_test, join='left', axis=1 )
test['Title'] = test.Name.str.extract('([A-Za-z]+ ).', expand=False)
 test['Title'] = test['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 test['Title'] = test['Title'].replace('Mlle', 'Miss')
 test['Title'] = test['Title'].replace('Ms', 'Miss')
 test['Title'] = test['Title'].replace('Mme', 'Mrs')
 Title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
test['Title'] = test['Title'].map(Title_mapping)
 test['Title'] = test['Title'].fillna(0)
 del test['Name']
test["Age"] = test["Age"].fillna(test["Age"].median())
 test["Sex"] = test["Sex"].map({"male" : 0, "female" : 1})
 test["Embarked"] = test["Embarked"].fillna("S")
 test["Embarked"] = test["Embarked"].map({"S" : 0, "Q" : 1, "C" : 2})
 test['Ticket_Lett'] = test['Ticket'].apply(lambda x: str(x)[0])
 test['Ticket_Lett'] = test['Ticket_Lett'].apply(lambda x: str(x))
test['Ticket_Lett'] = np.where(( test['Ticket_Lett'] ).isin(['1', '2', '3', 'S', 'P', 'C', 'A']), test['Ticket_Lett'], np.where(( test['Ticket_Lett'] ).isin(['W', '4', '7', '6', 'L', '5', '8']), '0','0'))
test['Ticket_Lett']=test['Ticket_Lett'].replace("1",1 ).replace("2",2 ).replace("3",3 ).replace("0",0 ).replace("S",3 ).replace("P",0 ).replace("C",3 ).replace("A",3)
 del test['Ticket']
test['Cabin_Lett'] = test['Cabin'].apply(lambda x: str(x)[0])
 test['Cabin_Lett'] = test['Cabin_Lett'].apply(lambda x: str(x))
test['Cabin_Lett'] = np.where(( test['Cabin_Lett'] ).isin([ 'F', 'E', 'D', 'C', 'B', 'A']),test['Cabin_Lett'], np.where(( test['Cabin_Lett'] ).isin(['W', '4', '7', '6', 'L', '5', '8']), '0','0'))
test['Cabin_Lett']=test['Cabin_Lett'].replace("A",1 ).replace("B",2 ).replace("C",1 ).replace("0",0 ).replace("D",2 ).replace("E",2 ).replace("F",1)
 del test['Cabin']
test["FamilySize"] = test["SibSp"] + test["Parch"] + 1
test["Fare"] = test["Fare"].fillna(test["Fare"].median())
 test<prepare_x_and_y>
skewed_feats = X_train[numeric_cols].apply(lambda x: stats.skew(x.dropna())).sort_values(ascending=False)
 y_train = y_train.apply(lambda x: np.log(x))
y_valid = y_valid.apply(lambda x: np.log(x))
log_col = []
for col in skewed_feats.index:
if(abs(skewed_feats[col])>0.8):
log_col.append(col)
 X_train[col]=X_train[col].apply(lambda x: np.log(x))
X_valid[col]=X_valid[col].apply(lambda x: np.log(x))
X_test[col]=X_test[col].apply(lambda x: np.log(x))
train_x = train.drop(['PassengerId'], axis=1)
 trainx_corr = train_x.corr()
plt.figure(figsize=(8, 8))
sns.heatmap(trainx_corr, cmap="Reds", annot=True, fmt="1.2f")
 train_x = train_x.drop(['Survived'], axis=1)
 train_y = train['Survived']
test_x= test.drop(['PassengerId'], axis=1 )<normalization>
def optimize_xgb(all_data):
xgb1 = XGBRegressor()
parameters = {'nthread':[1],
'objective':['reg:linear'],
'learning_rate': [.02,.01,.0075,.005],
'max_depth': [5, 6, 7],
'min_child_weight': [3, 4, 5],
'subsample': [0.7],
'colsample_bytree': [0.7],
'n_estimators': [2500, 5000]}
xgb_grid = GridSearchCV(xgb1,
parameters,
cv = 2,
n_jobs = 5,
verbose=True)
 xgb_grid.fit(X_train, y_train)
 print(xgb_grid.best_score_)
 print(xgb_grid.best_params_)
stdsc = StandardScaler()
trainx_std = stdsc.fit_transform(train_x)
 testx_std = stdsc.fit_transform(test_x)
 X_train = np.reshape(trainx_std,(-1, 11, 1))
Y_train = to_categorical(train_y, num_classes = 2)
 X_test = np.reshape(testx_std,(-1, 11, 1))<choose_model_class>
model = XGBRegressor(colsample_bytree=0.7, learning_rate=.01, max_depth=6, min_child_weight=3, n_estimators=3000,
nthread=1, objective='reg:squarederror', subsample=0.7, random_state=21,
early_stopping_rounds = 10, eval_set=[(X_valid, y_valid)], verbose=False )
model = Sequential()
model.add(Conv1D(32, 5, padding='same', input_shape=(11, 1), activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Conv1D(32, 5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(MaxPool1D(2, padding='same'))
model.add(Conv1D(64, 3, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(2, activation='softmax'))
model.summary()<save_to_csv>
print("Let's the training begin.Plase wait.")
 my_pipeline = Pipeline(steps=[('model', model)])
 my_pipeline.fit(X_train, y_train)
 print("Training finished! Now let's predict test values.")
 preds_test = my_pipeline.predict(X_test)
 preds_test = [np.exp(pred)for pred in preds_test]
output = pd.DataFrame({'Id': X_test.index+9,
'SalePrice': preds_test})
 output.to_csv('submission.csv', index=False)
 print("Everything finished correctly!")
 print(output.SalePrice )
plot_model(model, to_file='model.png', show_layer_names=False, show_shapes=True )<choose_model_class>
lightgbm = lgb.LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 lgb_model_full_data = lightgbm.fit(X_train, y_train )
model.compile(optimizer = "Adam" , loss='binary_crossentropy', metrics=["accuracy"] )<choose_model_class>
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 xgb_model_full_data = xgboost.fit(X_train, y_train )
<train_model><EOS> 
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def blend_models_predict(X):
return(( 1 * xgb_model_full_data.predict(X)) + \ 
(0 * lgb_model_full_data.predict(X)))
 print('RMSLE score on train data:')
 print(rmsle(y_valid, blend_models_predict(X_valid)) )