Kaggle competition: Mercedes-Benz Greener Manufacturing
<SOS> metric: R2Score Kaggle data source: mercedes-benz-greener-manufacturing<save_to_csv>
%matplotlib inline
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
print(os.listdir(".. /input"))
color = sns.color_palette()
X_test = df.iloc[:,1:].values
y_test = model.predict(X_test)
 pred = pd.DataFrame({"PassengerId": df.iloc[:,0].values, "Survived": y_test})
 pred.to_csv('results.csv', index=False, header=True )<set_options>
data = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sub_df = pd.read_csv('.. /input/sample_submission.csv' )
warnings.simplefilter('ignore')
<load_from_csv>
data = data[data.y < 250]
df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 df.head()<normalization>
train_objs_num = len(data)
 dataset = pd.concat(objs=[data, test], axis=0)
 dataset = pd.get_dummies(dataset)
 data = copy.copy(dataset[:train_objs_num])
 test = copy.copy(dataset[train_objs_num:] )
numeric_cols=[]
rScaler = RobustScaler()
mmScaler = MinMaxScaler()
def transform_data(X, ft = 'fit'):
global numeric_cols, rScaler,mmScaler
X['MSSubClass']=X['MSSubClass'].astype(np.object)
 numeric_cols = [cname for cname in X.columns if X[cname].dtype in ['int64', 'float64']]
categorical_cols = [cname for cname in X.columns if X[cname].dtype == 'object']
scaled_cols = ['OverallQual','OverallCond','YearBuilt','YearRemodAdd','BsmtFullBath','BsmtHalfBath','FullBath',
'HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd','Fireplaces','GarageYrBlt','GarageCars','MoSold','YrSold']
X = pd.get_dummies(X, prefix=categorical_cols, dummy_na=True, columns=categorical_cols)
 numeric_cols = list(set(numeric_cols)-set(scaled_cols))
if ft == 'fit':
X[scaled_cols] = mmScaler.fit_transform(X[scaled_cols])
 X[numeric_cols] = rScaler.fit_transform(X[numeric_cols])
 else:
X[scaled_cols] = mmScaler.transform(X[scaled_cols])
 X[numeric_cols] = rScaler.transform(X[numeric_cols])
 X[scaled_cols] = X[scaled_cols].fillna(0)
 X[numeric_cols] = X[numeric_cols].fillna(0)
 return X<define_variables>
useless_var = data.mean() [data.mean() == 0].index
y_train = data.y
data.drop('y', axis = 1, inplace=True)
 data.drop('ID', axis = 1, inplace= True)
 data.drop(useless_var.values, axis = 1, inplace=True)
 test.drop('y', axis = 1, inplace=True)
 test.drop('ID', axis = 1, inplace= True)
 test.drop(useless_var.values, axis = 1, inplace=True )
save_errors = []<split>
depth1= ['X314', 'X315', 'X118', 'X232', 'X136', 'X127', 'X189', 'X261', 'X47', 'X236', 'X5_n', 'X5_w', 'X383', 'X204', 'X339', 'X275', 'X152', 'X50', 'X1_f', 'X6_e', 'X5_r', 'X104', 'X5_aa', 'X240', 'X206', 'X241', 'X95', 'X1_l', 'X113', 'X5_ag', 'X8_s', 'X5_v', 'X100', 'X13', 'X8_k', 'X6_c', 'X223', 'X114', 'X19', 'X115', 'X1_u', 'X80', 'X4_c', 'X68', 'X168', 'X306', 'X61', 'X350', 'X300', 'X132', 'X342', 'X1_c', 'X6_l', 'X267', 'X379', 'X23', 'X316', 'X8_j', 'X6_g', 'X5_m', 'X12', 'X0_z', 'X0_x', 'X77', 'X354', 'X196', 'X8_a', 'X283', 'X51', 'X8_g', 'X225', 'X292', 'X2_r', 'X148', 'X8_d', 'X8_x', 'X203', 'X5_k', 'X8_o', 'X101', 'X220', 'X191', 'X5_ab', 'X8_i', 'X8_w', 'X56', 'X65', 'X0_ab', 'X131', 'X224', 'X155', 'X181', 'X351', 'X0_au', 'X64', 'X75', 'X5_af', 'X105', 'X6_j', 'X117', 'X151', 'X177', 'X0_l', 'X362', 'X3_g', 'X5_l', 'X8_v', 'X285', 'X378', 'X244', 'X343', 'X228', 'X5_ae', 'X5_p', 'X273', 'X380', 'X5_ah', 'X27', 'X1_t', 'X6_h', 'X345', 'X1_h', 'X176', 'X116', 'X171', 'X1_aa', 'X6_d', 'X201', 'X122', 'X3_a', 'X5_ad', 'X178', 'X143', 'X38', 'X142', 'X14', 'X361', 'X1_q']
depth2= [('X118', 'X314'),('X314', 'X315'),('X232', 'X314'),('X136', 'X315'),('X127', 'X232'),('X189', 'X315'),('X232', 'X315'),('X314', 'X47'),('X261', 'X47'),('X315', 'X47'),('X232', 'X261'),('X113', 'X118'),('X118', 'X261'),('X261', 'X5_r'),('X236', 'X47'),('X314', 'X61'),('X232', 'X5_k'),('X115', 'X261'),('X206', 'X50'),('X236', 'X261'),('X5_w', 'X8_k'),('X383', 'X50'),('X315', 'X6_c'),('X232', 'X47'),('X241', 'X5_n'),('X47', 'X5_w'),('X236', 'X275'),('X1_l', 'X5_aa'),('X127', 'X136'),('X261', 'X61'),('X152', 'X5_n'),('X143', 'X315'),('X118', 'X5_w'),('X118', 'X5_r'),('X5_aa', 'X5_w'),('X104', 'X118'),('X152', 'X383'),('X275', 'X5_n'),('X118', 'X236'),('X383', 'X5_n'),('X351', 'X47'),('X5_aa', 'X80'),('X113', 'X261'),('X240', 'X6_l'),('X236', 'X314'),('X13', 'X1_l'),('X316', 'X6_e'),('X115', 'X1_f'),('X275', 'X339'),('X114', 'X1_c'),('X136', 'X232'),('X127', 'X6_e'),('X339', 'X47'),('X47', 'X61'),('X115', 'X314'),('X168', 'X8_k'),('X1_u', 'X47'),('X151', 'X236'),('X127', 'X47'),('X152', 'X19'),('X223', 'X6_g'),('X118', 'X1_aa'),('X104', 'X68'),('X101', 'X152'),('X306', 'X5_w'),('X23', 'X8_k'),('X113', 'X314'),('X5_r', 'X5_w'),('X47', 'X5_n'),('X383', 'X6_l'),('X127', 'X5_n'),('X204', 'X5_n'),('X6_l', 'X8_j'),('X0_x', 'X47'),('X300', 'X383'),('X240', 'X8_s'),('X5_n', 'X5_w'),('X1_c', 'X5_r'),('X100', 'X342'),('X196', 'X5_n'),('X47', 'X5_aa'),('X0_z', 'X152'),('X47', 'X5_ad'),('X204', 'X383'),('X379', 'X5_m'),('X105', 'X232'),('X19', 'X5_n'),('X118', 'X1_q'),('X261', 'X339'),('X191', 'X339'),('X350', 'X47'),('X236', 'X240'),('X5_r', 'X8_v'),('X148', 'X95'),('X127', 'X339'),('X350', 'X5_n'),('X132', 'X283'),('X77', 'X8_j'),('X1_l', 'X56'),('X0_x', 'X1_u'),('X12', 'X5_ag'),('X1_f', 'X8_d'),('X204', 'X236'),('X204', 'X316'),('X116', 'X47'),('X113', 'X8_a'),('X6_l', 'X77'),('X204', 'X95'),('X225', 'X383'),('X100', 'X5_ab'),('X100', 'X1_f'),('X5_m', 'X8_s'),('X19', 'X6_g'),('X115', 'X8_s'),('X339', 'X383'),('X127', 'X1_f'),('X118', 'X316'),('X204', 'X5_aa'),('X118', 'X8_a'),('X115', 'X155'),('X1_f', 'X5_v'),('X104', 'X275'),('X132', 'X1_l'),('X1_f', 'X3_g'),('X1_c', 'X315'),('X23', 'X5_w'),('X132', 'X339'),('X116', 'X77'),('X12', 'X383'),('X2_r', 'X8_a'),('X131', 'X5_aa'),('X51', 'X5_k'),('X5_ah', 'X8_g'),('X204', 'X4_c'),('X383', 'X5_v'),('X362', 'X5_n'),('X220', 'X342'),('X204', 'X339'),('X132', 'X354'),('X132', 'X203'),('X204', 'X240'),('X383', 'X5_l'),('X236', 'X292'),('X1_c', 'X343'),('X267', 'X95'),('X292', 'X378'),('X350', 'X5_l'),('X345', 'X61'),('X240', 'X8_o'),('X117', 'X275'),('X306', 'X5_n'),('X100', 'X8_o'),('X0_x', 'X224'),('X5_n', 'X6_j'),('X178', 'X61'),('X236', 'X383'),('X19', 'X5_m'),('X236', 'X95'),('X132', 'X5_v'),('X132', 'X6_g'),('X132', 'X236'),('X19', 'X204'),('X5_ag', 'X5_k'),('X315', 'X6_g'),('X240', 'X8_x'),('X267', 'X316'),('X383', 'X75'),('X261', 'X383'),('X105', 'X5_ag'),('X267', 'X383'),('X5_v', 'X5_w'),('X65', 'X6_j'),('X177', 'X267'),('X342', 'X95'),('X354', 'X6_j'),('X306', 'X75'),('X12', 'X95'),('X115', 'X64'),('X105', 'X114'),('X181', 'X316'),('X350', 'X380'),('X383', 'X5_r'),('X383', 'X8_x'),('X292', 'X5_n'),('X113', 'X3_a'),('X142', 'X68'),('X0_x', 'X351'),('X292', 'X339'),('X342', 'X5_m'),('X1_l', 'X8_w'),('X204', 'X47'),('X151', 'X6_g'),('X6_h', 'X75'),('X240', 'X342'),('X1_h', 'X64'),('X12', 'X5_v'),('X267', 'X51'),('X342', 'X5_ag'),('X176', 'X8_s'),('X339', 'X4_c'),('X171', 'X64'),('X339', 'X8_d'),('X206', 'X383'),('X285', 'X350'),('X342', 'X5_v'),('X14', 'X68'),('X1_u', 'X203'),('X148', 'X244'),('X203', 'X4_c'),('X228', 'X8_w'),('X342', 'X5_aa'),('X12', 'X5_r'),('X0_au', 'X339'),('X132', 'X8_i'),('X203', 'X267'),('X0_au', 'X0_z'),('X100', 'X23'),('X350', 'X8_v'),('X292', 'X8_d'),('X177', 'X1_u'),('X4_c', 'X5_w'),('X148', 'X267'),('X104', 'X95'),('X339', 'X95'),('X1_t', 'X5_ae'),('X206', 'X8_i'),('X228', 'X5_aa'),('X315', 'X351'),('X104', 'X342'),('X5_p', 'X8_a'),('X0_l', 'X4_c'),('X104', 'X206'),('X5_v', 'X8_d'),('X5_m', 'X95'),('X316', 'X5_ah'),('X104', 'X5_w'),('X1_t', 'X8_a'),('X292', 'X5_ag'),('X292', 'X5_af'),('X1_u', 'X6_d'),('X306', 'X4_c'),('X0_ab', 'X4_c'),('X5_aa', 'X5_ah'),('X240', 'X275'),('X240', 'X95'),('X342', 'X5_af'),('X273', 'X5_ae'),('X0_au', 'X5_r'),('X115', 'X177'),('X0_ab', 'X204')]
depth3=[('X136', 'X314', 'X315'),('X127', 'X232', 'X314'),('X189', 'X314', 'X315'),('X232', 'X314', 'X315'),('X113', 'X118', 'X314'),('X314', 'X315', 'X47'),('X143', 'X314', 'X315'),('X232', 'X314', 'X47'),('X314', 'X315', 'X6_c'),('X118', 'X1_q', 'X314'),('X118', 'X261', 'X5_r'),('X236', 'X314', 'X47'),('X118', 'X1_aa', 'X314'),('X261', 'X315', 'X47'),('X232', 'X261', 'X5_k'),('X118', 'X314', 'X5_r'),('X118', 'X314', 'X8_a'),('X118', 'X1_aa', 'X261'),('X261', 'X5_r', 'X8_v'),('X127', 'X232', 'X261'),('X314', 'X47', 'X61'),('X236', 'X261', 'X275'),('X206', 'X383', 'X50'),('X178', 'X314', 'X61'),('X127', 'X136', 'X232'),('X115', 'X1_f', 'X261'),('X236', 'X261', 'X47'),('X115', 'X261', 'X8_s'),('X104', 'X118', 'X5_w'),('X136', 'X232', 'X261'),('X118', 'X236', 'X5_w'),('X168', 'X5_w', 'X8_k'),('X261', 'X47', 'X5_ad'),('X23', 'X5_w', 'X8_k'),('X275', 'X383', 'X5_n'),('X261', 'X47', 'X5_w'),('X261', 'X47', 'X61'),('X152', 'X19', 'X5_n'),('X261', 'X345', 'X61'),('X261', 'X5_r', 'X5_w'),('X127', 'X232', 'X47'),('X261', 'X350', 'X47'),('X113', 'X314', 'X47'),('X13', 'X1_l', 'X5_aa'),('X127', 'X232', 'X6_e'),('X0_x', 'X1_u', 'X47'),('X113', 'X261', 'X8_a'),('X0_x', 'X351', 'X47'),('X232', 'X51', 'X5_k'),('X306', 'X47', 'X5_w'),('X315', 'X351', 'X47'),('X241', 'X47', 'X5_n'),('X204', 'X316', 'X6_e'),('X115', 'X1_f', 'X314'),('X232', 'X5_ag', 'X5_k'),('X240', 'X383', 'X6_l'),('X114', 'X1_c', 'X5_r'),('X127', 'X232', 'X5_n'),('X113', 'X261', 'X3_a'),('X115', 'X155', 'X314'),('X241', 'X275', 'X5_n'),('X383', 'X6_l', 'X8_j'),('X101', 'X152', 'X383'),('X47', 'X5_n', 'X5_w'),('X19', 'X223', 'X6_g'),('X47', 'X5_w', 'X8_k'),('X196', 'X275', 'X5_n'),('X47', 'X5_aa', 'X80'),('X191', 'X275', 'X339'),('X0_x', 'X224', 'X47'),('X1_l', 'X47', 'X5_aa'),('X261', 'X339', 'X47'),('X236', 'X240', 'X275'),('X5_aa', 'X5_w', 'X80'),('X300', 'X383', 'X5_l'),('X0_z', 'X152', 'X19'),('X127', 'X232', 'X339'),('X148', 'X267', 'X95'),('X1_l', 'X5_aa', 'X5_w'),('X152', 'X204', 'X383'),('X204', 'X350', 'X5_n'),('X1_l', 'X56', 'X5_aa'),('X105', 'X232', 'X5_ag'),('X204', 'X236', 'X47'),('X105', 'X114', 'X232'),('X132', 'X283', 'X339'),('X19', 'X204', 'X5_n'),('X151', 'X236', 'X275'),('X339', 'X383', 'X47'),('X127', 'X1_f', 'X232'),('X204', 'X5_aa', 'X5_w'),('X100', 'X342', 'X5_ab'),('X100', 'X1_f', 'X5_v'),('X6_l', 'X77', 'X8_j'),('X131', 'X204', 'X5_aa'),('X12', 'X5_ag', 'X5_r'),('X225', 'X383', 'X5_r'),('X118', 'X204', 'X316'),('X204', 'X383', 'X5_v'),('X104', 'X275', 'X339'),('X116', 'X351', 'X47'),('X1_f', 'X236', 'X8_d'),('X132', 'X1_l', 'X5_v'),('X1_f', 'X3_g', 'X5_v'),('X132', 'X339', 'X354'),('X152', 'X19', 'X204'),('X204', 'X383', 'X50'),('X12', 'X383', 'X5_l'),('X204', 'X339', 'X5_n'),('X1_c', 'X315', 'X5_r'),('X5_aa', 'X5_r', 'X5_w'),('X151', 'X236', 'X47'),('X116', 'X13', 'X47'),('X117', 'X275', 'X339'),('X116', 'X6_l', 'X77'),('X306', 'X362', 'X5_n'),('X1_t', 'X2_r', 'X8_a'),('X236', 'X383', 'X95'),('X100', 'X342', 'X8_o'),('X220', 'X342', 'X5_aa'),('X19', 'X204', 'X5_m'),('X5_aa', 'X5_ah', 'X8_g'),('X132', 'X236', 'X5_v'),('X1_u', 'X47', 'X95'),('X100', 'X240', 'X342'),('X1_c', 'X343', 'X5_r'),('X5_aa', 'X5_w', 'X8_k'),('X379', 'X5_m', 'X95'),('X267', 'X316', 'X383'),('X240', 'X6_l', 'X8_o'),('X306', 'X5_n', 'X6_j'),('X12', 'X342', 'X95'),('X350', 'X380', 'X5_l'),('X5_aa', 'X5_v', 'X5_w'),('X261', 'X383', 'X5_r'),('X204', 'X383', 'X8_x'),('X19', 'X315', 'X6_g'),('X306', 'X383', 'X75'),('X292', 'X339', 'X5_n'),('X240', 'X6_l', 'X8_x'),('X1_l', 'X5_aa', 'X8_w'),('X267', 'X383', 'X51'),('X240', 'X342', 'X5_m'),('X1_h', 'X241', 'X5_n'),('X0_l', 'X379', 'X5_m'),('X204', 'X47', 'X95'),('X177', 'X1_u', 'X267'),('X132', 'X203', 'X6_g'),('X23', 'X5_aa', 'X5_w'),('X181', 'X204', 'X316'),('X176', 'X240', 'X8_s'),('X104', 'X142', 'X68'),('X306', 'X6_h', 'X75'),('X339', 'X4_c', 'X8_d'),('X0_ab', 'X65', 'X6_j'),('X0_ab', 'X354', 'X6_j'),('X115', 'X1_h', 'X64'),('X204', 'X240', 'X95'),('X5_m', 'X8_s', 'X95'),('X12', 'X5_r', 'X5_v'),('X104', 'X5_w', 'X68'),('X132', 'X203', 'X4_c'),('X151', 'X19', 'X6_g'),('X292', 'X339', 'X378'),('X342', 'X5_aa', 'X5_v'),('X148', 'X244', 'X95'),('X115', 'X177', 'X267'),('X203', 'X267', 'X4_c'),('X151', 'X236', 'X383'),('X0_au', 'X0_z', 'X339'),('X104', 'X206', 'X383'),('X240', 'X8_s', 'X95'),('X285', 'X350', 'X5_l'),('X1_u', 'X203', 'X47'),('X104', 'X14', 'X68'),('X342', 'X4_c', 'X5_ag'),('X171', 'X1_h', 'X64'),('X204', 'X4_c', 'X5_w'),('X228', 'X5_aa', 'X8_w'),('X100', 'X23', 'X5_w'),('X236', 'X292', 'X8_d'),('X104', 'X339', 'X95'),('X204', 'X240', 'X8_s'),('X132', 'X203', 'X8_i'),('X0_l', 'X5_m', 'X8_s'),('X0_l', 'X204', 'X4_c'),('X350', 'X5_l', 'X8_v'),('X104', 'X206', 'X8_i'),('X104', 'X342', 'X5_w'),('X1_t', 'X5_p', 'X8_a'),('X0_au', 'X273', 'X339'),('X236', 'X292', 'X5_ag'),('X316', 'X5_aa', 'X5_ah'),('X1_u', 'X47', 'X6_d'),('X0_ab', 'X306', 'X4_c'),('X236', 'X5_v', 'X8_d'),('X236', 'X292', 'X5_af'),('X240', 'X275', 'X95'),('X1_t', 'X273', 'X5_ae'),('X0_au', 'X4_c', 'X5_r'),('X0_ab', 'X204', 'X5_ag'),('X342', 'X4_c', 'X5_af'),('X122', 'X1_t', 'X5_ae'),('X19', 'X1_h', 'X5_n'),('X236', 'X292', 'X378'),('X204', 'X240', 'X261'),('X114', 'X228', 'X5_aa'),('X177', 'X203', 'X27'),('X0_au', 'X201', 'X4_c'),('X203', 'X27', 'X8_g'),('X0_l', 'X131', 'X38'),('X0_l', 'X131', 'X361')]
df.dropna(axis=0, subset=['SalePrice'], inplace=True)
 y_df = df['SalePrice']
X = df.drop(['SalePrice'], axis=1)
 y = np.log(y_df)
 X_train2, X_test2, y_train, y_test = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=42)
 X_train1 = transform_data(X_train2)
 X_test1 = transform_data(X_test2, 'transform')
 missing = list(set(X_train1.columns)- set(X_test1.columns))
for col in missing:
X_test1[col]=np.zeros(len(X_test1.index))
with pd.option_context('display.max_columns', None):
display(X_train1.head() ,X_train1.shape, X_test1.head() , X_test1.shape )<train_on_grid>
data = data[depth1]
test = test[depth1]
for(a,b)in depth2[0:17]:
data.loc[:,a+'_+_'+b] = data[a].add(data[b])
 data.loc[:,a+'_*_'+b] = data[a].mul(data[b])
 data.loc[:,a+'_abs(-)_'+b] = np.abs(data[a].sub(data[b]))
test.loc[:,a+'_+_'+b] = test[a].add(test[b])
 test.loc[:,a+'_*_'+b] = test[a].mul(test[b])
 test.loc[:,a+'_abs(-)_'+b] = np.abs(test[a].sub(test[b]))
for(a,b,c)in depth3[0:13]:
data.loc[:,a+'_+_'+b+'_+_'+c] = data[a].add(data[b] ).add(data[c])
 test.loc[:,a+'_+_'+b+'_+_'+c] = test[a].add(test[b] ).add(test[c])
 data.loc[:,a+'_*_'+b+'_*_'+c] = data[a].mul(data[b] ).mul(data[c])
 test.loc[:,a+'_*_'+b+'_*_'+c] = test[a].mul(test[b] ).mul(test[c])
 data.loc[:,a+'_+_'+b+'_*_'+c] = data[a].add(data[b] ).mul(data[c])
 test.loc[:,a+'_+_'+b+'_*_'+c] = test[a].add(test[b] ).mul(test[c])
 data.loc[:,a+'_*_'+b+'_+_'+c] = data[a].mul(data[b] ).add(data[c])
 test.loc[:,a+'_*_'+b+'_+_'+c] = test[a].mul(test[b] ).add(test[c])
 test.loc[:,a+'_-_'+b+'_+_'+c] = np.abs(test[a].sub(test[b])).add(test[c])
 test.loc[:,a+'_-_'+b+'_*_'+c] = np.abs(test[a].sub(test[b])).mul(test[c])
 test.loc[:,a+'_*_'+b+'_-_'+c] = test[a].mul(np.abs(( test[b] ).sub(test[c])))
 test.loc[:,a+'_+_'+b+'_-_'+c] = test[a].add(np.abs(( test[b] ).sub(test[c])))
 test.loc[:,a+'_-_'+b+'_-_'+c] = np.abs(np.abs(test[a].sub(test[b])).sub(test[c]))
data.loc[:,a+'_-_'+b+'_+_'+c] = np.abs(data[a].sub(data[b])).add(data[c])
 data.loc[:,a+'_-_'+b+'_*_'+c] = np.abs(data[a].sub(data[b])).mul(data[c])
 data.loc[:,a+'_*_'+b+'_-_'+c] = data[a].mul(np.abs(( data[b] ).sub(data[c])))
 data.loc[:,a+'_+_'+b+'_-_'+c] = data[a].add(np.abs(( data[b] ).sub(data[c])))
 data.loc[:,a+'_-_'+b+'_-_'+c] = np.abs(np.abs(data[a].sub(data[b])).sub(data[c]))
nlist=[]
las_scores=[]
las_scores2=[]
for i in np.linspace(0.000001,0.001,21):
est = linear_model.LassoCV(eps=i,cv=5,random_state=5)
 ft = SelectFromModel(est)
 rf = RFECV(est, cv=5)
 df1 = ft.fit_transform(X_train1,y_train)
 df2 = rf.fit_transform(df1,y_train)
 nlist.append(i)
 las_scores.append(rf.score(df1,y_train))
las_scores2.append(rf.score(X_test1[X_train1.columns].loc[:,ft.get_support() ],y_test))
plt.figure(figsize=(10,5))
plt.plot(nlist,las_scores,'b+-')
 plt.plot(nlist,las_scores2,'r+-')
 plt.xlabel('eps')
 plt.ylabel('R^2')
 plt.show() 
<compute_test_metric>
to_drop = []
for(c1,c2,score)in data_1:
if score >0.95:
to_drop.append(c2)
 print(to_drop)
 data.drop(to_drop, axis=1, inplace = True)
 test.drop(to_drop, axis=1, inplace = True)
display(pd.DataFrame({'ScoreTrain':[round(x,3)for x in las_scores],'ScoreTest':[round(x,3)for x in las_scores2]},index=[round(x, -int(floor(log10(x)))+1)for x in nlist]))<train_on_grid>
n_folds = 5
def r2_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(data.values)
 r2= cross_val_score(model, data.values, y_train, scoring="r2", cv = kf)
 return(r2)
est = linear_model.LassoCV(eps=0.00015,cv=5,random_state=5)
 ft = SelectFromModel(est)
 rf = RFECV(est, cv=5)
 print('Starting...')
 df1 = ft.fit_transform(X_train1,y_train)
 df2 = rf.fit_transform(df1,y_train)
 print('Done')
<feature_engineering>
model_xgb = xgb.XGBRegressor(alpha= 0, colsample_bytree= 1, eta= 0.005, reg_lambda= 0, max_depth= 4, min_child_weight= 0, n_estimators= 100 )
dfXtr = transform_data(X.loc[X_train.index,:])
 for col in list(set(X_train.columns)- set(dfXtr.columns)) :
dfXtr[col]=np.zeros(len(dfXtr.index))
dfXtr[X_train.columns]<train_model>
gridParams = {
'eta': [0.005],
'max_depth': [3,4,5],
'n_estimators': [100,200],
'min_child_weight' : [0,0.25],
'colsample_bytree' : [0.8,1],
'lambda' : [0,0.2],
'alpha': [0]
}
est.fit(X_train,y_train)
 with pd.option_context('display.max_rows', None):
display(pd.DataFrame({'Coef':est.coef_},index=X_train.columns ).sort_values('Coef', ascending=False))<find_best_model_class>
rf = RandomForestRegressor(max_depth= 6, min_samples_leaf= 2, min_samples_split= 30, n_estimators= 50 )
def cvScore(df1, y, outL=[]):
scores = []
kf = KFold(n_splits=5,shuffle =True,random_state=11)
 kfs = kf.split(df1.index)
 dfXtr = X.loc[df1.index,:]
for train_index,test_index in kfs:
X_tr2 = dfXtr.reset_index().loc[train_index,:]
X_tr = transform_data(X_tr2.drop('Id', axis=1))
X_tr['Id'] = X_tr2['Id']
col = X_tr.columns.tolist()
X_tr = X_tr[col[-1:] + col[:-1]]
for col in list(set(df1.columns)- set(X_tr.columns)) :
X_tr[col]=np.zeros(len(X_tr.index))
X_tr = X_tr.drop([idx[0] for idx in X_tr.iterrows() if idx[1][0] in outL])
 X_tr = X_tr[df1.columns]
X_tst2 = dfXtr.reset_index().loc[test_index,:]
X_tst = transform_data(X_tst2.drop('Id', axis=1))
X_tst['Id'] = X_tst2['Id']
col = X_tst.columns.tolist()
X_tst = X_tst[col[-1:] + col[:-1]]
for col in list(set(df1.columns)- set(X_tst.columns)) :
X_tst[col]=np.zeros(len(X_tst.index))
X_tst = X_tst[df1.columns]
y_tr = y.reset_index().loc[train_index]
y_tr = y_tr.drop([idx[0] for idx in y_tr.iterrows() if idx[1][0] in outL])
 y_tr = y_tr.drop('Id',axis=1)
 y_tst = y.reset_index(drop=True ).loc[test_index]
est.fit(X_tr,y_tr)
 p = est.predict(X_tst)
 scores.append(round(np.sqrt(mean_squared_error(y_tst, p)) ,4))
return scores<compute_test_metric>
gridParams = {
'max_depth': [3,6,7],
'n_estimators': [10,50,100],
'min_samples_leaf' : [2,10],
'min_samples_split' : [10,30,60]
}
print('Train R^2:',rf.score(X_train_selected,y_train))
scores = cvScore(X_train,y_train)
 error1 = round(np.mean(scores),4)
 print("Scores:", scores, '
Average Train RMSE:', error1)
 print("Accuracy: %0.2f(+/- %0.2f)" %(error1, np.std(scores)* 2))<compute_test_metric>
model_lgb = lgb.LGBMRegressor(feature_fraction= 1, lambda_l1= 0, learning_rate= 0.05, max_depth= 3, min_data_in_leaf= 50, n_estimators= 500, num_leaves= 4 )
print('Test R^2:', rf.score(X_test,y_test))
est.fit(X_train,y_train)
 preds = est.predict(X_test[X_train.columns])
 rmse1 = round(np.sqrt(mean_squared_error(y_test, preds)) ,4)
 print('Test RSME:',rmse1)
 display(pd.DataFrame({'Predicted Prices':np.around(np.e**preds,-2)},index=y_test.index)
.join(pd.DataFrame({'Actual Prices':np.around(np.e**y_test,2)})).head())
<split>
KRR = KernelRidge(kernel='polynomial', alpha = 1, coef0=3, degree = 2 )
num_feature_df = [colname for colname in X_train.columns if colname in numeric_cols]
eps_test(X_train[num_feature_df] )<find_best_params>
gridParams = {
'alpha': [1,2,3],
'degree': [2,3,4],
'coef0' : [1,2,3],
}
def get_eps(df,max_percent=9):
eps=0
j=0
for p in range(3):
for i in range(1,11):
j = j + 1/(10**p)
 check_outliers = DBSCAN(eps=j)
 clusters = check_outliers.fit_predict(df)
 outliers_found = [i for i in clusters if i != 0]
if(( len(np.unique(clusters)))<=2)and(len(outliers_found)<len(df.index)) :
if p != 2:
j = j-(1/(10**p))
eps = j
break
eps = round(eps,2)
 if eps > 0:
print('Epsilon=',eps,', Outliers: ',len(outliers_found),', ',round(len(outliers_found)/len(clusters)*100,2),'%')
 if(len(outliers_found)/len(clusters)*100)>= max_percent or eps==0:
print('eps=',eps,' Trying again...')
 for i in range(1,50):
check_outliers = DBSCAN(eps=i)
 clusters = check_outliers.fit_predict(df)
 outliers_found = [x for x in clusters if x !=0]
if(len(outliers_found)/len(clusters)*100)< max_percent:
j = i-1
for p in range(1,3):
for i in range(1,11):
j = j + 1/(10**p)
 check_outliers = DBSCAN(eps=j)
 clusters = check_outliers.fit_predict(df)
 outliers_found = [x for x in clusters if x !=0]
if(len(outliers_found)/len(clusters)*100)<= max_percent:
if p != 2:
j = j-(1/(10**p))
eps = j
break
break
eps = round(eps,2)
 print('Epsilon=',eps,', Outliers: ',len(outliers_found),', ',round(len(outliers_found)/len(clusters)*100,4),'%')
 return eps<find_best_params>
svr = SVR(C= 1, gamma= 0.001 )
eps = get_eps(X_train[num_feature_df] )<predict_on_test>
lasso = Lasso(alpha =0.0005, random_state=1 )
check_outliers = DBSCAN(eps=eps)
 clusters = check_outliers.fit_predict(X_train[num_feature_df])
 results = pd.DataFrame({'Result':clusters,'Count':X_train[num_feature_df].index},index=X_train[num_feature_df].index)
 display(results.groupby('Result' ).count())
 outliers = results[results['Result'] !=0]
print('Outliers: ',outliers.shape[0],', ',outliers.shape[0]/results.shape[0]*100,'%' )<merge>
gridParams = {
'alpha': [0,0.0005,0.005],
}
get_graphs(X_train.join(y_train),results, num_feature_df )<train_model>
et = ExtraTreesRegressor(max_depth= 5, min_samples_leaf= 40, n_estimators= 500)
scores2 = cvScore(X_train.drop([idx for idx in X_train.index if idx in list(outliers.index)]),
y_train.drop([idx for idx in y_train.index if idx in list(outliers.index)]))
error2 = round(np.mean(scores2),4)
 est.fit(X_train.drop([idx for idx in X_train.index if idx in list(outliers.index)]),
y_train.drop([idx for idx in y_train.index if idx in list(outliers.index)]))
print('Train R^2:',est.score(X_train.drop([idx for idx in X_train.index if idx in list(outliers.index)]),
y_train.drop([idx for idx in y_train.index if idx in list(outliers.index)])))
 print("Scores:", scores2, '
Average Train RMSE:', error2)
 print('Original CV Train RMSE mean:',error1)
 print("Accuracy: %0.2f(+/- %0.2f)" %(error2, np.std(scores2)* 2))<train_model>
gridParams = {
'n_estimators': [500],
'max_depth': [2,5,6,7],
'min_samples_leaf' : [30,40,50],
}
scores3 = cvScore(X_train,y_train,outliers.index)
 error3 = round(np.mean(scores3),4)
 est.fit(X_train.drop([idx for idx in X_train.index if idx in list(outliers.index)]),
y_train.drop([idx for idx in y_train.index if idx in list(outliers.index)]))
print('Train R^2:',est.score(X_train,y_train))
print("Scores:", scores3, '
Average Train RMSE:', error3)
 print('Original CV Train RMSE mean:',error1)
 print("Accuracy: %0.2f(+/- %0.2f)" %(error3, np.std(scores3)* 2)) 
<compute_train_metric>
ada = AdaBoostRegressor(learning_rate= 0.005, n_estimators= 50 )
est.fit(X_train.drop([idx for idx in X_train.index if idx in list(outliers.index)]),
y_train.drop([idx for idx in y_train.index if idx in list(outliers.index)]))
print('Test R^2:',est.score(X_test[X_train.columns],y_test))
preds = est.predict(X_test[X_train.columns])
 rmse2 = round(np.sqrt(mean_squared_error(y_test, preds)) ,4)
 print('RMSE:',rmse2,'
Old RMSE:',rmse1)
 display(pd.DataFrame({'Predicted Prices':np.around(np.e**preds,-2)},index=y_test.index)
.join(pd.DataFrame({'Actual Prices':np.e**y_test})).head() )<compute_train_metric>
gridParams = {
'n_estimators': [30,50,80],
'learning_rate': [0.005, 0.05,0.5,1]
}
be = BaggingRegressor(est, random_state=11)
 be.fit(X_train.drop([idx for idx in X_train.index if idx in list(outliers.index)])
 ,y_train.drop([idx for idx in X_train.index if idx in list(outliers.index)]))
preds4 = be.predict(X_test[X_train.columns])
 error4 = np.sqrt(mean_squared_error(y_test, preds4))
print('LASSO RSME:', rmse1, rmse2, 'BaggingRegressor RMSE:', round(error4,4))
print('Score train',be.score(X_train.drop([idx for idx in X_train.index if idx in list(outliers.index)])
 ,y_train.drop([idx for idx in X_train.index if idx in list(outliers.index)])) ,'Score test:',
be.score(X_test[X_train.columns],y_test))<compute_train_metric>
cat = CatBoostRegressor(iterations=3000,learning_rate=0.005,depth=2 )
abr = AdaBoostRegressor(est, random_state=11)
 abr.fit(X_train1.drop([idx for idx in X_train.index if idx in list(outliers.index)]),
y_train.drop([idx for idx in y_train.index if idx in list(outliers.index)]))
preds5 = abr.predict(X_test1[X_train1.columns])
 error5 = np.sqrt(mean_squared_error(y_test, preds5))
print('LASSO RSME:', rmse1, rmse2, 'AdaBoostRegressor RMSE:', round(error5,4))
print('Score train',abr.score(X_train1.drop([idx for idx in X_train.index if idx in list(outliers.index)])
 ,y_train.drop([idx for idx in X_train.index if idx in list(outliers.index)])) ,'Score test:',
abr.score(X_test1[X_train1.columns],y_test))<compute_train_metric>
gridParams = {
'iterations': [3000],
'learning_rate': [0.005],
'depth' : [2,4,6]
}
gbr = GradientBoostingRegressor(random_state=11)
 gbr.fit(X_train1.drop([idx for idx in X_train.index if idx in list(outliers.index)]),
y_train.drop([idx for idx in y_train.index if idx in list(outliers.index)]))
preds6 = gbr.predict(X_test1[X_train1.columns])
 error6 = np.sqrt(mean_squared_error(y_test, preds6))
print('LASSO RSME:', rmse1, rmse2, 'GradientBoostingRegressor RMSE:', round(error6,4))
print('Score train',gbr.score(X_train1.drop([idx for idx in X_train.index if idx in list(outliers.index)])
 ,y_train.drop([idx for idx in X_train.index if idx in list(outliers.index)])) ,'Score test:',
gbr.score(X_test1[X_train1.columns],y_test))<features_selection>
gbr = GradientBoostingRegressor(n_estimators=1000,learning_rate=0.005,max_depth=3,alpha=0.1 )
print('Starting...')
 X1 = transform_data(X)
 df1 = ft.fit_transform(X1,y)
 df2 = rf.fit_transform(df1,y)
 print('Done')
 selected_columns1 = ft.get_support()
X_selected = pd.DataFrame(data=df1, columns = X1.columns[selected_columns1],index=X1.index)
 selected_columns2 = rf.get_support()
X_New = pd.DataFrame(data=df2, columns = X_selected.columns[selected_columns2],index=X1.index)
 display(X_New.head())
 num_feature_dfX = [colname for colname in X_New.columns if colname in numeric_cols]
eps_test(X_New[num_feature_dfX])
<find_best_model_class>
gridParams = {
'n_estimators': [1000],
'learning_rate': [0.005],
'max_depth' : [3,5,7],
'alpha':[0.1, 0.5]
}
eps = get_eps(X1[num_feature_dfX],3)
 check_outliers = DBSCAN(eps=eps)
 clustersX = check_outliers.fit_predict(X1[num_feature_dfX])
 resultsX = pd.DataFrame({'Result':clustersX,'Count':X1[num_feature_dfX].index},index=X1[num_feature_dfX].index)
 display(resultsX.groupby('Result' ).count())
 outliersX = resultsX[resultsX['Result'] !=0]
get_graphs(X1.join(y),resultsX, num_feature_dfX)
<compute_test_metric>
eNet = ElasticNet(alpha=0,l1_ratio=0,max_iter=5 )
scoresX = cvScore(X_New,y,outliersX.index)
 print("Scores:", scoresX, '
Average:', round(np.mean(scoresX),4))
est.fit(X_New.drop(outliersX.index), y.drop(outliersX.index))
print('R^2:', est.score(X_New,y))
pd.DataFrame({'Coef':est.coef_},index=X_New.columns ).sort_values('Coef', ascending=False)
 predsX = est.predict(X_New)
 display(pd.DataFrame({'Predicted Prices':np.around(np.e**predsX,-2)},index=y.index)
.join(pd.DataFrame({'Actual Prices':np.around(np.e**y,2)})).head() )<load_from_csv>
gridParams = {
'alpha': [0],
'max_iter':[0,5,10,15],
'l1_ratio': [0]
}
df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')
 display(df_test.head())
 df_test = transform_data(df_test,'transform')
 missing = list(set(X_New.columns)- set(df_test.columns))
for col in missing:
df_test[col]=np.zeros(len(df_test.index))
display(df_test.head() )<predict_on_test>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
est.fit(X_New.drop(outliersX.index), y.drop(outliersX.index))
test_preds = est.predict(df_test[X_New.columns])
 e_test_preds = np.e**test_preds
df_out = pd.DataFrame({'SalePrice':e_test_preds},index=df_test.index)
 df_out.head()<save_to_csv>
averaged_models = AveragingModels(models =(KRR, model_lgb, rf, svr, model_xgb, lasso, eNet,ada, et, gbr, cat))
".format(score.mean() , score.std()))
df_out.to_csv('results.csv' )<set_options>
averaged_models.fit(data.values,y_train )
warnings.filterwarnings(action='ignore', category=DataConversionWarning)
 warnings.filterwarnings(action='ignore', category=DeprecationWarning)
 warnings.filterwarnings(action='ignore', category=FutureWarning )<load_from_csv>
predictions = np.expm1(averaged_models.predict(test.values))
train = pd.read_csv(".. /input/train.csv")
 compe = pd.read_csv(".. /input/test.csv")
 sample_sub = pd.read_csv(".. /input/sample_submission.csv")
 data = train.append(compe, sort=False )<create_dataframe>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=8):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y.values[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
<count_missing_values><EOS> 
sub_df.drop('y', axis = 1, inplace = True)
 sub_df['y'] = predictions
sub_df.to_csv("submission.csv", index=False)
 sub_df