Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<count_values>
df = pd.read_csv('.. /input/train.csv')
 enumSeries = []
numSeries = []
nonDiscreteSeries = []
for colname in df.columns:
datatype = str(df[colname].dtype)
 if colname == 'Id' or colname == 'SalePrice':
pass
elif datatype == 'int64' or datatype == 'float64':
numSeries.append(colname)
 else:
enumSeries.append(colname)
 df.fillna(pd.Series([0,1970,0, 5, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0], index=['LotFrontage', 'GarageYrBlt', 'MasVnrArea', 'OverallQual', 'GrLivArea10', 'GarageCars', '1stFlrSF10', 'TotalBsmtSF', 'GarageArea', 'BsmtFullBath', 'BsmtUnfSF', 'BsmtHalfBath', 'BsmtFinSF2', 'BsmtFinSF1']), inplace=True)
 assert df[pd.isnull(df[numSeries] ).any(axis=1)][numSeries].shape[0] == 0
df['LotAreaLog10'] = np.log10(df['LotArea'].values)
 df['1stFlrSFLog10'] = np.log10(df['1stFlrSF'].values)
 df['GrLivAreaLog10'] = np.log10(df['GrLivArea'].values)
 df['SalePriceLog10'] = np.log10(df['SalePrice'].values)
 if 'LotArea' in numSeries:
numSeries.remove('LotArea')
 numSeries.append('LotAreaLog10')
 if '1stFlrSF' in numSeries:
numSeries.remove('1stFlrSF')
 numSeries.append('1stFlrSFLog10')
 if 'GrLivArea' in numSeries:
numSeries.remove('GrLivArea')
 numSeries.append('GrLivAreaLog10')
 df[numSeries].head()
print("--- Train set ---")
 print(df_train['City Group'].value_counts())
 print("---- Test set ----")
 print(df_test['City Group'].value_counts() )<count_values>
allEnums = []
for seriesName in enumSeries:
allEnums.append([])
 enums = df[seriesName].value_counts()
for enum in enums.index.values:
allEnums[-1].append({ 'name': enum, 'series': seriesName, 'mean': df.query(seriesName + ' == "' + enum + '"')['SalePrice'].mean() , 'count': df.query(seriesName + ' == "' + enum + '"')['SalePrice'].count() })
 if pd.isnull(df[seriesName] ).any() :
val = 0.0
allEnums[-1].append({ 'name': None, 'series': seriesName, 'mean': df.query(seriesName + ' != ' + seriesName)['SalePrice'].mean() , 'count': df.query(seriesName + ' == "' + enum + '"')['SalePrice'].count() })
 else:
val = 1.0
allEnums[-1] = sorted(allEnums[-1], key=lambda k: k['mean'])
 meanSumForNull = 0.0
meanCntForNull = 0
for enumDict in allEnums[-1]:
enumDict['value'] = val
meanSumForNull += val * enumDict['count']
meanCntForNull += enumDict['count']
val += 1.0
if not pd.isnull(df[seriesName] ).any() :
allEnums[-1].append({ 'name': None, 'series': seriesName, 'mean': 0, 'value': meanSumForNull / float(meanCntForNull), 'count': 0 })
 for enumCollection in allEnums:
for enumDict in enumCollection:
if enumDict['name']:
print(enumDict['series'] + '-' + enumDict['name'] + ': ' + str(enumDict['value'])+ '(' + str(enumDict['count'])+ ')')
 else:
print(enumDict['series'] + '-NULL: ' + str(enumDict['value'])+ '(' + str(enumDict['count'])+ ')')
print("--- Train set ---")
 print(df_train['Type'].value_counts())
 print("---- Test set ----")
 print(df_test['Type'].value_counts())
<prepare_x_and_y>
for enumCollection in allEnums:
srcCol = enumCollection[0]['series']
destCol = enumCollection[0]['series'] + 'Numeric'
mapping = {}
naValue = 0.0
for enumDict in enumCollection:
if enumDict['name']:
mapping[enumDict['name']] = enumDict['value']
else:
naValue = enumDict['value']
df[destCol] = df[srcCol].map(mapping)
 df[destCol].fillna(naValue, inplace=True)
 if not destCol in numSeries:
numSeries.append(destCol)
 df[numSeries].head()
y = df_train['revenue']
df_train=df_train.drop('revenue', axis=1 )<count_unique_values>
ridge = Ridge()
parameters = { 'alpha': [ 1e-15, 1e-10, 1e-8, 1e-4, 1e-3, 1e-2, 1, 5, 10, 20 ]}
ridge_regressor = GridSearchCV(ridge, parameters, scoring='neg_mean_squared_error', cv=5)
 ridge_regressor.fit(df[numSeries].values, df['SalePriceLog10'])
 print(ridge_regressor.best_params_ )
print('There are {} cities which restaurant location have been collected.'.format(len(df_full['City'].unique())) )<drop_column>
ridge = Ridge(alpha=1)
 ridge.fit(df[numSeries].values, df['SalePriceLog10'])
 ridgePredictions = np.power(10, ridge.predict(df[numSeries].values))
targets = np.power(10, df['SalePriceLog10'])
 mae = mean_absolute_error(ridgePredictions, targets)
 mae
df_full = df_full.drop('City', axis=1)
 df_full.shape<define_variables>
tdf = pd.read_csv('.. /input/test.csv')
 tdf.fillna(pd.Series([0,1970,0, 5, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0], index=['LotFrontage', 'GarageYrBlt', 'MasVnrArea', 'OverallQual', 'GrLivArea10', 'GarageCars', '1stFlrSF10', 'TotalBsmtSF', 'GarageArea', 'BsmtFullBath', 'BsmtUnfSF', 'BsmtHalfBath', 'BsmtFinSF2', 'BsmtFinSF1']), inplace=True)
 tdf['LotAreaLog10'] = np.log10(tdf['LotArea'].values)
 tdf['1stFlrSFLog10'] = np.log10(tdf['1stFlrSF'].values)
 tdf['GrLivAreaLog10'] = np.log10(tdf['GrLivArea'].values)
 for enumCollection in allEnums:
srcCol = enumCollection[0]['series']
destCol = enumCollection[0]['series'] + 'Numeric'
mapping = {}
naValue = 0.0
for enumDict in enumCollection:
if enumDict['name']:
mapping[enumDict['name']] = enumDict['value']
else:
naValue = enumDict['value']
tdf[destCol] = tdf[srcCol].map(mapping)
 tdf[destCol].fillna(naValue, inplace=True)
 if not destCol in numSeries:
numSeries.append(destCol)
 assert tdf[pd.isnull(tdf[numSeries] ).any(axis=1)][numSeries].shape[0] == 0
tdf[numSeries].head()
<categorify><EOS> 
ridgeFinalPredictions = np.power(10, ridge.predict(tdf[numSeries].values))
assert ridgeFinalPredictions.shape[0] == tdf['Id'].shape[0]
res = pd.DataFrame([tdf['Id'], ridgeFinalPredictions] ).transpose().rename(columns={'Unnamed 0': 'SalePrice'})
 res.Id = res.Id.astype(int)
 res.to_csv('submission.csv', index=False)
 res