Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
pd.set_option('display.max_rows', 100)
 pd.set_option('display.max_columns', 100)
 filterwarnings('ignore')
 train_set=pd.read_csv(r'.. /input/train.csv')
 test_set=pd.read_csv(r'.. /input/test.csv')
 train_test = pd.concat([train_set,test_set],axis=0 ).reset_index(drop=True)
 train_test = train_test.drop('Id',axis=1 )
def clean_StadiumType(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt
def transform_StadiumType(txt):
if pd.isna(txt):
return np.nan
if 'outdoor' in txt or 'open' in txt:
return 1
if 'indoor' in txt or 'closed' in txt:
return 0
return np.nan
train_df["StadiumType"] = train_df["StadiumType"].apply(clean_StadiumType)
 train_df['StadiumType'] = train_df['StadiumType'].apply(transform_StadiumType )<define_variables>
EasyVariables = ['MiscFeature', 'MasVnrType', 'GarageType']
for var in EasyVariables:
train_test[var] = train_test[var].fillna('DNE' )
Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial',
'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural',
'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial',
'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train_df['Turf'] = train_df['Turf'].map(Turf)
 train_df['Turf'] = train_df['Turf'] == 'Natural'<categorify>
FillWith0 = ['BsmtFinType2', 'BsmtCond', 'BsmtQual', 'BsmtExposure', 'BsmtFinType1','BsmtFinSF1','BsmtFinSF2','TotalBsmtSF',
'BsmtUnfSF','BsmtHalfBath','BsmtFullBath', 'FireplaceQu','GarageYrBlt','MasVnrArea']
for var in FillWith0:
train_test[var] = train_test[var].fillna(0)
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train_df['PossessionTeam'].unique() :
map_abbr[abb] = abb
train_df['PossessionTeam'] = train_df['PossessionTeam'].map(map_abbr)
 train_df['HomeTeamAbbr'] = train_df['HomeTeamAbbr'].map(map_abbr)
 train_df['VisitorTeamAbbr'] = train_df['VisitorTeamAbbr'].map(map_abbr)
 train_df['HomePossesion'] = train_df['PossessionTeam'] == train_df['HomeTeamAbbr']
train_df['Field_eq_Possession'] = train_df['FieldPosition'] == train_df['PossessionTeam']
train_df['HomeField'] = train_df['FieldPosition'] == train_df['HomeTeamAbbr']<categorify>
train_test.Utilities = train_test.Utilities.fillna(4)
 train_test.Functional = train_test.Functional.fillna('Typ')
 train_test.SaleType = train_test.SaleType.fillna('WD')
 train_test.KitchenQual = train_test.KitchenQual.fillna(3)
 train_test.Electrical = train_test.Electrical.fillna('SBrkr')
 train_test.MSZoning = train_test.MSZoning.fillna('RL')
 train_test[['Exterior1st','Exterior2nd']] = train_test[['Exterior1st','Exterior2nd']].fillna('VinylSd')
off_form = train_df['OffenseFormation'].unique()
train_df = pd.concat([train_df.drop(['OffenseFormation'], axis=1), pd.get_dummies(train_df['OffenseFormation'], prefix='Formation')], axis=1)
 dummy_col = train_df.columns<feature_engineering>
train_test = train_test.replace({'MSSubClass':{
20:'1-STORY 1946 & NEWER ALL STYLES',
30:'1-STORY 1945 & OLDER',
40:'1-STORY W/FINISHED ATTIC ALL AGES',
45:'1-1/2 STORY - UNFINISHED ALL AGES',
50:'1-1/2 STORY FINISHED ALL AGES',
60:'2-STORY 1946 & NEWER',
70:'2-STORY 1945 & OLDER',
75:'2-1/2 STORY ALL AGES',
80:'SPLIT OR MULTI-LEVEL',
85:'SPLIT FOYER',
90:'DUPLEX - ALL STYLES AND AGES',
120:'1-STORY PUD(Planned Unit Development)- 1946 & NEWER',
150:'1-1/2 STORY PUD - ALL AGES',
160:'2-STORY PUD - 1946 & NEWER',
180:'PUD - MULTILEVEL - INCL SPLIT LEV/FOYER',
190:'2 FAMILY CONVERSION - ALL STYLES AND AGES'}})
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
train_df['GameClock'] = train_df['GameClock'].apply(strtoseconds )<feature_engineering>
train_test.loc[[2420,2503,2599],'PoolQC'] = 3
train_test.loc[2576,'GarageYrBlt'] = 1923
train_test.loc[2126,'GarageYrBlt'] = 1910
train_test.loc[[2576,2126],'GarageFinish'] = 1
train_test.loc[[2576,2126],'GarageQual'] = 3
train_test.loc[[2576,2126],'GarageCond'] = 3
train_test.loc[2576, 'GarageCars'] = 1
train_test.loc[2576, 'GarageArea'] = 288
train_test.loc[2592,'GarageYrBlt'] = 2007
train_test.loc[[2040, 2524, 2185, 2040],'BsmtCond'] = 3
train_test.loc[[2348,1487,948],'BsmtExposure'] = 3
train_test.loc[[2217, 2218],'BsmtQual'] = 3
train_test.loc[2284,'BsmtFinSF1'] = 638
train_test.loc[2284,'BsmtFinSF2'] = 0
train_test = train_test.drop(332, axis=0)
 train_test = train_test.drop([1200, 873],axis=0 )
def strtoinches(txt):
txt = txt.split('-')
 ans = int(txt[0])* 12 + int(txt[1])
 return ans
train_df['PlayerHeight'] = train_df['PlayerHeight'].apply(strtoinches)
 train_df['PlayerBMI'] = 703 *(train_df['PlayerWeight']/(train_df['PlayerHeight'])**2 )<feature_engineering>
index = train_test[(train_test.OverallQual==10)&(train_test.SalePrice<200000)].index
train_test = train_test.drop(index,axis=0)
 index = train_test[(train_test.GarageArea>1200)&(train_test.SalePrice<300000)].index
train_test = train_test.drop(index, axis=0)
 index = train_test[(train_test.YearRemodAdd<1970)&(train_test.SalePrice>300000)].index
train_test = train_test.drop(index,axis=0)
 index = train_test[(train_test['TotRmsAbvGrd']==14)].index
train_test = train_test.drop(index,axis=0 )
train_df['TimeHandoff'] = train_df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train_df['TimeSnap'] = train_df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train_df['TimeDelta'] = train_df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1 )<feature_engineering>
train_test1 = train_test.copy()
train_test1 = pd.get_dummies(train_test1)
 trainX = train_test1[train_test1.LotFrontage.isnull() ==False].drop(['SalePrice','LotFrontage'],axis=1)
 testX = train_test[train_test.LotFrontage.isnull() ==True].drop(['SalePrice','LotFrontage'],axis=1)
 trainY = train_test1.LotFrontage[train_test1.LotFrontage.isnull() ==False]
trainx, testx, trainy, testy = train_test_split(trainX, np.log(trainY), test_size=0.3, random_state=0)
 model = Lasso(alpha=0.0009)
 model.fit(trainx,trainy)
 Predictions = model.predict(testx)
 MSE = np.sqrt(mean_squared_error(np.exp(testy),np.exp(Predictions)))
 print('LotFrontage MSE:',MSE,'
MSE / LotFrontage mean ratio:', MSE/(np.exp(testy.mean())) )
train_df['PlayerBirthDate'] = train_df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train_df['PlayerAge'] = train_df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() / seconds_in_year, axis=1)
 train_df = train_df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1 )<feature_engineering>
trainX = train_test1[train_test1.LotFrontage.isnull() ==False].drop(['SalePrice','LotFrontage'],axis=1)
 testX = train_test1[train_test1.LotFrontage.isnull() ==True].drop(['SalePrice','LotFrontage'],axis=1)
 trainY = train_test1.LotFrontage[train_test1.LotFrontage.isnull() ==False]
model = Lasso(alpha=0.0009)
 model.fit(trainx,trainy)
 Predictions = model.predict(testX)
 train_test.loc[train_test.LotFrontage.isnull() ==True, 'LotFrontage'] = Predictions
train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+ int(x.split('-')[1])) / 2 if not pd.isna(x)and '-' in x else x)
 train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x:(int(x.split() [0])+ int(x.split() [-1])) / 2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x: float(x)if str(x ).isnumeric() else -1)
 def clean_WindDirection(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt
def transform_WindDirection(txt):
if pd.isna(txt):
return np.nan
if txt=='n':
return 0
if txt=='nne' or txt=='nen':
return 1/8
if txt=='ne':
return 2/8
if txt=='ene' or txt=='nee':
return 3/8
if txt=='e':
return 4/8
if txt=='ese' or txt=='see':
return 5/8
if txt=='se':
return 6/8
if txt=='ses' or txt=='sse':
return 7/8
if txt=='s':
return 8/8
if txt=='ssw' or txt=='sws':
return 9/8
if txt=='sw':
return 10/8
if txt=='sww' or txt=='wsw':
return 11/8
if txt=='w':
return 12/8
if txt=='wnw' or txt=='nww':
return 13/8
if txt=='nw':
return 14/8
if txt=='nwn' or txt=='nnw':
return 15/8
return np.nan
train_df['WindDirection'] = train_df['WindDirection'].apply(clean_WindDirection)
 train_df['WindDirection'] = train_df['WindDirection'].apply(transform_WindDirection )<feature_engineering>
train_test['t'] = train_test.YrSold.map({2006:1, 2007:2, 2008:3, 2009:4, 2010:5})
 train_test.t = train_test.MoSold * train_test.t
train_test = train_test.replace({'MoSold':{1:'Jan',2:'Feb',3:'Mar',4:'Apr',5:'May',6:
'Jun',7:'Jul',8:'Aug',9:'Sep',10:'Oct',11:'Nov',12:'Dec'}})
 train_test['GarageQualityOverall'] = train_test.GarageQual + train_test.GarageCond
train_test['HasPool'] = train_test.PoolQC.map(lambda x: 1 if x>0 else 0)
 train_test['HasGarage'] = train_test.GarageFinish.map(lambda x: 1 if x>0 else 0)
 train_test['HasFireplace'] = train_test.Fireplaces.map(lambda x: 1 if x>0 else 0)
 train_test['HasBasement'] = train_test.TotalBsmtSF.map(lambda x: 1 if x>0 else 0)
 train_test['AllStoriesFinished'] = train_test.HouseStyle.map(lambda x: 0 if(x=='1.5Unf' or x=='2.5Unf')else 1)
 train_test['CanHold2Cars'] = train_test.GarageCars.map(lambda x: 1 if x>=2 else 0)
 train_test['AfterCrisis'] = train_test.YrSold.map(lambda x: 1 if x>=2008 else 0)
 closed = train_test['EnclosedPorch'] + train_test['3SsnPorch'] + train_test['ScreenPorch']
opened = train_test['WoodDeckSF'] + train_test['OpenPorchSF']
train_test['PorchArea'] = closed + opened
train_test['HasPorch'] = train_test['PorchArea'].map(lambda x: 1 if x>0 else 0)
 train_test['PorchClosed'] = closed.map(lambda x: 1 if x > 0 else 0)
 IsFresh = train_test[(train_test.YearRemodAdd == train_test.YrSold)].index
train_test['IsFresh'] = train_test.index.map(lambda x: 1 if x in IsFresh else 0)
 train_test['BsmtFinType'] = train_test.BsmtFinType2 + train_test.BsmtFinType1
train_test['BsmtUnfinPercent'] = train_test.BsmtUnfSF / train_test.TotalBsmtSF
train_test.loc[train_test['BsmtUnfinPercent'].isnull() ,'BsmtUnfinPercent'] = 0
train_test['TotalHouseSF'] = train_test.TotalBsmtSF + train_test.GrLivArea
train_test['AmtOfStories'] = 0
train_test.loc[train_test.HouseStyle=='1Story','AmtOfStories'] = 1
train_test.loc[(train_test.HouseStyle=='1.5Fin')|(train_test.HouseStyle=='1.5Unf'), 'AmtOfStories'] = 1.5
train_test.loc[(train_test.HouseStyle=='2.5Fin')|(train_test.HouseStyle=='2.5Unf'), 'AmtOfStories'] = 2.5
train_test.loc[(train_test.HouseStyle=='2Story'), 'AmtOfStories'] = 2
train_test['OverallQuality'] = train_test.OverallQual + train_test.OverallCond
train_test['BasementQuality'] = train_test.BsmtCond + train_test.BsmtExposure
train_test['Bathrooms'] = train_test.BsmtFullBath + train_test.BsmtHalfBath/2 + train_test.FullBath + train_test.HalfBath/2
train_test['ExteriorQuality'] = train_test.ExterQual + train_test.ExterCond
train_test['Fireplaces_times_FireplaceQu'] = train_test.Fireplaces * train_test.FireplaceQu
train_test['BsmtFinSF'] = train_test.BsmtFinSF1 + train_test.BsmtFinSF2
train_test['YearBltandYearRemodAdd'] = train_test.YearBuilt + train_test.YearRemodAdd
train_test['GarageTypeGrades'] = train_test.GarageType.replace({'2Types':6,'Attchd':5,'Basment':4,'BuiltIn':3,'CarPort':2,'Detchd':1,'No_garage':0})
 Season = ['Mar', 'Apr', 'May', 'Jun']
train_test['SeasonForBuing'] = train_test.MoSold.map(lambda x: 1 if x in Season else 0)
 train_test = train_test.drop('Utilities',axis=1 )
train_df['PlayDirection'] = train_df['PlayDirection'].apply(lambda x: x.strip() == 'right' )<feature_engineering>
train_test['RichNeighborhood'] = np.NaN
ones = NeighborhoodsBinned[0:3].index
twos = NeighborhoodsBinned[3:12].index
threes = NeighborhoodsBinned[12:19].index
fours = NeighborhoodsBinned[19:22].index
fives = NeighborhoodsBinned[22:].index
i=0
for variable in [ones, twos, threes, fours, fives]:
i += 1
for neighborhood in variable:
train_test.loc[train_test.Neighborhood==neighborhood,'RichNeighborhood'] = i
train_test = train_test.drop('Neighborhood',axis=1 )
train_df['Team'] = train_df['Team'].apply(lambda x: x.strip() =='home' )<feature_engineering>
train_test.Exterior1st = train_test.Exterior1st.map(lambda x: 'Other' if(x!='CemntBd' and x!='VinylSd')else x)
 train_test.Exterior2nd = train_test.Exterior2nd.map(lambda x: 'Other' if(x!='CemntBd' and x!='VinylSd')else x)
 train_test.Functional = train_test.Functional.map(lambda x: 'Other' if x!='Typ' else x)
 train_test.GarageYrBlt = train_test.GarageYrBlt.map(lambda x: 'DNE' if x==0 else 'Classic' if x<=1935 else 'Modern' if x>=1995 else 'Mid-age')
 train_test.SaleType = train_test.SaleType.map(lambda x: 'Other' if x!='WD' else x)
 train_test.SaleCondition = train_test.SaleCondition.map(lambda x: 'Other' if(x!='Normal' and x!='Partial')else x)
 train_test.Condition1 = train_test.Condition1.map(lambda x: x if x in ['Norm', 'Feedr', 'Artery'] else 'Other')
 train_test.RoofMatl = train_test.RoofMatl.map(lambda x: 'Other' if x!='CompShg' else x)
 train_test = train_test.replace({'BldgType':{'Duplex':'Other', 'Twnhs':'Other', '2fmCon':'Other'},
'Electrical':{'FuseF':'Other','FuseP':'Other','Mix':'Other'},
'Foundation':{'Slab':'Other','Stone':'Other','Wood':'Other'},
'Heating':{'Grav':'Other','Wall':'Other','OthW':'Other','Floor':'Other'},
'HouseStyle':{'2.5Unf':'Other','1.5Unf':'Other','2.5Fin':'Other'},
'MSSubClass':{'1-1/2 STORY PUD - ALL AGES':'Other','1-STORY W/FINISHED ATTIC ALL AGES':'Other'\
,'PUD - MULTILEVEL - INCL SPLIT LEV/FOYER':'Other','1-1/2 STORY - UNFINISHED ALL AGES':'Other'\
,'2-1/2 STORY ALL AGES':'Other'},
'MSZoning':{'RH':'Other','C(all)':'Other'},
'MiscFeature':{'Gar2':'Other','TenC':'Other'}} )
train_df['GameWeather'] = train_df['GameWeather'].str.lower()
indoor = "indoor"
train_df['GameWeather'] = train_df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train_df['GameWeather'] = train_df['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train_df['GameWeather'] = train_df['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train_df['GameWeather'] = train_df['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
train_df['GameWeather'] = train_df['GameWeather'].apply(map_weather )<drop_column>
for var in ['1stFlrSF','BsmtFinSF2','MasVnrArea', 'BsmtFinSF', 'TotalBsmtSF', 'TotalHouseSF']:
train_test['LOG{}'.format(var)] = np.log(1+train_test[var])
 train_test['LOGYearBltandYearRemodAdd'] = train_test['YearBltandYearRemodAdd']**2
train_test = train_test.drop(['1stFlrSF','BsmtFinSF','TotalBsmtSF'],axis=1 )
train_df['IsRusher'] = train_df['NflId'] == train_df['NflIdRusher']
train_df.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True )<feature_engineering>
train_test.SalePrice = np.log(train_test.SalePrice)
 train_test = train_test.drop(['GarageQual','GarageCond','EnclosedPorch','3SsnPorch','OpenPorchSF','BsmtFinType2','BsmtFinType1','BsmtUnfSF','GrLivArea',
'BsmtCond','BsmtExposure','BsmtFullBath','BsmtHalfBath','FullBath','ExterCond'],axis=1 )
<feature_engineering><EOS> 
y =(train_test.SalePrice[train_test.SalePrice.isnull() ==False])
 train_test = pd.get_dummies(train_test)
 X = train_test[train_test.SalePrice.isnull() ==False].drop('SalePrice',axis=1)
 testX = train_test[train_test.SalePrice.isnull() ==True].drop('SalePrice',axis=1)
 model = Lasso(alpha = 0.0008)
 model.fit(X,y)
 predictions = model.predict(testX)
 model = ElasticNet(alpha = 0.0008)
 model.fit(X,y)
 predictions2 = model.predict(testX)
 model = XGBRegressor(n_estimators=4500)
 model.fit(X,y)
 predictions3 = model.predict(testX)
 predictions =(predictions + predictions2 + predictions2 + predictions3)/4
Final = pd.DataFrame({'Id':range(1461,2920),'SalePrice': np.exp(predictions)} ).set_index('Id')
 Final.to_csv(r'Blended.csv' )