Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<find_best_params>
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
hyperparameters = {'C' : [0.01,0.1,1,10,100]}
grid_search_cv(classifier_svc,hyperparameters )<compute_train_metric>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.describe()
model_evaluation(classifier_svc )<import_modules>
train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )
from sklearn.tree import DecisionTreeClassifier<choose_model_class>
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index )
classifier_dt = DecisionTreeClassifier(criterion = 'entropy' )<compute_test_metric>
train["SalePrice"] = np.log1p(train["SalePrice"])
 check_skewness('SalePrice' )
model_evaluation(classifier_dt )<import_modules>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
from sklearn.ensemble import RandomForestClassifier<choose_model_class>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na} )
classifier_rf = RandomForestClassifier(max_depth = 2,random_state = 0 )<compute_test_metric>
all_data.PoolQC.loc[all_data.PoolQC.notnull() ]
model_evaluation(classifier_rf )<import_modules>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )
from sklearn.neighbors import KNeighborsClassifier<import_modules>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )
from sklearn.neighbors import KNeighborsClassifier<import_modules>
all_data["Alley"] = all_data["Alley"].fillna("None" )
from sklearn.neighbors import KNeighborsClassifier<choose_model_class>
all_data["Fence"] = all_data["Fence"].fillna("None" )
classifier_knn = KNeighborsClassifier(leaf_size = 7, n_neighbors = 3,p = 1 )<find_best_params>
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
leaf_size = list(range(1,50))
n_neighbors = list(range(1,30))
p=[1,2]
hyperparameters = dict(leaf_size=leaf_size, n_neighbors=n_neighbors, p=p)
 grid_search_cv(classifier_knn,hyperparameters )<train_model>
grouped_df = all_data.groupby('Neighborhood')['LotFrontage']
for key, item in grouped_df:
print(key,"
")
 print(grouped_df.get_group(key))
break
model(classifier_knn )<compute_test_metric>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
model_evaluation(classifier_knn )<define_variables>
for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
all_data[col] = all_data[col].fillna('None' )
models = {'MODELS':['LOGISTIC REGRESSION','SUPPORT VECTOR CLASSIFIER','DECISION TREE CLASSIFIER','RANDOM FOREST CLASSIFIER','K-NEAREST NEIGHBORS'],
'CROSS VAL ACCURACY(%)':[78.73,79.19,81.59,78.29,81.10]}<prepare_output>
abc = ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond','GarageYrBlt', 'GarageArea', 'GarageCars']
all_data.groupby('GarageType')[abc].count()
cross_val = pd.DataFrame(models)
 cross_val.head()<load_from_csv>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
test = pd.read_csv('.. /input/titanic/test.csv')
 test.head()<drop_column>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
passenger_id = test['PassengerId']
test = test.drop(columns = ['PassengerId','Age','SibSp','Parch','Ticket','Cabin','Embarked'])
 test.head()<count_missing_values>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )
test.isnull().sum()<count_missing_values>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
test.isnull().sum()<feature_engineering>
all_data['MSZoning'].value_counts()
test['Fare'] = test['Fare'].fillna(7.75 )<categorify>
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )
get_initials(test,'Name')
 test['Name'] = le.fit_transform(test['Name'])
 test['Sex'] = le.fit_transform(test['Sex'])
 test.head()<feature_engineering>
all_data['Utilities'].value_counts()
test = sc.fit_transform(test )<save_to_csv>
all_data = all_data.drop(['Utilities'], axis=1 )
prediction = classifier_knn.predict(test)
 submission = pd.DataFrame({'PassengerId':passenger_id,'Survived':prediction})
 submission.to_csv('TITANIC_SUBMISSION.csv',index = False)
 submission<set_options>
all_data["Functional"] = all_data["Functional"].fillna("Typ" )
%matplotlib inline
pd.options.display.float_format = '{:.2f}'.format<load_from_csv>
mode_col = ['Electrical','KitchenQual', 'Exterior1st', 'Exterior2nd', 'SaleType']
for col in mode_col:
all_data[col] = all_data[col].fillna(all_data[col].mode() [0] )
train = pd.read_csv('.. /input/titanic/train.csv')
 train.head()<count_missing_values>
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )
train.isnull().sum()<count_missing_values>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
train.isnull().sum()<count_values>
all_data['OverallCond'].value_counts()
sur = train['Survived'].value_counts()
survival_rate = [sur[0]/len(train)*100,sur[1]/len(train)*100]<feature_engineering>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
survived_male = len(train[(train['Sex'] == 'male')&(train['Survived'] == 1)])/len(train[train['Sex'] == 'male'])*100
survived_female = len(train[(train['Sex'] == 'female')&(train['Survived'] == 1)])/len(train[train['Sex'] == 'female'])*100
male = [survived_male,100-survived_male]
female = [survived_female,100-survived_female]<feature_engineering>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
survived_class_1 = len(train[(train['Pclass'] == 1)& train['Survived'] == 1])/len(train[(train['Pclass'] == 1)])*100
survived_class_2 = len(train[(train['Pclass'] == 2)& train['Survived'] == 1])/len(train[(train['Pclass'] == 2)])*100
survived_class_3 = len(train[(train['Pclass'] == 3)& train['Survived'] == 1])/len(train[(train['Pclass'] == 3)])*100
class_1 = [survived_class_1,100-survived_class_1]
class_2 = [survived_class_2,100-survived_class_2]
class_3 = [survived_class_3,100-survived_class_3]<feature_engineering>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
survived_S = len(train[(train['Embarked'] == 'S')&(train['Survived'] == 1)])/len(train[train['Embarked'] == 'S'])*100
survived_C = len(train[(train['Embarked'] == 'C')&(train['Survived'] == 1)])/len(train[train['Embarked'] == 'C'])*100
survived_Q = len(train[(train['Embarked'] == 'Q')&(train['Survived'] == 1)])/len(train[train['Embarked'] == 'Q'])*100
S = [survived_S,100-survived_S]
C = [survived_C,100-survived_C]
Q = [survived_Q,100-survived_Q]<feature_engineering>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam )
train['Age_Mode'] = train['Age'].fillna(value = 24)
 train['Age_Med'] = train['Age'].fillna(train['Age'].median())
 train['Age_Mean'] = train['Age'].fillna(train['Age'].mean() )<feature_engineering>
all_data = pd.get_dummies(all_data)
 all_data.shape
train['Age'] = train['Age'].fillna(train['Age'].median())
 train = train.drop(columns= ['Age_Mode','Age_Med','Age_Mean'])
 train.head()<create_dataframe>
train = all_data[:ntrain]
test = all_data[ntrain:]
train.shape
def get_age_group(dataframe,column_name):
dataframe[column_name] = dataframe[column_name].apply(np.ceil)
 age_group = {0:list(range(0,21)) ,1:list(range(21,41)) ,2:list(range(41,61)) ,3:list(range(61,81)) }
col = list(dataframe.columns)
 index = col.index(column_name)
 age = []
for j in range(len(dataframe)) :
for k in age_group.keys() :
for i in range(len(age_group[k])) :
if(age_group[k][i] == dataframe.iloc[j,index]):
age.append(k)
 dataframe['Age_Group'] = age<feature_engineering>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
def get_initials(dataframe,column_name):
sub = []
initials = ['Mrs.','Ms.','Mr.','Miss.','Master.','Lady.','Don.','Rev.','Dr.','Mme.','Major.','Sir.','Mlle.','Col.','Capt.','Countess.','Jonkheer.','Dona.']
name = dataframe[column_name]
for i in range(len(name)) :
split_names = name[i].split()
for j in range(len(split_names)) :
if(split_names[j] in initials):
sub.append(split_names[j])
 dataframe[column_name] = sub<count_values>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
get_initials(train,'Name')
 train['Name'].value_counts()<categorify>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
le = LabelEncoder()
train['Name'] = le.fit_transform(train['Name'])
 encoded_values = train['Name'].unique()
decoded_values = le.inverse_transform(encoded_values)
 initials = {}
for i in range(len(encoded_values)) :
initials.setdefault(decoded_values[i],encoded_values[i] )<categorify>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
score = rmsle_cv(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
train['Name'] = le.fit_transform(train['Name'])
 train['Sex'] = le.fit_transform(train['Sex'])
 embarked = {0:'S',1:'C',2:'Q'}
train['Embarked'] = train['Embarked'].fillna('Q')
 train['Embarked'] = le.fit_transform(train['Embarked'] )<drop_column>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
train = train.drop(columns = ['PassengerId','Ticket','Cabin','Age_Group'])
 train.head()<import_modules>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2<prepare_x_and_y>
LassoMd = lasso.fit(train.values,y_train)
 ENetMd = ENet.fit(train.values,y_train)
 KRRMd = KRR.fit(train.values,y_train)
 GBoostMd = GBoost.fit(train.values,y_train )
features = train.iloc[:,1:]
target = train.iloc[:,0]<train_model>
finalMd =(np.expm1(LassoMd.predict(test.values)) + np.expm1(ENetMd.predict(test.values)) + np.expm1(KRRMd.predict(test.values)) + np.expm1(GBoostMd.predict(test.values)))/ 4
finalMd
<drop_column><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = finalMd
sub.to_csv('submission.csv',index=False )