Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<groupby>
from optuna.integration import lightgbm as lgb
df_ticket = df_full.groupby('Ticket')['Name'].count()
df_full['TicketSize'] = df_full['Ticket'].map(df_ticket )<categorify>
train=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test=pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 train.head()
df_full['TicketSizeBin'] = df_full['TicketSizeBin'].map({'alone': 0, 'normal': 1, 'big': 2} )<data_type_conversions>
cols =('MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities',
'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st',
'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual',
'Functional', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual',
'GarageCond', 'PavedDrive', 'PoolQC', 'Fence', 'MiscFeature',
'SaleType', 'SaleCondition')
 cols_defaults =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(train[c].values))
train[c] = lbl.transform(list(train[c].values))
print('Shape train: {}'.format(train.shape))
df_full['IsAloneT'] =(df_full['TicketSizeBin'] == 0 ).astype(int)
 catgroup(df_full.loc[mask_train], 'IsAloneT', 'Survived' )<feature_engineering>
for c in cols:
lbl = LabelEncoder()
lbl.fit(list(test[c].values))
test[c] = lbl.transform(list(test[c].values))
print('Shape test: {}'.format(test.shape))
fare_scaler = 'TicketSize'
df_full['Fare'] = df_full['Fare'] / df_full[fare_scaler]<filter>
train_test=pd.concat([train,test],axis=0,sort=False)
 train_test.head()
mask_zerofare = df_full['Fare'] == 0
df_full.loc[mask_zerofare]<data_type_conversions>
train.isnull().sum()
df_full['Fare'].fillna(df_full['Fare'].median() , inplace=True )<feature_engineering>
train_test.groupby('SaleCondition' ).count() ['Id']
df_full['FareOrig'] = df_full['Fare'] * df_full[fare_scaler]<data_type_conversions>
corr_dict=correlation_train['SalePrice'].sort_values(ascending=False ).to_dict()
unimportant_columns=[]
for key,value in corr_dict.items() :
if(( value>=-0.1)&(value<=0)) |(value==0.1):
unimportant_columns.append(key)
 unimportant_columns
df_full['FareBin'] = pd.qcut(df_full['Fare'], 4, labels=False ).astype(int )<groupby>
train2= train.drop(['BsmtFinSF2',
'Utilities',
'BsmtHalfBath',
'MiscVal',
'Id',
'LowQualFinSF',
'YrSold',
'SaleType',
'LotConfig',
'OverallCond',
'MSSubClass',
'BldgType',
'Heating'],axis=1 )
df_familygroup = df_full.loc[mask_train].groupby(['LastName', 'FareOrig'], as_index=False)['Survived', 'Name'].agg({'Survived': 'mean', 'Name': 'count'})
 df_familygroup = df_familygroup.groupby('Name', as_index=False)['Survived'].mean()
df_familygroup = df_familygroup.sort_values(by='Survived')
 df_familygroup<groupby>
train3= train.loc[:,['MSZoning', 'LotFrontage', 'LotArea', 'Street', 'Alley', 'LotShape',
'LandContour', 'LandSlope',
'Condition1', 'Condition2',
'HouseStyle', 'OverallQual', 'YearBuilt', 'YearRemodAdd', 'RoofStyle',
'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'MasVnrArea',
'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual', 'BsmtCond',
'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1', 'BsmtFinType2',
'BsmtUnfSF', 'TotalBsmtSF', 'HeatingQC', 'CentralAir', 'Electrical',
'2ndFlrSF', 'GrLivArea', 'BsmtFullBath', 'FullBath',
'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'KitchenQual',
'Functional', 'Fireplaces', 'FireplaceQu', 'GarageType',
'GarageFinish',
'GarageArea', 'GarageQual',
'GarageCond', 'PavedDrive', 'WoodDeckSF', 'OpenPorchSF',
'EnclosedPorch', '3SsnPorch', 'ScreenPorch',
'PoolQC',
'Fence', 'MiscFeature',
'SaleCondition', 'SalePrice']]
df_familygroup = df_full.groupby(['LastName', 'FareOrig'])['Name'].count()
df_full['NameFareSize'] = df_full[['LastName', 'FareOrig']].apply(lambda row: df_familygroup[(row[0], row[1])], axis=1 )<groupby>
corr_dict3=correlation_train3['SalePrice'].sort_values(ascending=False ).to_dict()
unimportant_columns_train3=[]
for key,value in corr_dict3.items() :
if(( value>=-0.1)&(value<=0)) |(value==0.1):
unimportant_columns_train3.append(key)
 unimportant_columns_train3
df_full['Group'] = ''
df_groups = df_full.groupby(['LastName', 'FareOrig'])
 for group, df_group in df_groups:
for idx, row in df_group.iterrows() :
group_members = df_group.drop(idx)['PassengerId'].tolist()
df_full.at[idx, 'Group'] = group_members<categorify>
%matplotlib inline
df_groups = df_full.groupby('Ticket')
 for group, df_group in df_groups:
for idx, row in df_group.iterrows() :
group_members = df_group.drop(idx)['PassengerId'].tolist()
df_full.at[idx, 'Group'].extend(group_members)
 df_full['Group'] = df_full['Group'].map(set )<feature_engineering>
test= test.loc[:,['MSZoning', 'LotFrontage', 'LotArea', 'Street', 'Alley', 'LotShape',
'LandContour', 'LandSlope',
'Condition1', 'Condition2',
'HouseStyle', 'OverallQual', 'YearBuilt', 'YearRemodAdd', 'RoofStyle',
'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'MasVnrArea',
'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual', 'BsmtCond',
'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1', 'BsmtFinType2',
'BsmtUnfSF', 'TotalBsmtSF', 'HeatingQC', 'CentralAir', 'Electrical',
'2ndFlrSF', 'GrLivArea', 'BsmtFullBath', 'FullBath',
'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'KitchenQual',
'Functional', 'Fireplaces', 'FireplaceQu', 'GarageType',
'GarageFinish',
'GarageArea', 'GarageQual',
'GarageCond', 'PavedDrive', 'WoodDeckSF', 'OpenPorchSF',
'EnclosedPorch', '3SsnPorch', 'ScreenPorch',
'PoolQC',
'Fence', 'MiscFeature',
'SaleCondition'
]]
def group_survived(group):
mask_group = df_full['PassengerId'].isin(group)
 s = df_full.loc[mask_group, 'Survived'].max()
return s if pd.notnull(s)else 0.5
df_full['GroupSurvived'] = df_full['Group'].apply(group_survived )<feature_engineering>
X_train = train3.drop([ 'SalePrice'], axis=1)
 y_train = np.log1p(train3['SalePrice'] )
df_full['GroupSize'] = df_full['Group'].str.len() + 1<groupby>
y_preds = []
models = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=0)
 params = {
'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'boosting_type': 'gbdt',
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.loc[train_index, :]
X_val = X_train.loc[valid_index, :]
y_tr = y_train[train_index]
y_val = y_train[valid_index]
lgb_train = lgb.Dataset(X_tr,
y_tr, 
)
 lgb_eval = lgb.Dataset(X_val,
y_val,
reference=lgb_train, 
)
 model = lgb.train(params,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 oof_train[valid_index] = model.predict(X_val,
num_iteration=model.best_iteration)
 y_pred = model.predict(test,
num_iteration=model.best_iteration)
 y_preds.append(y_pred)
 models.append(model )
df_fullgroup = df_full.loc[mask_train].groupby('GroupSize', as_index=False)['Survived'].mean()
df_fullgroup = df_fullgroup.sort_values(by='Survived')
 df_fullgroup<drop_column>
print(f'CV: {np.sqrt(mean_squared_error(y_train, oof_train)) }' )
list_drop_features = [name for name, include in used_features.items() if not include]
df_full.drop(columns=list_drop_features, inplace=True )<normalization>
sub = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 y_sub = sum(y_preds)/ len(y_preds)
 y_sub = np.expm1(y_sub)
 sub['SalePrice'] = y_sub
sub.to_csv('submission.csv', index=False)
 sub.head()