Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<sort_values>
train_test_split,
cross_val_score 
)
idx = np.random.permutation(range(n)) [:200]
idx.sort()
small_train_df = train_df.iloc[idx[:100]]
small_test_df = train_df.iloc[idx[100:]]
small_cont_vars = ['YearBuilt', '1stFlrSF', '2ndFlrSF']
small_cat_vars = ['Neighborhood', 'Exterior1st']
small_train_df = small_train_df[small_cat_vars + small_cont_vars + ['SalePrice']]
small_test_df = small_test_df[small_cat_vars + small_cont_vars + ['SalePrice']]<categorify>
train = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv")
 test = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 sample_submission = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sample_submission.csv")
 items = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 item_categories = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/item_categories.csv")
 shops = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/shops.csv" )
categorify = Categorify(small_cat_vars, small_cont_vars)
 categorify(small_train_df)
 categorify(small_test_df, test=True )<correct_missing_values>
train['date']=train['date'].apply(lambda x:datetime.datetime.strptime(x, '%d.%m.%Y'))
fill_missing = FillMissing(small_cat_vars, small_cont_vars)
 fill_missing(small_train_df)
 fill_missing(small_test_df, test=True )<load_from_csv>
train = train.join(items, on='item_id', rsuffix='_' ).drop(['item_id_', 'item_name'], axis=1 )
train_df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'test.csv' )<feature_engineering>
train[train['item_price']<=0]
train_df['set'] = 'train'
test_df['set'] = 'test'<define_variables>
train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)]
procs=[FillMissing, Categorify, Normalize]<define_variables>
train.loc[train.item_price<0,'item_price'] = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)].item_price.mean()
cat_vars = ['MSSubClass', 'MSZoning', 'Street', 'Alley', 'LandContour', 'Utilities', 'LotConfig', 'Neighborhood', 'Condition1',
'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType',
'Foundation', 'Heating', 'Electrical', 'GarageType', 'GarageFinish', 'PavedDrive', 'Fence', 'MiscFeature',
'SaleType', 'SaleCondition','CentralAir'] + ['ExterCond', 'ExterQual','BsmtCond', 'BsmtQual', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2','KitchenQual','Functional', 'FireplaceQu', 'GarageQual', 'GarageCond', 'PoolQC','HeatingQC','LotShape',]
cont_vars = ['LotFrontage', 'LotArea', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea',
'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'TotRmsAbvGrd', 'Fireplaces',
'GarageYrBlt','MiscVal', 'MoSold', 'YrSold',
'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
] 
<concatenate>
train_monthly = train.sort_values('date' ).groupby(['date_block_num', 'shop_id','item_category_id', 'item_id'], as_index=False )
df = pd.concat([train_df,test_df])
 df.head()
print(len(df))<categorify>
train_monthly = train_monthly.agg({'item_price':['median', 'mean'], 'item_cnt_day':['sum', 'count']} )
fill_missing = FillMissing(cat_vars, cont_vars)
 fill_missing(df)
 fill_missing(df, test=True)
 categorify = Categorify(cat_vars, cont_vars)
 categorify(df)
 categorify(df, test=True)
 normalize = Normalize(cat_vars, cont_vars)
 normalize(df)
 normalize(df, test=True )<filter>
train_monthly.columns = ['date_block_num', 'shop_id', 'item_category_id','item_id', 'item_price_median', 'item_price_mean', 'item_cnt', 'transactions']
dep_var = 'SalePrice'
train_df = df.loc[df.set == 'train']
test_df = df.loc[df.set == 'test']<define_variables>
train_monthly_by_category = train_monthly.groupby(['date_block_num','shop_id', 'item_category_id'], as_index=False )
max_log_y = np.log(np.max(train_df['SalePrice'])*1.2)
 y_range = torch.tensor([0, max_log_y], device=defaults.device )<compute_test_metric>
train_monthly_by_category = train_monthly_by_category.agg({'item_price_median':['mean'], 'item_price_mean':['mean'],'item_cnt':['sum', 'mean'], 'transactions': ['mean']} )
def rmsle(inp,targ):
"Mean squared error between `inp` and `targ`."
return torch.sqrt(msle(inp, targ))<choose_model_class>
train_cat_no_shop = train_monthly.groupby(['date_block_num', 'item_category_id'], as_index=False )
learn = tabular_learner(data, layers=[1000,500], ps=[0.02,0.04], emb_drop=0.05,
y_range=y_range, metrics=rmsle, loss_func=rmsle)
 learn.model_dir = '/kaggle/working/'<find_best_params>
train_cat_no_shop = train_cat_no_shop.agg({'item_price_median':['mean'], 'item_price_mean':['mean'],'item_cnt':['sum', 'mean'], 'transactions': ['mean']} )
learn.model<find_best_params>
train_monthly['year'] = train_monthly['date_block_num'].apply(lambda x:(( x//12+2013)))
 train_monthly['month'] = train_monthly['date_block_num'].apply(lambda x:(x%12+1))
learn.lr_find()<train_model>
train_monthly_by_category['year'] = train_monthly_by_category['date_block_num'].apply(lambda x:(( x//12+2013)))
 train_monthly_by_category['month'] = train_monthly_by_category['date_block_num'].apply(lambda x:(x%12+1))
learn.fit_one_cycle(50, 1e-2, wd=0.5 )<save_model>
train_cat_no_shop['year'] = train_cat_no_shop['date_block_num'].apply(lambda x:(( x//12+2013)))
 train_cat_no_shop['month'] = train_cat_no_shop['date_block_num'].apply(lambda x:(x%12+1))
learn.save('lr 1e-1' )<create_dataframe>
train_monthly_by_category.columns = ['date_block_num', 'shop_id','item_category_id','item_price_median', 'item_price_mean', 'item_cnt_sum', 'item_cnt', 'transactions', 'year', 'month']
train_cat_no_shop.columns = ['date_block_num', 'item_category_id','item_price_median', 'item_price_mean', 'item_cnt_sum', 'item_cnt', 'transactions', 'year', 'month']
test_data =(TabularList.from_df(test_df, path=path, cat_names=cat_vars, cont_names=cont_vars, procs=data.processor))
print(type(test_data[0]))
print(test_data[0]['Id'] )<predict_on_test>
train_monthly[train_monthly['item_cnt']>1000]
test_data =(TabularList.from_df(test_df, path=path, cat_names=cat_vars, cont_names=cont_vars, procs=data.processor))
results = []
ids = []
for i in range(len(test_data)) :
print(i)
 ids.append(test_data[i]['Id'])
 pred = learn.predict(test_data[i])
 pred = torch.exp(pred[1])
 results.append(float(pred))
print(results)
<save_to_csv>
uniq_pairs = train_monthly.groupby(['shop_id','item_id'] ).size().reset_index()
uniq_pairs.shape
submission = pd.DataFrame({'Id': ids, 'SalePrice':results})
 print(submission)
 submission.to_csv("submission.csv", index=False )<set_options>
empty_df = pd.DataFrame(index = uniq_pairs.index, columns = ['date_block_num','shop_id','item_id'])
 empty_df[['shop_id', 'item_id']] = uniq_pairs[['shop_id','item_id']]
empty_df_2 = pd.DataFrame(columns = ['date_block_num','shop_id','item_id'])
 for i in range(34):
empty_df_1 = empty_df.copy()
empty_df_1['date_block_num'] = i
empty_df_2 = pd.concat([empty_df_2, empty_df_1] )
pd.options.display.max_rows = 1000
pd.options.display.max_columns = 20<import_modules>
full_train_monthly = empty_df_2.merge(train_monthly,on=['date_block_num','shop_id', 'item_id'], how='left' ).fillna(0 ).drop(['index'], axis=1 )
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import skew
from sklearn.model_selection import train_test_split, KFold
from sklearn.linear_model import ElasticNet
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.metrics import mean_squared_error<load_from_csv>
full_train_monthly['year'] = full_train_monthly['date_block_num'].apply(lambda x:(( x//12+2013)))
 full_train_monthly['month'] = full_train_monthly['date_block_num'].apply(lambda x:(x%12+1))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<define_variables>
full_train_monthly = full_train_monthly.join(items, on='item_id', rsuffix='_' ).drop(['item_id_', 'item_name', 'item_category_id'], axis=1)
 full_train_monthly = full_train_monthly.rename(columns={'item_category_id_':'item_category_id'} )
features = ['GrLivArea','OverallQual','2ndFlrSF','TotalBsmtSF','1stFlrSF','GarageCars','YearBuilt', 'GarageArea','ExterQual',
'YearRemodAdd','FireplaceQu','GarageYrBlt','FullBath','MSSubClass','LotArea','Fireplaces','KitchenQual','MSZoning',
'GarageType','BsmtFinSF1','Neighborhood','BsmtQual','TotRmsAbvGrd','HalfBath','BldgType','GarageFinish',
'Foundation','BedroomAbvGr','HouseStyle','CentralAir','OpenPorchSF','HeatingQC','BsmtFinType1','BsmtUnfSF',
'GarageCond','GarageQual','KitchenAbvGr','OverallCond','BsmtCond','MasVnrArea','BsmtFullBath','Exterior1st',
'Exterior2nd','PavedDrive','WoodDeckSF','LandContour','MasVnrType','BsmtFinType2','Functional','Fence']<feature_engineering>
uniq_pairs_cat = train_monthly_by_category.groupby(['shop_id','item_category_id'] ).size().reset_index()
train = train[features+['SalePrice']]
test = test[['Id']+features]
all_data = pd.concat(( train.loc[:,'GrLivArea':'Fence'],
test.loc[:,'GrLivArea':'Fence']))
all_data['SalePrice'] = train['SalePrice']<define_variables>
empty_df = pd.DataFrame(index = uniq_pairs_cat.index, columns = ['date_block_num','shop_id','item_category_id'])
 empty_df[['shop_id', 'item_category_id']] = uniq_pairs_cat[['shop_id','item_category_id']]
empty_df_2 = pd.DataFrame(columns = ['date_block_num','shop_id','item_category_id'])
 for i in range(34):
empty_df_1 = empty_df.copy()
empty_df_1['date_block_num'] = i
empty_df_2 = pd.concat([empty_df_2, empty_df_1] )
obj_list = []
for i in all_data.columns :
if all_data[i].dtypes == 'object':
obj_list.append(i )<define_variables>
full_train_monthly_by_category = empty_df_2.merge(train_monthly_by_category,
on=['date_block_num','shop_id', 'item_category_id'],how='left' ).fillna(0 ).drop(['index'], axis=1 )
f_drop_num = [
'BedroomAbvGr','OverallCond','MSSubClass','KitchenAbvGr'
]<define_variables>
empty_df = pd.DataFrame(index = uniq_cat, columns = ['date_block_num','item_category_id'])
 empty_df['item_category_id'] = uniq_cat
empty_df_2 = pd.DataFrame(columns = ['date_block_num','item_category_id'])
 for i in range(34):
empty_df_1 = empty_df.copy()
empty_df_1['date_block_num'] = i
empty_df_2 = pd.concat([empty_df_2, empty_df_1] )
print(*[c for c in train.select_dtypes(include=['object'] ).columns if c in basement], sep=", ")
 basement_categorical = [c for c in train.select_dtypes(include=['object'] ).columns if c in basement]<data_type_conversions>
full_train_cat_no_shop = empty_df_2.merge(train_cat_no_shop,
on=['date_block_num', 'item_category_id'],how='left' ).fillna(0 ).drop(['index'], axis=1 )
f_categorical = [f for f in all_data.columns if all_data.dtypes[f] == 'object']
for c in f_categorical:
all_data[c] = all_data[c].astype('category')
 if all_data[c].isnull().any() :
all_data[c] = all_data[c].cat.add_categories(['MISSING'])
 all_data[c] = all_data[c].fillna('MISSING' )<data_type_conversions>
full_train_monthly_by_category['year'] = full_train_monthly_by_category['date_block_num'].apply(lambda x:(( x//12+2013)))
 full_train_monthly_by_category['month'] = full_train_monthly_by_category['date_block_num'].apply(lambda x:(x%12+1))
full_train_cat_no_shop['year'] = full_train_cat_no_shop['date_block_num'].apply(lambda x:(( x//12+2013)))
 full_train_cat_no_shop['month'] = full_train_cat_no_shop['date_block_num'].apply(lambda x:(x%12+1))
def make_MISSING(data):
f_categorical = [f for f in data.columns if data.dtypes[f] == 'object']
for c in f_categorical:
data[c] = data[c].astype('category')
 if data[c].isnull().any() :
data[c] = data[c].cat.add_categories(['MISSING'])
 data[c] = data[c].fillna('MISSING' )<categorify>
full_train_monthly['item_cnt_next_month'] = full_train_monthly.sort_values('date_block_num' ).groupby(['shop_id','item_id'])['item_cnt'].shift(-1 )
BsmtQual_map = {"Ex":3, "Gd":2, "TA":2, "Fa":1, "MISSING":1}
BsmtFinType1_map = {"GLQ":2, "ALQ":2, "Unf":2,"Rec":1, "LwQ":1, "BLQ":1, "MISSING":1}
BsmtCond_map = {"Gd":3, "TA":3, "MISSING":2, "Fa":2, "Po":1}
BsmtFinType2_map = {"Unf":4, "ALQ":4, "MISSING":3, "Rec":3, "BLQ":2, "LwQ":2, "GLQ":1 }
all_data["BsmtQual"] = all_data.BsmtQual.replace(BsmtQual_map)
 all_data["BsmtFinType1"] = all_data.BsmtFinType1.replace(BsmtFinType1_map)
 all_data["BsmtCond"] = all_data.BsmtCond.replace(BsmtCond_map)
 all_data["BsmtFinType2"] = all_data.BsmtFinType2.replace(BsmtFinType2_map )<set_options>
full_train_monthly_by_category['item_cnt_next_month'] = full_train_monthly_by_category.sort_values('date_block_num' ).groupby(['shop_id', 'item_category_id'])['item_cnt'].shift(-1 )
pd.options.display.max_rows = 1000
pd.options.display.max_columns = 25<define_variables>
full_train_cat_no_shop['item_cnt_next_month'] = full_train_cat_no_shop.sort_values('date_block_num' ).groupby(['item_category_id'])['item_cnt'].shift(-1 )
ExterQual_map = {"Gd":2, "TA":2, "Ex":3, "Fa":1}
FireplaceQu_map = {"MISSING":2, "TA":3, "Gd":3,"Fa":1, "Ex":4, "Po":1}
KitchenQual_map = {"Gd":1, "TA":2, "MISSING":5, "Fa":4, "Ex":3}
MSZoning_map = {"RL":1, "RM":1, "C（all）":2, "FV":3, "RH":3, "MISSING":4}
GarageType_map = {"Attchd":1, "Detchd":1, "BuiltIn":2, "CarPort":3, "MISSING":1, "Basment":4, "2Types":4}
Neighborhood_map = {}
BldgType_map = {"1Fam":1, "2fmCon":2, "Duplex":2, "TwnhsE":1, "Twnhs":2}
GarageFinish_map = {"RFn":1, "Unf":2, "Fin":1,"MISSING":2}
Foundation_map = {"PConc":1, "CBlock":1, "BrkTil":2, "Wood":3, "Slab":2, "Stone":3}
HouseStyle_map = {"2Story":1, "1Story":1, "1.5Fin":2,"1.5Unf":3, "SFoyer":4, "SLvl":2, "2.5Unf":4, "2.5Fin":5}
CentralAir_map = {"Y":1, "N":2}
HeatingQC_map = {"Ex":1, "Gd":1, "TA":1,"Fa":2, "Po":3}
GarageCond_map = {"TA":1, "Fa":2, "MISSING":1,"Gd":2, "Po":2, "Ex":3}
GarageQual_map = {"TA":1, "Fa":2, "Gd":3,"MISSING":1, "Ex":4, "Po":5}
Exterior1st_map = {}
Exterior2nd_map = {}
PavedDrive_map = {"Y":1, "N":2, "P":2}
LandContour_map = {"Lvl":1, "Bnk":2, "Low":3,"HLS":1}
MasVnrType_map = {"BrkFace":1, "None":1, "Stone":3,"BrkCmn":2, "MISSING":3}
Functional_map = {"Typ":1, "Min1":4, "Maj1":2,"Min2":1, "Mod":1, "Maj2":2, "Sev":3, "MISSING":2}
Fence_map = {"MISSING":1, "MnPrv":1, "GdWo":2,"GdPrv":2, "MnWw":3}
all_data["ExterQual"] = all_data.ExterQual.replace(ExterQual_map)
 all_data["FireplaceQu"] = all_data.FireplaceQu.replace(FireplaceQu_map)
 all_data["KitchenQual"] = all_data.KitchenQual.replace(KitchenQual_map)
 all_data["MSZoning"] = all_data.MSZoning.replace(MSZoning_map)
 all_data["GarageType"] = all_data.GarageType.replace(GarageType_map)
 all_data["BldgType"] = all_data.BldgType.replace(BldgType_map)
 all_data["GarageFinish"] = all_data.GarageFinish.replace(GarageFinish_map)
 all_data["Foundation"] = all_data.Foundation.replace(Foundation_map)
 all_data["HouseStyle"] = all_data.HouseStyle.replace(HouseStyle_map)
 all_data["CentralAir"] = all_data.CentralAir.replace(CentralAir_map)
 all_data["HeatingQC"] = all_data.HeatingQC.replace(HeatingQC_map)
 all_data["GarageCond"] = all_data.GarageCond.replace(GarageCond_map)
 all_data["GarageQual"] = all_data.GarageQual.replace(GarageQual_map)
 all_data["PavedDrive"] = all_data.PavedDrive.replace(PavedDrive_map)
 all_data["LandContour"] = all_data.LandContour.replace(LandContour_map)
 all_data["MasVnrType"] = all_data.MasVnrType.replace(MasVnrType_map)
 all_data["Functional"] = all_data.Functional.replace(Functional_map)
 all_data["Fence"] = all_data.Fence.replace(Fence_map )<categorify>
full_train_monthly = pd.concat(
 [full_train_monthly, pd.get_dummies(full_train_monthly['year'], prefix='year')],axis=1 
)
 full_train_monthly = pd.concat(
 [full_train_monthly, pd.get_dummies(full_train_monthly['month'], prefix='month')],axis=1
)
Neighborhood_map = {
'CollgCr':6, 'Veenker':4, 'Crawfor':6, 'NoRidge':1, 'Mitchel':5,
'Blmngtn':5, 'Blueste':5, 'BrDale':1, 'ClearCr':7, 'Edwards':2,
'Gilbert':5, 'IDOTRR':5, 'MeadowV':5, 'NAmes':5, 'NPkVill':5,
'NWAmes':0, 'NridgHt':4, 'OldTown':0, 'SWISU':5, 'Sawyer':0,
'SawyerW':4, 'Somerst':0, 'StoneBr':3, 'Timber':4, 'BrkSide':5,
}
all_data["Neighborhood"] = all_data.Neighborhood.replace(Neighborhood_map )<categorify>
full_train_monthly_by_category = pd.concat(
 [full_train_monthly_by_category, pd.get_dummies(full_train_monthly_by_category['year'], prefix='year')],axis=1 
)
 full_train_monthly_by_category = pd.concat(
 [full_train_monthly_by_category, pd.get_dummies(full_train_monthly_by_category['month'], prefix='month')],axis=1
)
Exterior1st_map = {
'AsbShng':7, 'AsphShn':6, 'BrkComm':8, 'BrkFace':7, 'CBlock':6,
'CemntBd':5, 'HdBoard':3, 'ImStucc':0, 'MetalSd':4, 'Plywood':3,
'Stone':2, 'Stucco':4, 'VinylSd':3, 'Wd Sdng':3, 'WdShing':3,
'MISSING':1,
}
all_data["Exterior1st"] = all_data.Exterior1st.replace(Exterior1st_map )<categorify>
full_train_cat_no_shop = pd.concat(
 [full_train_cat_no_shop, pd.get_dummies(full_train_cat_no_shop['year'], prefix='year')],axis=1 
)
 full_train_cat_no_shop = pd.concat(
 [full_train_cat_no_shop, pd.get_dummies(full_train_cat_no_shop['month'], prefix='month')],axis=1
)
Exterior2nd_map = {
'AsbShng':7, 'AsphShn':8, 'Brk Cmm':7, 'BrkFace':7, 'CBlock':6,
'CmentBd':5, 'HdBoard':4, 'ImStucc':5, 'MetalSd':4, 'Other':0, 'Plywood':4,
'Stone':3, 'Stucco':3, 'VinylSd':2, 'Wd Sdng':2, 'Wd Shng':2,
'MISSING':1,
}
all_data["Exterior2nd"] = all_data.Exterior2nd.replace(Exterior2nd_map )<filter>
t = test.copy()
t = t.join(items, on='item_id', rsuffix='_' ).drop(['item_id_', 'item_name'], axis=1 )
MISSING_KitchenQual = all_data.loc[all_data['KitchenQual'] == 5]
MISSING_Exterior1st_0 = all_data.loc[all_data['Exterior1st'] == 0]
MISSING_Exterior1st_1 = all_data.loc[all_data['Exterior1st'] == 1]
MISSING_Exterior2nd_0 = all_data.loc[all_data['Exterior2nd'] == 0]
MISSING_Exterior2nd_1 = all_data.loc[all_data['Exterior2nd'] == 1]
MISSING_KitchenQual
MISSING_Exterior1st_0
MISSING_Exterior1st_1
MISSING_Exterior2nd_0
MISSING_Exterior2nd_1<drop_column>
t_train_cat = full_train_monthly_by_category.copy()
all_data.drop([95, 1187, 691, 595, 691], inplace=True)
 test['Id'].drop([95, 1187, 691, 595, 691], inplace=True )<feature_engineering>
%%time
m_test = pd.merge(t, t_train[t_train.date_block_num==33], how = 'left', on=['shop_id', 'item_id'])
 m_test = m_test.rename(columns={'item_category_id_x':'item_category_id'})
 m_test = m_test.drop('item_category_id_y', axis=1 )
train['SalePrice'] = np.log1p(train['SalePrice'])
 numeric_feats = train.dtypes[train.dtypes != "object"].index
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )<data_type_conversions>
%%time
null_test = pd.merge(m_test[m_test['item_cnt'].isnull() ][['ID', 'shop_id', 'item_id','item_category_id']],t_train_cat[t_train_cat.date_block_num==33],how = 'left',on = ['shop_id', 'item_category_id'])
 null_test.index = null_test['ID']
for i in m_test.columns:
m_test.loc[m_test.ID.isin(null_test.ID),i] = null_test[i]
for feature in all_data.select_dtypes(exclude=['object', 'category'] ).columns:
all_data[feature].fillna(all_data[feature].median() , inplace = True)
<drop_column>
all_data.drop('SalePrice', axis=1, inplace=True)
<categorify>
m_test.loc[m_test['item_cnt'].isnull() ,'year'] = 2015
m_test.loc[m_test['item_cnt'].isnull() ,'month'] = 10
m_test.loc[m_test['item_cnt'].isnull() ,'month_10'] = 1
m_test.loc[m_test['item_cnt'].isnull() ,'date_block_num'] = 33
all_data = pd.get_dummies(all_data)
 print(all_data.shape )<prepare_x_and_y>
m_test = m_test.fillna(0 )
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice<split>
drop_cols = ['date_block_num',
'ID',
'shop_id',
'item_id',
'item_price_median',
'year',
'month',
'item_cnt_next_month',
'item_category_id']
X_test = m_test.drop(drop_cols, axis=1 )
train_x, test_x, train_y, test_y = train_test_split(X_train, y, test_size=0.3, random_state=0 )<choose_model_class>
X_test.to_csv('X_test.csv', index=False )
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=0.5, random_state=3))<train_model>
ENet.fit(train_x, train_y)
 ENet.score(test_x,test_y)*100<train_model>
ENet.fit(X_train.values, y)
 ENet.score(test_x,test_y)*100<predict_on_test>
%%time
train_set = t_train.query('date_block_num>=6 and date_block_num <33' ).copy()
train_set_cat = t_train_cat.query('date_block_num>=6 and date_block_num <33' ).copy()
def predict_cv(model, train_x, train_y, test_x):
preds = []
preds_test = []
va_indexes = []
kf = KFold(n_splits=4, shuffle=True, random_state=6785)
 for i,(tr_idx, va_idx)in enumerate(kf.split(train_x)) :
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
model.fit(tr_x, tr_y)
 tr_pred = model.predict(tr_x)
 pred = model.predict(va_x)
 preds.append(pred)
 pred_test = model.predict(test_x)
 preds_test.append(pred_test)
 va_indexes.append(va_idx)
 print(' score Train : {:.6f}'.format(np.sqrt(mean_squared_error(tr_y, tr_pred))),
' score Valid : {:.6f}'.format(np.sqrt(mean_squared_error(va_y, pred))),
' Correct answer rate Train: {:.6f}'.format(model.score(tr_x, tr_y)) ,
' Correct answer rate Valid: {:.6f}'.format(model.score(va_x, va_y)) , 
)
 va_indexes = np.concatenate(va_indexes)
 preds = np.concatenate(preds, axis=0)
 order = np.argsort(va_indexes)
 pred_train = pd.DataFrame(preds[order])
 preds_test = pd.DataFrame(np.mean(preds_test, axis=0))
print('Score : {:.6f}'.format(np.sqrt(mean_squared_error(train_y, pred_train))))
return pred_train, preds_test, model<predict_on_test>
for_fit = np.setxor1d(np.arange(rand_n), for_val)
 print(for_fit, for_fit.shape )
predict_cv(ENet, train_x, train_y, test_x )<train_model>
x1 = train_set[train_set['date_block_num']==6].reset_index().iloc[for_val]
for i in range(7,33):
x = train_set[train_set['date_block_num']==i].reset_index().iloc[for_val]
x1 = x1.append(x)
 val_data = x1.copy()
ENet.fit(X_train.values, y)
 ENet.score(test_x,test_y)*100<predict_on_test>
x2 = train_set[train_set['date_block_num']==6].reset_index().iloc[for_fit]
for i in range(7,33):
xx = train_set[train_set['date_block_num']==i].reset_index().iloc[for_fit]
x2 = x2.append(xx)
 fit_data = x2.copy()
ENet_pred = np.expm1(ENet.predict(X_test.values))<save_to_csv>
%%time
drop_cols = ['date_block_num', 'shop_id', 'item_id', 'item_price_median', 'year', 'month', 'item_cnt_next_month','item_category_id', 'index']
X_train = fit_data.drop(drop_cols, axis=1)
 Y_train = fit_data['item_cnt_next_month']
X_val = val_data.drop(drop_cols, axis=1)
 Y_val = val_data['item_cnt_next_month']
sub = pd.DataFrame()
sub['Id'] = test['Id']
sub['SalePrice'] = ENet_pred
sub.to_csv('submission.csv',index=False )<import_modules>
def LGBReg(x_train, y_train, x_val, y_val):
lgb_reg = LGBMRegressor(
 n_jobs=-1,
tree_method='auto',
learning_rate=0.02,
max_depth=8,
n_estimators=1000,
colsample_bytree=0.8,
subsample=0.8,
seed=42)
 lgb_reg.fit(
 x_train,
y_train,
eval_metric="rmse",
eval_set=[(x_train, y_train),(x_val, y_val)],
verbose=10,
early_stopping_rounds = 10)
 return lgb_reg
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
import sklearn.metrics as metrics
import math 
<set_options>
%%time
lgb_reg_1 = LGBReg(X_train, Y_train, X_val, Y_val )
warnings.filterwarnings('ignore' )<load_from_csv>
lgb_test_pred = lgb_reg_1.predict(X_test ).clip(0, 20 )
sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 c_test = test.copy()
c_train = train.copy()<concatenate>
submission37 = pd.DataFrame(test['ID'])
 submission37['item_cnt_month'] = lgb_test_pred
submission37.to_csv('submission37.csv', index=False )
c_train['train'] = 1
c_test['train'] = 0
df = pd.concat([c_train, c_test], axis=0,sort=False)
<create_dataframe>
submission38 = pd.DataFrame(test['ID'])
 submission38['item_cnt_month'] = lgb_test_pred.round()
submission38.to_csv('submission38.csv', index=False )
Null_val = [(c, df[c].isna().mean() *100)for c in df]
Null_val = pd.DataFrame(Null_val, columns=["column_name", "percentage"] )<sort_values>
from statsmodels.regression.linear_model import OLS
import statsmodels.api as sm
Null_val = Null_val[Null_val.percentage > 50]
Null_val.sort_values("percentage", ascending=False)
<drop_column>
x = sm.add_constant(X_train.fillna(0))
df = df.drop(['Alley','PoolQC','Fence','MiscFeature'],axis=1)
<define_variables>
model2 = OLS(
 Y_train,
x
).fit()
print(model2.summary() )
object_columns_df = df.select_dtypes(include=['object'])
 numerical_columns_df =df.select_dtypes(exclude=['object'])
<count_missing_values>
X_test['const'] = 1
null_counts = object_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))<data_type_conversions>
y_predict_OLS = model2.predict(X_test ).clip(0, 20 )
columns_None = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','FireplaceQu','GarageCond']
object_columns_df[columns_None]= object_columns_df[columns_None].fillna('None')
<data_type_conversions>
submission39 = pd.DataFrame(test['ID'])
 submission39['item_cnt_month'] = y_predict_OLS
submission39.to_csv('submission39.csv', index=False )
columns_with_lowNA = ['MSZoning','Utilities','Exterior1st','Exterior2nd','MasVnrType','Electrical','KitchenQual','Functional','SaleType']
object_columns_df[columns_with_lowNA] = object_columns_df[columns_with_lowNA].fillna(object_columns_df.mode().iloc[0])
<count_missing_values>
rmse_train_1 = sqrt(mean_squared_error(Y_train, model2.predict(x)))
 rmse_train_1
null_counts = numerical_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))<feature_engineering>
rmse_val_1 = sqrt(mean_squared_error(Y_val,model2.predict(sm.add_constant(X_val.fillna(0)))))
 rmse_val_1