Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<find_best_params>
plt.rcParams["figure.figsize"] =(20,4)
 %matplotlib inline
warnings.filterwarnings('ignore', category=FutureWarning )
random_cv.best_estimator_<choose_model_class>
folder = '/kaggle/input/competitive-data-science-predict-future-sales/'
train = pd.read_csv(folder+'sales_train.csv')
 shops = pd.read_csv(folder+'shops.csv')
 items = pd.read_csv(folder+'items.csv')
 itemcat = pd.read_csv(folder+'item_categories.csv')
 test = pd.read_csv(folder+'test.csv' )
regressor=xgboost.XGBRegressor(base_score=0.75, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.15, max_delta_step=0, max_depth=2,
min_child_weight=3, monotone_constraints='() ',
n_estimators=500, n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None )<train_model>
train[['item_id','item_price','shop_id']].drop_duplicates().sort_values(by = 'item_price',ascending = False ).head(3 )
regressor.fit(X_train,Y_train )<predict_on_test>
items[items.item_id==6066]
Y_pred_xgb=regressor.predict(X_test)
 Y_xgb_train=regressor.predict(X_train )<compute_test_metric>
test[test.item_id==6066]
print("RMSE of XgBosst Model is",np.sqrt(mean_squared_error(Y_train,Y_xgb_train)) )<compute_test_metric>
train[train.item_id==6066]
r2_score=lr_model.score(X_train,Y_train)
 print("R2_score of XgBosst Model is ",r2_score )<save_to_csv>
train.drop(train[train.item_id==6066].index, inplace = True )
sub_df=pd.DataFrame(data={'Id':test_df1['Id'].values,'SalePrice':np.exp(Y_pred_xgb)})
 sub_df.to_csv('submission.csv',index=False )<load_from_csv>
expensive = train[train.item_price>10000][['item_id']].drop_duplicates()
test_expensive = expensive.merge(test[['item_id']].drop_duplicates() , on = 'item_id', how = 'inner')
 print('There are {} items expensiver than 10000 in train.
 {} of them are in test'.format(
 len(expensive),len(test_expensive)))
tomate = pd.read_csv(".. /input/-32-house-prices-version-7-score-007483/submission.csv")
 pepino = pd.read_csv(".. /input/7-lines-of-code-to-reach-6th-place/submission.csv" )<feature_engineering>
train[train.item_id==11365].merge(items, on = 'item_id',how = 'left' ).sort_values(by = 'item_price',ascending = False ).head()
tomate["SalePrice"] = tomate["SalePrice"]*0.91 + pepino["SalePrice"]*0.09<save_to_csv>
items[items.item_category_id==9]
tomate.to_csv('submission.csv', index=False )<import_modules>
train = train.merge(items, on = 'item_id', how = 'left' )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
import sklearn.metrics as metrics
import math<load_from_csv>
np.percentile(train[train.item_category_id==9].item_price,99.9 )
test = pd.read_csv(".. /input/datafiles/test.csv")
 train = pd.read_csv(".. /input/datafiles/train.csv")
 Copy_test = test.copy()
Copy_train = train.copy()<concatenate>
shops[shops.shop_id==12]
Copy_train['train'] = 1
Copy_test['train'] = 0
df = pd.concat([Copy_train, Copy_test], axis=0,sort=False )<create_dataframe>
def str_clean(x):
return re.sub(re.compile('\W'),' ',x.lower() ).strip()
shops['shop_name'] = shops['shop_name'].apply(str_clean)
 itemcat['item_category_name'] = itemcat['item_category_name'].apply(str_clean )
NAN = [(c, df[c].isna().mean() *100)for c in df]
NAN = pd.DataFrame(NAN, columns=["column_name", "percentage"] )<sort_values>
items['item_name'] = items['item_name'].apply(str_clean )
NAN = NAN[NAN.percentage > 50]
NAN.sort_values("percentage", ascending=False )<drop_column>
duplicate_shops = {57:0,58:1,40:39,11:10}
def fix_shop_id(x):
if x not in duplicate_shops.keys() :
return x
return duplicate_shops[x]
train['shop_id'] = train['shop_id'].apply(fix_shop_id)
 test['shop_id'] = test['shop_id'].apply(fix_shop_id)
 shops.drop(shops[shops.shop_id.isin(duplicate_shops.keys())].index,inplace = True )
df = df.drop(['Alley','PoolQC','Fence','MiscFeature'],axis=1 )<count_missing_values>
shops['city'] = shops.shop_name.apply(lambda x: x.split() [0] )
null_counts = object_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))<data_type_conversions>
le = LabelEncoder()
shops['city_id'] = le.fit_transform(shops['city'] )
columns_None = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','FireplaceQu','GarageCond']
object_columns_df[columns_None]= object_columns_df[columns_None].fillna('None' )<data_type_conversions>
prices = train.groupby(['item_id','shop_id','date_block_num'],as_index=False ).agg(
 {
'item_price':{'mean_price':'mean','price_std':'std','price_count':'count'},
'item_cnt_day':{'sold_total':'sum'}
}
).fillna(0 )
columns_with_lowNA = ['MSZoning','Utilities','Exterior1st','Exterior2nd','MasVnrType','Electrical','KitchenQual','Functional','SaleType']
object_columns_df[columns_with_lowNA] = object_columns_df[columns_with_lowNA].fillna(object_columns_df.mode().iloc[0] )<count_missing_values>
prices.columns = ['item_id','shop_id','date_block_num','mean_price','price_std','price_count','sold_total']
null_counts = numerical_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))<feature_engineering>
prices['date_block_num']+=1
numerical_columns_df['GarageYrBlt'] = numerical_columns_df['GarageYrBlt'].fillna(numerical_columns_df['YrSold']-35)
 numerical_columns_df['LotFrontage'] = numerical_columns_df['LotFrontage'].fillna(68 )<drop_column>
shops[(shops.shop_id==55)|(shops.shop_id==12)]
object_columns_df = object_columns_df.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1 )<filter>
X.sort_values(by = ['shop_id','item_id','date_block_num'],inplace=True)
 month_ago = 3
for i in range(1, month_ago+1):
colname = 'prev' + str(i)
 X[colname] = X.target.shift(i)
 mask1 = X.shop_id != X.shop_id.shift(i)
 mask2 = X.item_id!= X.item_id.shift(i)
 mask = mask1|mask2
X[colname][mask] = 0
train_columns = ['shop_id','item_id','target']+['prev'+str(i)for i in range(1,month_ago+1)]
test = test.merge(X[X.date_block_num==33][train_columns],on = ['shop_id','item_id'],how = 'left' )
Negatif = numerical_columns_df[numerical_columns_df['Age_House'] < 0]
Negatif<feature_engineering>
test.rename(columns = {'prev3':'prev4','prev2':'prev3','prev1':'prev2','target':'prev1'}, inplace = True)
 del test['prev'+str(month_ago+1)]
numerical_columns_df['TotalBsmtBath'] = numerical_columns_df['BsmtFullBath'] + numerical_columns_df['BsmtFullBath']*0.5
numerical_columns_df['TotalBath'] = numerical_columns_df['FullBath'] + numerical_columns_df['HalfBath']*0.5
numerical_columns_df['TotalSA']=numerical_columns_df['TotalBsmtSF'] + numerical_columns_df['1stFlrSF'] + numerical_columns_df['2ndFlrSF']<categorify>
X_target = X.target.values
bin_map = {'TA':2,'Gd':3, 'Fa':1,'Ex':4,'Po':1,'None':0,'Y':1,'N':0,'Reg':3,'IR1':2,'IR2':1,'IR3':0,"None" : 0,
"No" : 2, "Mn" : 2, "Av": 3,"Gd" : 4,"Unf" : 1, "LwQ": 2, "Rec" : 3,"BLQ" : 4, "ALQ" : 5, "GLQ" : 6
}
object_columns_df['ExterQual'] = object_columns_df['ExterQual'].map(bin_map)
 object_columns_df['ExterCond'] = object_columns_df['ExterCond'].map(bin_map)
 object_columns_df['BsmtCond'] = object_columns_df['BsmtCond'].map(bin_map)
 object_columns_df['BsmtQual'] = object_columns_df['BsmtQual'].map(bin_map)
 object_columns_df['HeatingQC'] = object_columns_df['HeatingQC'].map(bin_map)
 object_columns_df['KitchenQual'] = object_columns_df['KitchenQual'].map(bin_map)
 object_columns_df['FireplaceQu'] = object_columns_df['FireplaceQu'].map(bin_map)
 object_columns_df['GarageQual'] = object_columns_df['GarageQual'].map(bin_map)
 object_columns_df['GarageCond'] = object_columns_df['GarageCond'].map(bin_map)
 object_columns_df['CentralAir'] = object_columns_df['CentralAir'].map(bin_map)
 object_columns_df['LotShape'] = object_columns_df['LotShape'].map(bin_map)
 object_columns_df['BsmtExposure'] = object_columns_df['BsmtExposure'].map(bin_map)
 object_columns_df['BsmtFinType1'] = object_columns_df['BsmtFinType1'].map(bin_map)
 object_columns_df['BsmtFinType2'] = object_columns_df['BsmtFinType2'].map(bin_map)
 PavedDrive = {"N" : 0, "P" : 1, "Y" : 2}
object_columns_df['PavedDrive'] = object_columns_df['PavedDrive'].map(PavedDrive)
<categorify>
del X['target']
rest_object_columns = object_columns_df.select_dtypes(include=['object'])
 object_columns_df = pd.get_dummies(object_columns_df, columns=rest_object_columns.columns)
<concatenate>
test['date_block_num']=34
X['ID'] = X.index
df_final = pd.concat([object_columns_df, numerical_columns_df], axis=1,sort=False)
 df_final.head()<drop_column>
X = X.append(test )
df_final = df_final.drop(['Id',],axis=1)
 df_train = df_final[df_final['train'] == 1]
df_train = df_train.drop(['train',],axis=1)
 df_test = df_final[df_final['train'] == 0]
df_test = df_test.drop(['SalePrice'],axis=1)
 df_test = df_test.drop(['train',],axis=1)
<prepare_x_and_y>
X = X.merge(items[['item_id','item_category_id', 'item_name']], on = 'item_id', how = 'left')
 X = X.merge(shops[['shop_id','city_id','shop_name']], on = 'shop_id', how = 'left' )
target= df_train['SalePrice']
df_train = df_train.drop(['SalePrice'],axis=1 )<split>
X = X.merge(itemcat, on = 'item_category_id', how = 'left' )
x_train,x_test,y_train,y_test = train_test_split(df_train,target,test_size=0.33,random_state=0 )<choose_model_class>
X = X.merge(prices, on = ['item_id','shop_id','date_block_num'], how = 'left' )
xgb =XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=2400,
n_jobs=1, nthread=None, objective='reg:linear',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=12000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.4,
)<train_model>
X['month'] = X['date_block_num']%12 + 1
xgb.fit(x_train, y_train)
 lgbm.fit(x_train, y_train,eval_metric='rmse' )<predict_on_test>
X['is_internet'] = X['shop_id'].apply(lambda x: 1 if x in [12,55] else 0 )
predict1 = xgb.predict(x_test)
 predict = lgbm.predict(x_test )<compute_test_metric>
X['target_block_num'] = X['date_block_num']-1
X['target_month'] = X['target_block_num']%12+1
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict1))))
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict))))<train_model>
X[(~X.item_id.isin([20949])) &(~X.item_category_id.isin(['9'])) ].sort_values(
 by = 'prev1', ascending = False ).head().T
xgb.fit(df_train, target)
 lgbm.fit(df_train, target,eval_metric='rmse' )<save_to_csv>
X['is_bag'] = X.item_id.apply(lambda x: 1 if x == 20949 else 0 )
submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": predict_y
})
 submission.to_csv('submission.csv', index=False )<import_modules>
popular_games = X[(X.item_category_id==31)&(X.prev1>100)].sort_values(by = 'prev1', ascending = False ).item_id.drop_duplicates().head(10 ).to_list()
import numpy as np
import pandas as pd
from sklearn import preprocessing
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
import lightgbm as lgb<categorify>
game = 3730
def label_encoding(train: pd.DataFrame, test: pd.DataFrame, col_definition: dict):

 n_train = len(train)
 train = pd.concat([train, test], sort=False ).reset_index(drop=True)
 for f in col_definition['encode_col']:
try:
lbl = preprocessing.LabelEncoder()
train[f] = lbl.fit_transform(list(train[f].values))
except:
print(f)
 test = train[n_train:].reset_index(drop=True)
 train = train[:n_train]
return train, test<load_from_csv>
X[X.item_id==game].groupby('target_block_num',as_index = False)['prev1'].sum()
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 sub = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv' )<count_unique_values>
X[X.prev1>100].item_id.nunique()
unique_cols = list(train.columns[train.nunique() == 1])
 print(unique_cols )<count_duplicates>
col = 'item_category_id'
for col in ['item_category_id','item_id','shop_id','city_id']:
X = X.merge(
 X.groupby(col, as_index=False)['prev1'].agg({col+'_mean':'mean',col+'_median':'median',col+'_std':'std'}),
on = col,
how = 'left'
)
duplicated_cols = list(train.columns[train.T.duplicated() ])
 print(duplicated_cols )<categorify>
ftr = [
'month',
'city_id',
'item_id',
'shop_id',
'date_block_num',
'prev1','prev2','prev3',
'item_category_id',
'mean_price',
'price_count',
'sold_total',
'is_internet',
'is_bag'
]
train, test = label_encoding(train, test, col_definition={'encode_col': categorical_cols} )<prepare_x_and_y>
for i in [x for x in ftr if x not in ['prev1','prev2','prev3']]:
X[i] = X[i].clip(0,np.percentile(X[i],99.9))
X_train = train.drop(['Id', 'SalePrice'], axis=1)
 y_train = np.log1p(train['SalePrice'])
 X_test = test.drop(['Id', 'SalePrice'], axis=1 )<prepare_x_and_y>
for i in ['prev1','prev2','prev3']:
X[i] = X[i].clip(0,20 )
y_preds = []
models = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=0)
 params = {
'num_leaves' : 24,
'max_depth' : 6,
'objective' : 'regression',
'metric' : 'rmse',
'learning_rate' : 0.05
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.loc[train_index, :]
X_val = X_train.loc[valid_index, :]
y_tr = y_train[train_index]
y_val = y_train[valid_index]
lgb_train = lgb.Dataset(X_tr, y_tr, categorical_feature=categorical_cols)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train, categorical_feature=categorical_cols)
 model = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_eval], verbose_eval=10, num_boost_round=1000, early_stopping_rounds=10)
 oof_train[valid_index] = model.predict(X_val, num_iteration=model.best_iteration)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 y_preds.append(y_pred)
 models.append(model )<import_modules>
lgbm = LGBMRegressor(random_state = 42,
max_depth = 8,
objective = 'rmse',
)
print(f'CV: {np.sqrt(mean_squared_error(y_train, oof_train)) }' )<compute_test_metric>
months = pd.DataFrame()
for m in range(32):
lgbm.fit(X[(X.date_block_num>m)&(X.date_block_num<33)][ftr],
X_target[X[(X.date_block_num>m)&(X.date_block_num<33)].index].clip(0,20))
pred = lgbm.predict(X[X.date_block_num==33][ftr])
 months.loc[m,'score'] = mean_squared_error(
 pred.clip(0,20),X_target[X[X.date_block_num==33].index].clip(0,20)) **0.5
print(m,": ",months.loc[m,'score'])
accuracy_score(np.round(y_train), np.round(oof_train))<save_to_csv>
begin_month = 10
lgbm.fit(X[(X.date_block_num>begin_month)&(X.date_block_num<33)][ftr],
X_target[X[(X.date_block_num>begin_month)&(X.date_block_num<33)].index])
 pred = lgbm.predict(X[X.date_block_num==33][ftr])
 score = mean_squared_error(pred.clip(0,20),X_target[X[X.date_block_num==33].index].clip(0,20)) **0.5
print(score )
y_sub = sum(y_preds)/ len(y_preds)
 y_sub = np.expm1(y_sub)
 sub['SalePrice'] = y_sub
sub.to_csv('submission.csv', index=False)
 sub.head()<set_options>
lgbm.fit(X[(X.date_block_num>begin_month)&(X.date_block_num<34)][ftr],
X_target[X[(X.date_block_num>begin_month)&(X.date_block_num<34)].index].clip(0,20))
pred = lgbm.predict(X[X.date_block_num==34][ftr] ).clip(0,20)
 res = pd.DataFrame()
res['ID'] = test['ID']
res['item_cnt_month'] = pred
%reload_ext autoreload
%autoreload 2
path = Path('/kaggle/input/house-prices-advanced-regression-techniques/')
 path.ls()<load_from_csv>
nt = test.merge(items[['item_id','item_category_id']], on = 'item_id',how = 'left')
 res.loc[nt[(nt.item_category_id==9)&(nt.shop_id!=12)].index,'item_cnt_month'] = 0
<define_variables><EOS> 
res.to_csv('submission.csv', index = False )