Kaggle competition: House Prices - Advanced Regression Techniques
from itertools import product<concatenate>
warnings.filterwarnings('ignore' )
cartesian_test = []
cartesian_test.append(np.array(jan))
cartesian_test.append(np.array(feb))<concatenate>
import seaborn as sns
import matplotlib.pyplot as plt
from scipy.stats import norm
import scipy.stats
cartesian_test = np.vstack(cartesian_test )<create_dataframe>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
cartesian_test_df = pd.DataFrame(cartesian_test, columns = ['shop_id', 'item_id', 'date_block_num'] )<data_type_conversions>
tri_df=corrmat.mask(mask)
 print([c for c in tri_df.columns if any(tri_df[c]>0.65)] )
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float16)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df<create_dataframe>
columns_to_work_engg = ['GarageYrBlt','YearRemodAdd','1stFlrSF','2ndFlrSF','BedroomAbvGr','GarageCars']
cartesian_df = pd.DataFrame(np.vstack(cartesian), columns = ['shop_id', 'item_id', 'date_block_num'], dtype=np.int32 )<groupby>
train_1=train[train['GrLivArea']<4500]
x = train.groupby(['shop_id', 'item_id', 'date_block_num'])['item_cnt_day'].sum().rename('item_cnt_month' ).reset_index()
x.head()<merge>
train_1=train[train['TotalBsmtSF']<3100]
new_train = pd.merge(cartesian_df, x, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0 )<feature_engineering>
train_1=train[train['GarageArea']<1200]
new_train['item_cnt_month'] = np.clip(new_train['item_cnt_month'], 0, 20 )<drop_column>
num_col=[c for c in train_1.columns if train_1[c].dtype !='object']
del x
del cartesian_df
del cartesian
del cartesian_test
del cartesian_test_df
del feb
del jan
del items_test_list
del items_train_list
del train<sort_values>
skew_before=[]
skew_after=[]
for c in num_col:
skew_before.append(train_1[c].skew())
 skew_after.append(np.log1p(train_1[c] ).skew())
 skew_df=pd.DataFrame(zip(skew_before,skew_after), columns=['skew_before','skew_after'],index=num_col)
 skew_df = skew_df[abs(skew_df['skew_before'])>abs(skew_df['skew_after'])]
skew_df['skew_diff'] = abs(skew_df['skew_before'])-abs(skew_df['skew_after'])
 skew_df[skew_df['skew_diff']>1]
new_train.sort_values(['date_block_num','shop_id','item_id'], inplace = True)
 new_train.head()<prepare_output>
from sklearn.preprocessing import PowerTransformer
from scipy.stats import skew
test.insert(loc=3, column='date_block_num', value=34 )<feature_engineering>
log=PowerTransformer()
skew_before=[]
skew_after=[]
for c in num_col:
skew_before.append(train_1[c].skew())
 log.fit(train_1[c].values.reshape(-1,1))
skew_after.append(skew(( log.transform(train_1[c].values.reshape(-1,1)))) [0])
 skew_df=pd.DataFrame(zip(skew_before,skew_after), columns=['skew_before','skew_after'],index=num_col)
 skew_df = skew_df[abs(skew_df['skew_before'])>abs(skew_df['skew_after'])]
skew_df['skew_diff'] = abs(skew_df['skew_before'])-abs(skew_df['skew_after'])
 skew_df[skew_df['skew_diff']>1]
test['item_cnt_month'] = 0<concatenate>
columns_to_work_log=['LotArea', 'BsmtFinSF2', 'LowQualFinSF', 'GrLivArea', 'KitchenAbvGr', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'MiscVal']
new_train = new_train.append(test.drop('ID', axis = 1))<merge>
columns_to_work_drop=['MiscVal']
new_train = pd.merge(new_train, shops, on=['shop_id'], how='left')
 new_train.head()<merge>
num_cols=[c for c in train_1.columns if train_1[c].dtype !='object']
print(num_cols )
new_train = pd.merge(new_train, items.drop('item_name', axis = 1), on=['item_id'], how='left')
 new_train.head()<merge>
columns_to_work_preprocess = ['MSSubClass']
new_train = pd.merge(new_train, categories.drop('item_category_name', axis = 1), on=['item_category_id'], how='left')
 new_train.head()<merge>
cat_cols=[c for c in train_1.columns if train_1[c].dtype =='object']
print(num_cols )
def generate_lag(train, months, lag_column):
for month in months:
train_shift = train[['date_block_num', 'shop_id', 'item_id', lag_column]].copy()
train_shift.columns = ['date_block_num', 'shop_id', 'item_id', lag_column+'_lag_'+ str(month)]
train_shift['date_block_num'] += month
train = pd.merge(train, train_shift, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 return train<drop_column>
y = train_1['SalePrice'].reset_index(drop=True)
 feature_train = train_1.drop('SalePrice',axis=1)
 print(feature_train.shape,y.shape,test.shape )
del items
del categories
del shops
del test<categorify>
all_data = pd.concat([feature_train,test],ignore_index=True ).reset_index(drop=True)
 all_data.shape
new_train = downcast_dtypes(new_train )<set_options>
all_data['MSSubClass'] = all_data['MSSubClass'].astype('object')
 all_data.MSSubClass.dtype
gc.collect()<define_search_space>
all_data.isna().sum().nlargest(15 )
%%time
new_train = generate_lag(new_train, [1,2,3,4,5,6,12], 'item_cnt_month' )<merge>
columns_to_work_drop.extend(['PoolQC','MiscFeature','Alley','Fence','FireplaceQU','GarageYrBlt','GarageCars'])
 print(columns_to_work_drop )
%%time
group = new_train.groupby(['date_block_num', 'item_id'])['item_cnt_month'].mean().rename('item_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'item_id'], how='left')
 new_train = generate_lag(new_train, [1,2,3,6,12], 'item_month_mean')
 new_train.drop(['item_month_mean'], axis=1, inplace=True )<merge>
all_data.drop(['MiscVal', 'PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'GarageYrBlt','GarageCars'], axis=1,inplace=True )
%%time
group = new_train.groupby(['date_block_num', 'shop_id'])['item_cnt_month'].mean().rename('shop_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'shop_id'], how='left')
 new_train = generate_lag(new_train, [1,2,3,6,12], 'shop_month_mean')
 new_train.drop(['shop_month_mean'], axis=1, inplace=True )<merge>
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))
%%time
group = new_train.groupby(['date_block_num', 'shop_id', 'item_category_id'])['item_cnt_month'].mean().rename('shop_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 new_train = generate_lag(new_train, [1, 2], 'shop_category_month_mean')
 new_train.drop(['shop_category_month_mean'], axis=1, inplace=True )<merge>
for c in ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond','MasVnrType']:
all_data[c].fillna('None', inplace=True )
%%time
group = new_train.groupby(['date_block_num', 'main_category_id'])['item_cnt_month'].mean().rename('main_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'main_category_id'], how='left')
 new_train = generate_lag(new_train, [1], 'main_category_month_mean')
 new_train.drop(['main_category_month_mean'], axis=1, inplace=True )<merge>
all_data['MasVnrArea'].fillna(all_data['MasVnrArea'].mean() , inplace=True)
 all_data['HasMasVnr'] = all_data['MasVnrType'].apply(lambda x: 0 if x == 'None' else 1)
 all_data['HasMasVnr'] = all_data['HasMasVnr'].astype('object' )
%%time
group = new_train.groupby(['date_block_num', 'sub_category_id'])['item_cnt_month'].mean().rename('sub_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'sub_category_id'], how='left')
 new_train = generate_lag(new_train, [1], 'sub_category_month_mean')
 new_train.drop(['sub_category_month_mean'], axis=1, inplace=True )<feature_engineering>
all_data['MSZoning']=all_data.groupby('MSSubClass')['MSZoning'].transform(lambda x:x.fillna(x.mode() [0]))
new_train['month'] = new_train['date_block_num'] % 12<define_variables>
for c in ['BsmtFullBath','BsmtHalfBath','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','GarageArea']:
all_data[c].fillna(0,inplace=True )
holiday_dict = {
0: 6,
1: 3,
2: 2,
3: 8,
4: 3,
5: 3,
6: 2,
7: 8,
8: 4,
9: 8,
10: 5,
11: 4,
}<feature_engineering>
for c in ['Utilities', 'Functional', 'Exterior1st', 'Exterior2nd', 'Electrical','KitchenQual', 'SaleType']:
all_data[c].fillna(all_data[c].mode() [0], inplace=True )
new_train['holidays_in_month'] = new_train['month'].map(holiday_dict )<categorify>
all_data.isna().sum().nlargest()
moex = {
12: 659, 13: 640, 14: 1231,
15: 881, 16: 764, 17: 663,
18: 743, 19: 627, 20: 692,
21: 736, 22: 680, 23: 1092,
24: 657, 25: 863, 26: 720,
27: 819, 28: 574, 29: 568,
30: 633, 31: 658, 32: 611,
33: 770, 34: 723,
}<feature_engineering>
print([c for c in all_data.columns if(all_data[c].value_counts().iloc[0]>2700)&(all_data[c].dtype=='object')] )
new_train['moex_value'] = new_train.date_block_num.map(moex )<categorify>
all_data.drop(['Street', 'Utilities', 'LandSlope', 'Condition2', 'RoofMatl', 'Heating','Functional'], axis=1, inplace=True )
new_train = downcast_dtypes(new_train )<import_modules>
mask=all_data['YearRemodAdd']==all_data['YearBuilt']
all_data['RemodAdd'] = all_data['YearRemodAdd'].mask(mask)
 all_data['RemodAdd'] = all_data['RemodAdd'].fillna(0)
 all_data['RemodAdd'] = all_data['RemodAdd'].apply(lambda x:1 if x != 0 else 0 ).astype('object' )
import xgboost as xgb<prepare_x_and_y>
all_data.drop('YearRemodAdd', axis=1, inplace=True )
new_train = new_train[new_train.date_block_num > 11] 
<set_options>
all_data['TotalSF']=all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['TotalBathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+ all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['TotalPorchSF'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] + all_data['EnclosedPorch'] + all_data['ScreenPorch'] + all_data['WoodDeckSF'] )
gc.collect()<data_type_conversions>
all_data['has1stfloor'] = all_data['1stFlrSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasBsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasOpenPorch'] = all_data['OpenPorchSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['has3SsnPorch'] = all_data['3SsnPorch'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasEnclosedPorch'] = all_data['EnclosedPorch'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasScreenPorch'] = all_data['ScreenPorch'].apply(lambda x: 1 if x > 0 else 0 ).astype('object')
 all_data['hasWoodDeckPorch'] = all_data['WoodDeckSF'].apply(lambda x: 1 if x > 0 else 0 ).astype('object' )
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
df[col].fillna(0, inplace=True)
 return df
new_train = fill_na(new_train )<train_model>
all_data.drop(['TotalBsmtSF','1stFlrSF','2ndFlrSF','FullBath','HalfBath','BsmtFullBath','BsmtHalfBath','OpenPorchSF','3SsnPorch',
'EnclosedPorch','ScreenPorch','WoodDeckSF'],
axis=1,inplace=True )
def xgtrain() :
regressor = xgb.XGBRegressor(n_estimators = 5000,
learning_rate = 0.01,
max_depth = 10,
subsample = 0.5,
colsample_bytree = 0.5)
 regressor_ = regressor.fit(new_train[new_train.date_block_num < 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num < 33]['item_cnt_month'].values,
eval_metric = 'rmse',
eval_set = [(new_train[new_train.date_block_num < 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num < 33]['item_cnt_month'].values), 
(new_train[new_train.date_block_num == 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num == 33]['item_cnt_month'].values)
 ],
verbose=True,
early_stopping_rounds = 50, 
)
 return regressor_<train_model>
all_data.drop(['TotRmsAbvGrd'],axis=1,inplace=True )
%%time
regressor_ = xgtrain()<predict_on_test>
all_data.drop('Id',axis=1,inplace=True )
predictions = regressor_.predict(new_train[new_train.date_block_num == 34].drop(['item_cnt_month'], axis = 1 ).values )<feature_engineering>
all_data_dum = pd.get_dummies(all_data, drop_first=True)
 all_data_dum.shape
submission['item_cnt_month'] = predictions<save_to_csv>
num_col=[c for c in all_data.columns if all_data[c].dtype !='object']
print(num_col )
submission.to_csv('sales_faster_learn.csv', index=False )<import_modules>
log=PowerTransformer()
skew_before=[]
skew_after=[]
for c in num_col:
skew_before.append(X[c].skew())
 log.fit(X[c].values.reshape(-1,1))
skew_after.append(skew(( log.transform(X[c].values.reshape(-1,1)))) [0])
 skew_df=pd.DataFrame(zip(skew_before,skew_after), columns=['skew_before','skew_after'],index=num_col)
 skew_df = skew_df[abs(skew_df['skew_before'])>abs(skew_df['skew_after'])]
skew_df['skew_diff'] = abs(skew_df['skew_before'])-abs(skew_df['skew_after'])
 skew_df = skew_df[skew_df['skew_diff']>0.1]
columns_to_work_log = list(skew_df.index)
 skew_df
FileLinks('.' )<define_variables>
logx = PowerTransformer()
for c in ['LotFrontage', 'LotArea', 'OverallQual', 'OverallCond', 'YearBuilt', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',
'LowQualFinSF', 'GrLivArea', 'KitchenAbvGr', 'Fireplaces', 'PoolArea', 'MoSold', 'TotalSF', 'TotalBathrooms', 'TotalPorchSF']:
X[c] = logx.fit_transform(X[[c]])
 X_test[c] = logx.transform(X_test[[c]] )
vega_url = 'https://cdn.jsdelivr.net/npm/vega@' + alt.SCHEMA_VERSION
vega_lib_url = 'https://cdn.jsdelivr.net/npm/vega-lib'
vega_lite_url = 'https://cdn.jsdelivr.net/npm/vega-lite@' + alt.SCHEMA_VERSION
vega_embed_url = 'https://cdn.jsdelivr.net/npm/vega-embed@3'
noext = "?noext"
altair_paths = {
'vega': vega_url + noext,
'vega-lib': vega_lib_url + noext,
'vega-lite': vega_lite_url + noext,
'vega-embed': vega_embed_url + noext
}
workaround =
def add_autoincrement(render_func):
cache = {}
def wrapped(chart, id="vega-chart", autoincrement=True):

 if autoincrement:
if id in cache:
counter = 1 + cache[id]
cache[id] = counter
else:
cache[id] = 0
actual_id = id if cache[id] == 0 else id + '-' + str(cache[id])
 else:
if id not in cache:
cache[id] = 0
actual_id = id
return render_func(chart, id=actual_id)
 return wrapped
@add_autoincrement
def render_alt(chart, id="vega-chart"):
chart_str =
return HTML(
 chart_str.format(
 id=id,
chart=json.dumps(chart)if isinstance(chart, dict)else chart.to_json(indent=None)
)
)
 HTML("".join((
"<script>",
workaround.format(paths=json.dumps(altair_paths)) ,
"</script>" 
)) )<set_options>
logy = PowerTransformer()
y = logy.fit_transform(y.values.reshape(-1,1))
alt.data_transformers.enable('default', max_rows=None )<load_from_csv>
from xgboost import XGBRegressor
from sklearn.linear_model import ElasticNet, Lasso, LassoLarsCV
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.model_selection import cross_val_score, RandomizedSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.metrics import mean_squared_error, make_scorer
from sklearn.feature_selection import RFE
from sklearn.decomposition import PCA
train = pd.read_csv('.. /input/sales_train.csv' )<groupby>
def rmse_cv(model):
neg_mse = cross_val_score(model, X,y, cv=5, scoring='neg_mean_squared_error')
 rmse=np.sqrt(-neg_mse)
 print('RMSE mean: {0:.5f} , RMSE std: {1:.5f}'.format(rmse.mean() , rmse.std()))
alt_df = train.groupby('date_block_num')['item_cnt_day'].sum().rename('sum' ).reset_index()
alt_df.head(2 )<define_variables>
def rmse_cv_return(model):
neg_mse = cross_val_score(model, X,y, cv=5, scoring='neg_mean_squared_error')
 rmse=np.sqrt(-neg_mse)
 return rmse.mean()
dict_date_block_num = {0: 'January 2013', 1: 'February 2013', 2: 'March 2013', 3: 'April 2013',
4: 'May 2013', 5: 'June 2013', 6: 'July 2013', 7: 'August 2013',
8: 'September 2013', 9: 'October 2013', 10: 'November 2013', 11: 'December 2013',
12: 'January 2014', 13: 'February 2014', 14: 'March 2014', 15: 'April 2014',
16: 'May 2014', 17: 'June 2014', 18: 'July 2014', 19: 'August 2014',
20: 'September 2014', 21: 'October 2014', 22: 'November 2014', 23: 'December 2014',
24: 'January 2015', 25: 'February 2015', 26: 'March 2015', 27: 'April 2015',
28: 'May 2015', 29: 'June 2015', 30: 'July 2015', 31: 'August 2015',
32: 'September 2015', 33: 'October 2015'}<string_transform>
model_xgb = XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor', random_state=1111)
 rmse_cv(model_xgb )
split = alt_df['date_block_num'].str.split(" ", n = 1, expand = True )<feature_engineering>
model_xgboost1 = make_pipeline(StandardScaler() , XGBRegressor())
 rmse_cv(model_xgboost1 )
alt_df_two = alt_df.copy()
alt_df_two['month'] = split[0]
alt_df_two['year'] = split[1]
alt_df_two.head()<set_options>
model_xgb_tuned = XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=4.31172,n_estimators=800,max_depth=2,learning_rate=0.112414 ,random_state=1111)
 rmse_cv(model_xgb_tuned )
alt.themes.enable('opaque' )<feature_engineering>
model_xgb_tuned1 = XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=17.144285714285715,n_estimators=500,max_depth=3,learning_rate=0.07827586206896552 ,random_state=1111)
 rmse_cv(model_xgb_tuned1 )
alt_df_three = train.groupby(['date_block_num', 'shop_id'])['item_cnt_day'].sum().rename('sum' ).reset_index()
alt_df_three['date_block_num'].replace(dict_date_block_num, inplace=True)
 split = alt_df_three['date_block_num'].str.split(" ", n = 1, expand = True)
 alt_df_three['month'] = split[0]
alt_df_three['year'] = split[1]
alt_df_three.head(2 )<set_options>
model_xgb_tuned1_sc = make_pipeline(StandardScaler() ,XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=17.144285714285715,n_estimators=500,max_depth=3,learning_rate=0.07827586206896552 ,random_state=1111))
rmse_cv(model_xgb_tuned1_sc )
%matplotlib inline
warnings.filterwarnings("ignore")
 pd.set_option('max_rows', 500)
<load_from_csv>
model_xgb_tuned2 = XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=16.362109350140535,n_estimators=400,max_depth=2,learning_rate=0.04200863893086535 ,random_state=1111)
 rmse_cv(model_xgb_tuned2 )
def read_data() :
print('Reading train.csv file.... ')
 train = pd.read_csv('/kaggle/input/data-science-bowl-2019/train.csv')
 print('Training.csv file have {} rows and {} columns'.format(train.shape[0], train.shape[1]))
print('Reading test.csv file.... ')
 test = pd.read_csv('/kaggle/input/data-science-bowl-2019/test.csv')
 print('Test.csv file have {} rows and {} columns'.format(test.shape[0], test.shape[1]))
print('Reading train_labels.csv file.... ')
 train_labels = pd.read_csv('/kaggle/input/data-science-bowl-2019/train_labels.csv')
 print('Train_labels.csv file have {} rows and {} columns'.format(train_labels.shape[0], train_labels.shape[1]))
print('Reading specs.csv file.... ')
 specs = pd.read_csv('/kaggle/input/data-science-bowl-2019/specs.csv')
 print('Specs.csv file have {} rows and {} columns'.format(specs.shape[0], specs.shape[1]))
print('Reading sample_submission.csv file.... ')
 sample_submission = pd.read_csv('/kaggle/input/data-science-bowl-2019/sample_submission.csv')
 print('Sample_submission.csv file have {} rows and {} columns'.format(sample_submission.shape[0], sample_submission.shape[1]))
return train, test, train_labels, specs, sample_submission<categorify>
model_xgb_tuned2_sc = make_pipeline(StandardScaler() ,XGBRegressor(tree_method='gpu_hist',n_jobs=-1,booster='gbtree',predictor='gpu_predictor',
reg_lambda=16.362109350140535,n_estimators=400,max_depth=2,learning_rate=0.04200863893086535 ,random_state=1111))
rmse_cv(model_xgb_tuned2_sc )
def encode_title(train, test, train_labels):
train['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), train['title'], train['event_code']))
test['title_event_code'] = list(map(lambda x, y: str(x)+ '_' + str(y), test['title'], test['event_code']))
all_title_event_code = list(set(train["title_event_code"].unique() ).union(test["title_event_code"].unique()))
list_of_user_activities = list(set(train['title'].unique() ).union(set(test['title'].unique())))
 list_of_event_code = list(set(train['event_code'].unique() ).union(set(test['event_code'].unique())))
 list_of_event_id = list(set(train['event_id'].unique() ).union(set(test['event_id'].unique())))
 list_of_worlds = list(set(train['world'].unique() ).union(set(test['world'].unique())))
 activities_map = dict(zip(list_of_user_activities, np.arange(len(list_of_user_activities))))
activities_labels = dict(zip(np.arange(len(list_of_user_activities)) , list_of_user_activities))
activities_world = dict(zip(list_of_worlds, np.arange(len(list_of_worlds))))
assess_titles = list(set(train[train['type'] == 'Assessment']['title'].value_counts().index ).union(set(test[test['type'] == 'Assessment']['title'].value_counts().index)))
 train['title'] = train['title'].map(activities_map)
 test['title'] = test['title'].map(activities_map)
 train['world'] = train['world'].map(activities_world)
 test['world'] = test['world'].map(activities_world)
 train_labels['title'] = train_labels['title'].map(activities_map)
 win_code = dict(zip(activities_map.values() ,(4100*np.ones(len(activities_map)) ).astype('int')))
 win_code[activities_map['Bird Measurer(Assessment)']] = 4110
train['timestamp'] = pd.to_datetime(train['timestamp'])
 test['timestamp'] = pd.to_datetime(test['timestamp'])
 return train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code<define_variables>
def get_data(user_sample, test_set=False):

 last_activity = 0
user_activities_count = {'Clip':0, 'Activity': 0, 'Assessment': 0, 'Game':0}
last_session_time_sec = 0
accuracy_groups = {0:0, 1:0, 2:0, 3:0}
all_assessments = []
accumulated_accuracy_group = 0
accumulated_accuracy = 0
accumulated_correct_attempts = 0
accumulated_uncorrect_attempts = 0
accumulated_actions = 0
counter = 0
time_first_activity = float(user_sample['timestamp'].values[0])
 durations = []
last_accuracy_title = {'acc_' + title: -1 for title in assess_titles}
event_code_count: Dict[str, int] = {ev: 0 for ev in list_of_event_code}
event_id_count: Dict[str, int] = {eve: 0 for eve in list_of_event_id}
title_count: Dict[str, int] = {eve: 0 for eve in activities_labels.values() }
title_event_code_count: Dict[str, int] = {t_eve: 0 for t_eve in all_title_event_code}
for i, session in user_sample.groupby('game_session', sort=False):
session_type = session['type'].iloc[0]
session_title = session['title'].iloc[0]
session_title_text = activities_labels[session_title]
if(session_type == 'Assessment')&(test_set or len(session)>1):
all_attempts = session.query(f'event_code == {win_code[session_title]}')
 true_attempts = all_attempts['event_data'].str.contains('true' ).sum()
false_attempts = all_attempts['event_data'].str.contains('false' ).sum()
features = user_activities_count.copy()
features.update(last_accuracy_title.copy())
 features.update(event_code_count.copy())
 features.update(event_id_count.copy())
 features.update(title_count.copy())
 features.update(title_event_code_count.copy())
 features.update(last_accuracy_title.copy())
 features['installation_id'] = session['installation_id'].iloc[-1]
features['session_title'] = session['title'].iloc[0]
features['accumulated_correct_attempts'] = accumulated_correct_attempts
features['accumulated_uncorrect_attempts'] = accumulated_uncorrect_attempts
accumulated_correct_attempts += true_attempts
accumulated_uncorrect_attempts += false_attempts
if durations == []:
features['duration_mean'] = 0
else:
features['duration_mean'] = np.mean(durations)
 durations.append(( session.iloc[-1, 2] - session.iloc[0, 2] ).seconds)
 features['accumulated_accuracy'] = accumulated_accuracy/counter if counter > 0 else 0
accuracy = true_attempts/(true_attempts+false_attempts)if(true_attempts+false_attempts)!= 0 else 0
accumulated_accuracy += accuracy
last_accuracy_title['acc_' + session_title_text] = accuracy
if accuracy == 0:
features['accuracy_group'] = 0
elif accuracy == 1:
features['accuracy_group'] = 3
elif accuracy == 0.5:
features['accuracy_group'] = 2
else:
features['accuracy_group'] = 1
features.update(accuracy_groups)
 accuracy_groups[features['accuracy_group']] += 1
features['accumulated_accuracy_group'] = accumulated_accuracy_group/counter if counter > 0 else 0
accumulated_accuracy_group += features['accuracy_group']
features['accumulated_actions'] = accumulated_actions
if test_set:
all_assessments.append(features)
 elif true_attempts+false_attempts > 0:
all_assessments.append(features)
 counter += 1
def update_counters(counter: dict, col: str):
num_of_session_count = Counter(session[col])
 for k in num_of_session_count.keys() :
x = k
if col == 'title':
x = activities_labels[k]
counter[x] += num_of_session_count[k]
return counter
event_code_count = update_counters(event_code_count, "event_code")
 event_id_count = update_counters(event_id_count, "event_id")
 title_count = update_counters(title_count, 'title')
 title_event_code_count = update_counters(title_event_code_count, 'title_event_code')
 accumulated_actions += len(session)
 if last_activity != session_type:
user_activities_count[session_type] += 1
last_activitiy = session_type
if test_set:
return all_assessments[-1]
return all_assessments<groupby>
model_xgb_tuned2.fit(X,y)
 y_pred_test = model_xgb_tuned2.predict(X_test)
 y_pred_test = logy.inverse_transform(y_pred_test.reshape(-1,1))
y_pred_test = y_pred_test.reshape(-1 )
def get_train_and_test(train, test):

 compiled_train = []
compiled_test = []
for i,(ins_id, user_sample)in tqdm(enumerate(train.groupby('installation_id', sort = False)) , total = 17000):
compiled_train += get_data(user_sample)
 for ins_id, user_sample in tqdm(test.groupby('installation_id', sort = False), total = 1000):
test_data = get_data(user_sample, test_set = True)
 compiled_test.append(test_data)
 reduce_train = pd.DataFrame(compiled_train)
 reduce_test = pd.DataFrame(compiled_test)
 categoricals = ['session_title']
return reduce_train, reduce_test, categoricals<categorify>
output = pd.DataFrame({'Id': test['Id'],
'SalePrice': y_pred_test})
 output.to_csv('submission.csv', index=False )