Kaggle competition: KaggleDays Paris
fill_na_num_0(features,'BsmtFinSF2')
 features[['BsmtFinSF1','BsmtFinType1']].head() 
<categorify>
warnings.filterwarnings('ignore')
fill_na_cat_none(features,'BsmtFinType1')
 fill_na_num_0(features,'BsmtFinSF1')
 fill_na_cat_none(features,'BsmtFinType2')
 fill_na_num_0(features,'BsmtFinSF2')
 features[['TotalBsmtSF','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF']].head()<drop_column>
data_path = '.. /input/kaggledays-paris/'
train = pd.read_csv(data_path + 'train.csv',encoding='UTF-8')
 test = pd.read_csv(data_path + 'test.csv',encoding='UTF-8')
 colorfile = pd.read_csv('.. /input/color-feature/color_transcode2.csv',encoding='UTF-8')
 train = train.merge(colorfile, on='color', how='left')
 test = test.merge(colorfile, on='color', how='left')
 train['color'] = train['new_color']
test['color'] = test['new_color']
print(train.shape, test.shape)
 print(train.head())
 train['is_test'] = 0
test['is_test'] = 1
df_all = pd.concat([train, test], axis=0)
 print('train ',train.shape)
 print('test ',test.shape )
fill_na_num_0(features,'TotalBsmtSF')
 fill_na_num_0(features,'BsmtUnfSF' )<data_type_conversions>
print('Preprocessing text...')
 tfidf = TfidfVectorizer(max_features=5,norm='l2',)
 tfidf.fit(df_all[ 'en_US_description'].astype(str ).values)
 tfidf_all = np.array(tfidf.transform(df_all[ 'en_US_description'].astype(str ).values ).toarray() , dtype=np.float16)
 for i in tqdm(range(5)) :
df_all['en_US_description_tfidf_' + str(i)] = tfidf_all[:, i]
del tfidf, tfidf_all
gc.collect()
print('Done.')
 print('train ',train.shape)
 print('test ',test.shape )
fill_na_num_0(features,'BsmtFullBath')
 fill_na_num_0(features,'BsmtHalfBath' )<drop_column>
print('Label Encoder...')
 cols = [f_ for f_ in df_all.columns if df_all[f_].dtype == 'object' and 'sku' not in f_ and 'ID' not in f_]
print(cols)
 cnt = 0
for c in tqdm(cols):
le = LabelEncoder()
df_all[c] = le.fit_transform(df_all[c].astype(str))
cnt += 1
del le
print('len(cols)= {}'.format(cnt))
train = df_all.loc[df_all['is_test'] == 0].drop(['is_test'], axis=1)
 test = df_all.loc[df_all['is_test'] == 1].drop(['is_test'], axis=1)
 print('train ',train.shape)
 print('test ',test.shape )
features.drop('GarageYrBlt', inplace = True, axis = 1)
 features.drop(['Utilities'], axis=1, inplace = True)
 fill_na_cat_none(features,'GarageType')
 fill_na_cat_none(features,'GarageFinish')
 fill_na_cat_none(features,'GarageQual')
 fill_na_cat_none(features,'GarageCond')
 fill_na_num_0(features,'GarageCars')
 fill_na_num_0(features,'GarageArea' )<define_variables>
vimages = pd.read_csv(data_path + 'vimages.csv')
 pca = PCA(n_components=10)
 vpca = pca.fit_transform(vimages.drop('sku_hash', axis=1 ).values)
 vimages_pca = vimages[['sku_hash']]
for i in tqdm(range(vpca.shape[1])) :
vimages_pca['dim_pca_{}'.format(i)] = vpca[:, i]
train = train.merge(vimages_pca, on='sku_hash', how='left')
 test = test.merge(vimages_pca, on='sku_hash', how='left')
 print('train ',train.shape)
 print('test ',test.shape )
new_numerical_features = features.dtypes[features.dtypes != "object"].index
new_categorical_features = features.dtypes[features.dtypes == "object"].index
new_num_df = features[new_numerical_features]
new_cat_df = features[new_categorical_features]
features[['BsmtQual','BsmtCond','BsmtExposure','TotalBsmtSF']].head()<categorify>
sales = pd.read_csv(data_path + 'sales.csv')
 sales = sales.merge(df_all[['sku_hash','model','function']], on='sku_hash', how='left')
 sales = sales.sort_values('Date', ascending=True)
 salesg = sales.groupby('sku_hash' ).agg({
'sales_quantity': ['sum', 'mean'],
'Month_transaction': ['last'],
} ).reset_index()
salesg.columns = ['%s%s' %(a, '_%s' % b if b else '')for a, b in salesg.columns]
print(salesg.head())
 train = train.merge(salesg, on='sku_hash', how='left')
 test = test.merge(salesg, on='sku_hash', how='left')
 aggs={}
aggs['sales_quantity']= {'sum':'sum'}
agg_tmp = sales.groupby(['sku_hash','Date'] ).agg(aggs)
 agg_tmp.columns = [ '{}_{}'.format(k, agg)for k in aggs.keys() for agg in aggs[k]]
agg_tmp=agg_tmp.reset_index()
agg_tmp = agg_tmp.sort_values('Date', ascending=True)
 XX_sales = agg_tmp.groupby(['sku_hash'])['sales_quantity_sum'].apply(lambda x: sales_features(x))
XX_sales = expand_series_to_df(XX_sales.reset_index())
 XX_sales.columns = ['sku_hash',
'sales_quantity_sum_linear_trend_rvalue',
'sales_quantity_sum_linear_trend_intercept',
'sales_quantity_sum_linear_trend_attr_slope',
'sales_quantity_sum_linear_trend_attr_stderr',
'sales_quantity_sum_kurtosis',
'sales_quantity_sum_skewness'
]
train = train.merge(XX_sales, on='sku_hash', how='left')
 test = test.merge(XX_sales, on='sku_hash', how='left')
 print('train ',train.shape)
 print('test ',test.shape )
fill_na_cat_none(features,'BsmtExposure')
 fill_na_cat_none(features,'BsmtCond')
 fill_na_cat_none(features,'BsmtQual')
 fill_na_mode(features,'MSZoning')
 fill_na_mode(features,'SaleType')
 fill_na_mode(features,'KitchenQual')
 fill_na_mode(features,'Electrical')
 fill_na_mode(features,'Exterior1st')
 fill_na_mode(features,'Exterior2nd')
 fill_na_mode(features,'Functional')
 features[['FireplaceQu','Fireplaces']].head()<concatenate>
agg_tmp = sales.groupby(['model','Date'] ).agg(aggs)
 agg_tmp.columns = [ '{}_{}'.format(k, agg)for k in aggs.keys() for agg in aggs[k]]
agg_tmp=agg_tmp.reset_index()
agg_tmp = agg_tmp.sort_values('Date', ascending=True)
 XX_sales_BYMODEL = agg_tmp.groupby(['model'])['sales_quantity_sum'].apply(lambda x: sales_features(x))
XX_sales_BYMODEL = expand_series_to_df(XX_sales_BYMODEL.reset_index())
 XX_sales_BYMODEL.columns = ['model',
'BYMODELsales_quantity_sum_linear_trend_rvalue',
'BYMODELsales_quantity_sum_linear_trend_intercept',
'BYMODELsales_quantity_sum_linear_trend_attr_slope',
'BYMODELsales_quantity_sum_linear_trend_attr_stderr',
'BYMODELsales_quantity_sum_kurtosis',
'BYMODELsales_quantity_sum_skewness'
]
train = train.merge(XX_sales_BYMODEL, on='model', how='left')
 test = test.merge(XX_sales_BYMODEL, on='model', how='left')
 print('train ',train.shape)
 print('test ',test.shape )
fill_na_cat_none(features,'FireplaceQu' )<categorify>
def extract_features(df):
df['month_real'] =(df['month']+df['Month_transaction_last'] ).apply(lambda x: x%12)
 pass
print('Extracting month_real features for train:')
 extract_features(train)
 print('Extracting month_real features for test:')
 extract_features(test)
 print('train ',train.shape)
 print('test ',test.shape )
features['LandContour'] = features['LandContour'].replace(dict(Lvl=4, Bnk=3, HLS=2, Low=1))
features['LandSlope'] = features['LandSlope'].replace(dict(Gtl=3, Mod=2, Sev=1))
features['ExterQual'] =features['ExterQual'].replace(dict(Ex=5, Gd=4, TA=3, Fa=2, Po=1))
features['ExterCond'] =features['ExterCond'].replace(dict(Ex=5, Gd=4, TA=3, Fa=2, Po=1))
features['BsmtQual'] = features['BsmtQual'].replace(dict(Ex=5, Gd=4, TA=3, Fa=2, Po=1))
features['BsmtCond'] =features['BsmtCond'].replace(dict(Ex=5, Gd=4, TA=3, Fa=2, Po=1))
features['BsmtCond'] =features['BsmtCond'].replace('None',0)
 features['BsmtExposure'] =features['BsmtExposure'].replace(dict(Gd=4, Av=3, Mn=2, No=1))
features['BsmtExposure'] =features['BsmtExposure'].replace('None',0)
 features['BsmtFinType1'] = features['BsmtFinType1'].replace(dict(GLQ=6, ALQ=5, BLQ=4, Rec=3, LwQ=2, Unf=1))
features['BsmtFinType1'] =features['BsmtFinType1'].replace('None',0)
 features['BsmtFinType2'] = features['BsmtFinType2'].replace(dict(GLQ=6, ALQ=5, BLQ=4, Rec=3, LwQ=2, Unf=1))
features['BsmtFinType2'] = features['BsmtFinType2'].replace('None',0)
 features['HeatingQC'] = features['HeatingQC'].replace(dict(Ex=5, Gd=4, TA=3, Fa=2, Po=1))
features['CentralAir'] = features['CentralAir'].replace(dict(Y=1, N=0))
features['KitchenQual'] =features['KitchenQual'].replace(dict(Ex=5, Gd=4, TA=3, Fa=2, Po=1))
features['Functional'] = features['Functional'].replace(dict(Typ=8, Min1=7, Min2=6, Mod=5, Maj1=4, Maj2=3, Sev=2, Sal=1))
features['FireplaceQu'] = features['FireplaceQu'].replace(dict(Ex=5, Gd=4, TA=3, Fa=2, Po=1))
features['FireplaceQu'] = features['FireplaceQu'].replace('None', 0)
 features['GarageQual'] = features['GarageQual'].replace(dict(Ex=5, Gd=4, TA=3, Fa=2, Po=1))
features['GarageQual'] =features['GarageQual'].replace('None', 0)
 features['GarageCond'] = features['GarageCond'].replace(dict(Ex=5, Gd=4, TA=3, Fa=2, Po=1))
features['GarageCond'] = features['GarageCond'].replace('None', 0)
 features['LotShape'] = features['LotShape'].replace(dict(Reg=4, IR1=3, IR2=2, IR3=1))
features['GarageFinish'] = features['GarageFinish'].replace(dict(Fin=3, RFn=2, Unf=1))
features['GarageFinish'] = features['GarageFinish'].replace('None', 0)
 features['PavedDrive'] =features['PavedDrive'].replace(dict(Y=3, P=2, N=1))
features =features.astype({"Functional": int} )<data_type_conversions>
y = np.log1p(train['target'].values)
 f = [
'en_US_description_tfidf_0', 'en_US_description_tfidf_1', 'en_US_description_tfidf_2', 'en_US_description_tfidf_3', 'en_US_description_tfidf_4',
'dim_pca_0', 'dim_pca_1', 'dim_pca_2', 'dim_pca_3','dim_pca_4',
'dim_pca_5', 'dim_pca_6', 'dim_pca_7', 'dim_pca_8','dim_pca_9',
'sales_quantity_sum',
'sales_quantity_mean',
'fr_FR_price',
'product_type',
'month',
'month_real',
'product_gender',
'macro_function',
'function',
'macro_material',
'color',
'Month_transaction_last',
'sales_quantity_sum_linear_trend_rvalue',
'sales_quantity_sum_linear_trend_intercept',
'sales_quantity_sum_linear_trend_attr_slope',
'sales_quantity_sum_linear_trend_attr_stderr',
'sales_quantity_sum_kurtosis',
'sales_quantity_sum_skewness',
'BYMODELsales_quantity_sum_linear_trend_rvalue',
'BYMODELsales_quantity_sum_linear_trend_intercept',
'BYMODELsales_quantity_sum_linear_trend_attr_slope',
'BYMODELsales_quantity_sum_linear_trend_attr_stderr',
'BYMODELsales_quantity_sum_kurtosis',
'BYMODELsales_quantity_sum_skewness',
]
features['BsmtQual'] =features['BsmtQual'].replace('None',0)
 features =features.astype({"KitchenQual": int})
 features =features.astype({"Functional": int})
<categorify>
clfs, importances, err, oof_predict = train_classifiers(train,y,f)
features['MSSubClass'] =features['MSSubClass'].astype('category')
 features['MoSold'] = features['MoSold'].astype('category')
 features['YrSold'] =features['YrSold'].astype('category')
 cat_cols = ['MSZoning','Street','MSSubClass', 'MoSold', 'YrSold', 'LotConfig', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'Heating', 'Electrical', 'GarageType', 'SaleType', 'SaleCondition']
dumies=pd.get_dummies(features[cat_cols],drop_first=True)
 dumies.head()<drop_column>
preds_ = None
for clf in clfs:
if preds_ is None:
preds_ = clf.predict(test[f], num_iteration=clf.best_iteration_)
 else:
preds_ += clf.predict(test[f], num_iteration=clf.best_iteration_)
 preds_ = preds_ / len(clfs)
 preds_ = np.expm1(preds_)
 subm = pd.DataFrame()
subm['ID'] = test['ID'].values
subm['target'] = preds_
subm.to_csv('submission{}.csv'.format(err), index=False)