Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<count_values>
!pip install googletrans
!pip install pytrends
train.month.value_counts()<feature_engineering>
%matplotlib inline
data['description'] = data['description'].fillna("")
 data['description_len'] = data['description'].apply(lambda x : len(x.split()))
train = data[data.deal_probability.notnull() ]<feature_engineering>
pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 ori_data_path = '/kaggle/input/competitive-data-science-predict-future-sales/'
model_path = '/'
processed_data_path = '/'
tables_path = '/'
items_IP_path = '.. /input/predict-future-sales-ip-groups/items_IPs_v1.csv'
items_IP_trend_path = '.. /input/predict-future-sales-ip-groups/items_IP_trend.csv'
data['title'] = data['title'].fillna(" ")
 data['title_len'] = data['title'].apply(lambda x : len(x.split()))
train = data[data.deal_probability.notnull() ]<count_values>
df = pd.read_csv(ori_data_path+'sales_train.csv')
 df_test = pd.read_csv(ori_data_path+'test.csv')
 sample_submission = pd.read_csv(ori_data_path+'sample_submission.csv')
 item_categories = pd.read_csv(ori_data_path+'item_categories.csv')
 items = pd.read_csv(ori_data_path+'items.csv')
 shops = pd.read_csv(ori_data_path+'shops.csv' )
train.title_len.value_counts()<feature_engineering>
df[['date_block_num','shop_id']] = df[['date_block_num','shop_id']].astype('int8')
 df[['item_cnt_day','item_id']] = df[['item_cnt_day','item_id']].astype('int16')
 df['item_price'] = df['item_price'].astype('float32' )
data['title_len_char'] = data['title'].str.len()
train = data[data.deal_probability.notnull() ]<count_values>
shop_test = df_test['shop_id'].nunique()
item_test = df_test['item_id'].nunique()
total_product_test = df_test[['shop_id','item_id']].drop_duplicates()
shop_train = df['shop_id'].nunique()
item_train = df['item_id'].nunique()
total_product_train = df[['shop_id','item_id']].drop_duplicates()
print('We need to predict products from {} shops and {} kinds of products(total shop-item {} pairs)'.format(shop_test,item_test,total_product_test.shape[0]))
print('given that historical daily sale number from {} shops and {} kinds of products(total shop-item {} pairs)'.format(shop_train,item_train,total_product_train.shape[0]))
train.title_len_char.value_counts()<feature_engineering>
common_pairs = total_product_test.merge(total_product_train,how='inner',on=['shop_id','item_id'] ).shape[0]
test_only_pairs = total_product_test.shape[0] - common_pairs
print(f'{common_pairs} pairs of products are shared by historical and to be predicted.')
 print(f'{test_only_pairs} pairs are new pairs, so there is no corresponding historical data for prediciton.' )
text_cols = ['description', 'param_1', 'param_2', 'param_3','title']
data["total_text"] = data[text_cols].fillna("" ).sum(axis=1 )<normalization>
total_items_test = total_product_test['item_id'].drop_duplicates()
total_items_train = total_product_train['item_id'].drop_duplicates()
total_items_join = pd.merge(total_items_test,total_items_train,how='outer',on=['item_id'],indicator=True)
 total_items_join = total_items_join.replace({'both':'hisory and target','left_only':'target only','right_only':'hisory only'})
 total_items_join['_merge'].value_counts()
tfidf = TfidfVectorizer(ngram_range=(1,1)).fit_transform(data.total_text)
<define_variables>
del shop_test
del item_test
del total_product_test
del shop_train
del item_train
del total_product_train
del common_pairs
del test_only_pairs
del total_items_test
del total_items_train
del total_items_join
gc.collect()
data.index = range(data.shape[0] )<prepare_output>
del shop_item_pair_num_month
del ax
del all_pairs
del all_existed_pair_month
del all_items
del all_shops
del item_trend
del item_1
del item_2
del item_3
del fig
del ax1
del ax2
del ax3
gc.collect()
n_comps = 10
tsvd = pd.DataFrame(TruncatedSVD(n_components=n_comps, algorithm='arpack' ).fit_transform(tfidf),
columns=["svd_comp_" + str(i)for i in range(n_comps)], index=data.index)
 tsvd.head()<merge>
cnt_month = df.groupby(['shop_id','item_id','date_block_num'] ).sum() ['item_cnt_day']
cnt_month.name = 'item_cnt_month'
dataset = dataset.merge(cnt_month,on=['shop_id','item_id','date_block_num'],how='left')
 dataset['item_cnt_month'] = dataset['item_cnt_month'].fillna(0 ).clip(0,20)
 del cnt_month
gc.collect()
data = data.join(tsvd)
 train = data[data.deal_probability.notnull() ]<import_modules>
df_test['date_block_num'] = 34
dataset = pd.concat([dataset,df_test.drop('ID',axis=1)],axis=0 ).reset_index(drop=True)
 assert dataset.isnull().sum().max() == len(df_test)
 dataset = dataset.fillna(0 )
from keras.preprocessing import image
from keras.applications.vgg16 import VGG16
from keras.applications.resnet50 import ResNet50
from keras.applications.xception import Xception
from keras.applications.inception_v3 import InceptionV3
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot<load_pretrained>
float32_cols = ['revenue','avg_price']
int16_cols =['item_id']
int8_cols = ['date_block_num','shop_id','item_cnt_month']
dataset[int8_cols] = dataset[int8_cols].astype('int8')
 dataset[int16_cols] = dataset[int16_cols].astype('int16')
 dataset[float32_cols] = dataset[float32_cols].astype('float32' )
model = VGG16(weights=None, include_top=False)
 model.load_weights('.. /input/keras-pretrained-models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5' )<import_modules>
def add_lag_v3(dataset,lookup_set,lag_months,lag_features_list,on_key,fillna_value,dtypes):

 if lookup_set is None:
lookup_set = dataset[on_key + lag_features_list].copy()
else:
dataset = pd.merge(dataset,lookup_set,how='left',on=on_key)
 for lag in lag_months:
dataset_alignment = dataset[on_key + lag_features_list].copy()
dataset_lag = lookup_set[on_key + lag_features_list]
dataset_lag['date_block_num'] = dataset_lag['date_block_num'] + lag
dataset_alignment = pd.merge(dataset_alignment,dataset_lag,how='left',on=on_key,suffixes=['','_lag_'+str(lag)+'m'])
 dataset_alignment = dataset_alignment.drop(on_key + lag_features_list,axis=1)
 dataset = pd.concat([dataset,dataset_alignment],axis=1,join='outer')
 del dataset_lag
del dataset_alignment
gc.collect()
for i,feature in enumerate(lag_features_list):
new_columns = dataset.columns.str.startswith(feature + '_lag_')
 dataset.loc[:,new_columns] = dataset.loc[:,new_columns].fillna(fillna_value[i] ).astype(dtypes[i])
 return dataset
from keras.applications.vgg16 import preprocess_input
import zipfile
import cv2<load_pretrained>
def add_first_last_sale(dataset,source_set,keys,prefix):

 for m in range(1,35):
tmp = source_set[source_set['date_block_num']<m].drop_duplicates(keys+['date_block_num'] ).groupby(keys ).agg({'date_block_num':[np.min,np.max]})
 tmp.columns = [prefix + '_first_sold', prefix + '_last_sold']
tmp = tmp.reset_index()
tmp['date_block_num'] = m
if m == 1:
first_last_sold = tmp
else:
first_last_sold = pd.concat([first_last_sold,tmp],axis=0)
 del tmp
gc.collect()
dataset = dataset.merge(first_last_sold,on=keys+['date_block_num'],how='left')
 dataset[prefix + '_first_sold_offset'] = dataset[prefix + '_first_sold'] - dataset['date_block_num']
dataset[prefix + '_last_sold_offset'] = dataset[prefix + '_last_sold'] - dataset['date_block_num']
columns = [prefix + '_first_sold', prefix + '_last_sold',prefix + '_first_sold_offset', prefix + '_last_sold_offset']
dataset[columns] = dataset[columns].fillna(0)
 dataset[columns] = dataset[columns].astype('int8')
 return dataset
myzip = zipfile.ZipFile('.. /input/avito-demand-prediction/train_jpg.zip')
 files_in_zip = myzip.namelist()
print(files_in_zip[:5])
 print("total", len(files_in_zip))<set_options>
def create_mean_encoding_table(dataset:pd.DataFrame, encode_item:list, prefix:str):
encoding_table = dataset[encode_item +['date_block_num','item_cnt_month','revenue','avg_price']]
encoding_table.loc[:,['item_cnt_month','revenue','avg_price']] = encoding_table.loc[:,['item_cnt_month','revenue','avg_price']].replace(0,np.nan)
 encoding_table = encoding_table.groupby(encode_item + ['date_block_num'] ).agg({'item_cnt_month':[np.mean],'revenue':[np.mean],'avg_price':[np.mean]})
 encoding_table = encoding_table.fillna(0)
 encoding_table.columns = [prefix+'_avg_cnt_month', prefix+'_avg_revenue_month',prefix+'_avg_price_month']
encoding_table = encoding_table.reset_index()
encoding_table[['date_block_num',prefix+'_avg_cnt_month']] = encoding_table[['date_block_num',prefix+'_avg_cnt_month']].astype('float16')
 encoding_table[encode_item] = encoding_table[encode_item].astype('int16')
 encoding_table[[prefix+'_avg_revenue_month',prefix+'_avg_price_month']] = encoding_table[[prefix+'_avg_revenue_month',prefix+'_avg_price_month']].astype('float32')
 return encoding_table
myzip.close()<import_modules>
def first_sale_cnt(dataset,source_set,keys,prefix,forward_range=6):

 source_set = source_set[keys+['date_block_num','item_first_sold_offset','item_cnt_month']]
source_set = source_set[source_set['item_first_sold_offset']==0]
source_set = source_set.drop('item_first_sold_offset',axis=1)
 source_set = source_set[source_set['item_cnt_month']>0]
for m in range(1,35):
tmp = source_set[source_set['date_block_num'].between(m-1-6,m-1)].groupby(keys ).agg({'item_cnt_month':[np.mean]})
 tmp.columns = [prefix + '_avg_first_sold_cnt_month']
tmp = tmp.reset_index()
tmp['date_block_num'] = m
if m == 1:
avg_first_sold_cnt = tmp
else:
avg_first_sold_cnt = pd.concat([avg_first_sold_cnt,tmp],axis=0)
 del tmp
dataset = dataset.merge(avg_first_sold_cnt,on=keys+['date_block_num'],how='left')
 columns = [prefix + '_avg_first_sold_cnt_month']
dataset[columns] = dataset[columns].fillna(0)
 dataset[columns] = dataset[columns].astype('float16')
 return dataset
from time import time<feature_engineering>
def get_google_trends(IPs, file_path, reference):

 pytrend = TrendReq(hl='ru', tz=360)
 try:
IP_trend = pd.read_csv(file_path)
 existed_columns = IP_trend.columns
except FileNotFoundError:
IP_trend = None
existed_columns = []
for IP in IPs:
if IP in existed_columns:
continue
if IP == reference:
keywords = [reference]
else:
keywords = [IP,reference]
pytrend.build_payload(kw_list=keywords,cat=0,timeframe='2013-01-01 2015-10-31',geo='RU',gprop='')
 trend_df = pytrend.interest_over_time()
trend_df = trend_df.drop(reference,axis=1)
 if len(trend_df)== 0:
if IP_trend is not None:
IP_trend[IP] = 0
else:
trend_df = trend_df.reset_index()
trend_df['year'] = trend_df['date'].dt.year
trend_df['month'] = trend_df['date'].dt.month
trend_df = trend_df.drop(['date','isPartial'],axis=1)
 trend_df = trend_df.groupby(['year','month'] ).max().reset_index()
if IP_trend is None:
IP_trend = trend_df
else:
IP_trend = IP_trend.merge(trend_df,on=['year','month'],how='left')
 return IP_trend
for c in obj_columns:
data[c] = pd.factorize(data[c])[0]<count_missing_values>
%%time
lag_months = [1,2,3,6,12]
lag_features_list = ['item_cnt_month']
on_key = ['shop_id','item_id','date_block_num']
fillna = [0]
dtypes = ['int8']
dataset = add_lag_v3(dataset,None,lag_months,lag_features_list,on_key,fillna,dtypes )
data.isnull().sum(axis=0 )<drop_column>
%%time
lag_months = [1]
lag_features_list = ['avg_price']
on_key = ['shop_id','item_id','date_block_num']
fillna = [0]
dtypes = ['float32']
dataset = add_lag_v3(dataset,None,lag_months,lag_features_list,on_key,fillna,dtypes )
test = data[data.deal_probability.isnull() ].drop("deal_probability", axis=1)
 train = data.drop(test.index)
 del data<import_modules>
dataset = add_first_last_sale(dataset,df,['shop_id','item_id'],'pair')
 dataset = dataset.drop(['pair_first_sold','pair_last_sold'],axis=1 )
from sklearn.model_selection import KFold
from lightgbm import LGBMRegressor<train_model>
%%time
item_month_sale = create_mean_encoding_table(dataset, ['item_id'], 'item')
 lag_months = [1]
lag_features_list = ['item_avg_cnt_month', 'item_avg_revenue_month', 'item_avg_price_month']
on_key = ['item_id','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,item_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['item_avg_cnt_month','item_avg_revenue_month', 'item_avg_price_month'],axis=1)
 del item_month_sale
gc.collect()
LGBMRegressor.fit?<train_on_grid>
%%time
dataset = add_first_last_sale(dataset,df,['item_id'],'item')
 dataset = dataset.drop(['item_first_sold','item_last_sold'],axis=1 )
def train_CV(train, test, folds, **params):
test_preds = []
train_preds = []
for i,(train_idx, test_idx)in enumerate(folds):
X_train = train.drop("deal_probability", axis=1 ).iloc[train_idx]
y_train = train.deal_probability.iloc[train_idx]
X_valid = train.drop("deal_probability", axis=1 ).iloc[test_idx]
y_valid = train.deal_probability.iloc[test_idx]
learner = LGBMRegressor(n_estimators=10000, **params)
 learner.fit(X_train, y_train, early_stopping_rounds=10,
eval_metric="rmse", verbose=100,
eval_set=[(X_train, y_train), 
(X_valid, y_valid)] 
)
 preds = pd.Series(learner.predict(X_valid), index=X_valid.index, name="deal_probability")
 train_preds.append(preds)
 preds = pd.Series(learner.predict(test), index=test.index, name="fold_" + str(i))
test_preds.append(preds)
 return pd.concat(train_preds ).clip(0, 1),\
pd.concat(test_preds, axis=1 ).mean(axis=1 ).clip(0, 1 ).rename("deal_probability" )<train_on_grid>
dataset = pd.merge(dataset,items[['item_id','item_category_id']],how='left',on=['item_id'])
 dataset['item_category_id'] = dataset['item_category_id'].astype('int16' )
folds = list(KFold(n_splits=5, shuffle=True ).split(train))
res = pd.Series(index=[2 ** i for i in range(5, 11)])
 best_res = 10
for nl in res.index:
print("*" * 20)
 print("doing", nl)
 train_preds, test_preds = train_CV(train, test, folds, num_leaves=nl)
 res = np.power(np.power(train.deal_probability - train_preds.loc[train.index], 2 ).mean() , 0.5)
 if res < best_res:
print("*" * 20)
 print("*" * 20)
 print("got best with nl {}: {}".format(nl, res))
print("*" * 20)
 print("*" * 20)
 best_res = res
best_test_preds = test_preds<save_to_csv>
try:
item_categories_plus = pd.read_csv(tables_path + 'item_categories_plus.csv')
 except FileNotFoundError:
item_categories_plus = item_categories.copy()
trans = Translator()
item_categories_plus['item_category_name_en'] = item_categories_plus['item_category_name'].apply(lambda name: trans.translate(name,dest='en' ).text)
 item_categories_plus
item_categories_plus.to_csv(tables_path + 'item_categories_plus.csv')
 item_categories_plus['main_category'] = item_categories_plus['item_category_name_en'].apply(lambda name: name.split('-')[0].strip())
 item_categories_plus['sub_category'] = item_categories_plus['item_category_name_en'].apply(
 lambda name:(name.split('-')[1].strip())if len(name.split('-')) >1 else(name.split('-')[0].strip()))
item_categories_plus.loc[24,'main_category'] = 'Games'
item_categories_plus.loc[71,'main_category'] = 'Platic Bags and Gift'
item_categories_plus.loc[78,'main_category'] = 'Program'
item_categories_plus.loc[32,'main_category'] = 'Payment cards'
item_categories_plus.loc[32,'sub_category'] = '(Movies, Music, Games)'
item_categories_plus.loc[36,'main_category'] = 'Payment cards'
item_categories_plus.loc[37:41,'main_category'] = 'Movies'
main_category_encoder = LabelEncoder()
item_categories_plus['main_category_enc'] = main_category_encoder.fit_transform(item_categories_plus['main_category'] ).astype('int8')
 sub_category_encoder = LabelEncoder()
item_categories_plus['sub_category_enc'] = sub_category_encoder.fit_transform(item_categories_plus['sub_category'] ).astype('int8')
 dataset = pd.merge(dataset,item_categories_plus[['item_category_id','main_category_enc','sub_category_enc']],how='left',on=['item_category_id'])
 del item_categories_plus
gc.collect()
best_test_preds.to_csv("preds.csv", header=True, index=True )<import_modules>
import numpy as np
import pandas as p
from matplotlib import pyplot as plt
import seaborn as sns
from math import sqrt
from sklearn import metrics<load_from_csv>
%%time
main_cate_month_sale = create_mean_encoding_table(dataset, ['main_category_enc'], 'main_cate')
 lag_months = [1]
lag_features_list = ['main_cate_avg_cnt_month', 'main_cate_avg_revenue_month', 'main_cate_avg_price_month']
on_key = ['main_category_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,main_cate_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['main_cate_avg_cnt_month','main_cate_avg_revenue_month', 'main_cate_avg_price_month'],axis=1)
 del main_cate_month_sale
gc.collect()
df_base0 = p.read_csv('.. /input/public-solutions/base0_0.2211.csv',names=["item_id","deal_probability0"], skiprows=[0],header=None)
 df_base1 = p.read_csv('.. /input/public-solutions/base1_0.2212.csv',names=["item_id","deal_probability1"], skiprows=[0],header=None)
 df_base2 = p.read_csv('.. /input/public-solutions/base2_0.2212.csv',names=["item_id","deal_probability2"], skiprows=[0],header=None)
 df_base3 = p.read_csv('.. /input/public-solutions/base3_0.2213.csv',names=["item_id","deal_probability3"], skiprows=[0],header=None)
 df_base4 = p.read_csv('.. /input/public-solutions/base4_0.2215.csv',names=["item_id","deal_probability4"], skiprows=[0],header=None)
 df_base5 = p.read_csv('.. /input/public-solutions/base5_0.2219.csv',names=["item_id","deal_probability5"], skiprows=[0],header=None)
 df_base6 = p.read_csv('.. /input/public-solutions/base6_0.2220.csv',names=["item_id","deal_probability6"], skiprows=[0],header=None)
 df_base7 = p.read_csv('.. /input/public-solutions/base7_0.2222.csv',names=["item_id","deal_probability7"], skiprows=[0],header=None)
 df_base8 = p.read_csv('.. /input/public-solutions/base8_0.2224.csv',names=["item_id","deal_probability8"], skiprows=[0],header=None)
 df_base9 = p.read_csv('.. /input/public-solutions/base9_0.2226.csv',names=["item_id","deal_probability9"], skiprows=[0],header=None)
 df_base10 = p.read_csv('.. /input/public-solutions/base10_0.2227.csv',names=["item_id","deal_probability10"], skiprows=[0],header=None)
 df_base11 = p.read_csv('.. /input/public-solutions/base11_0.2228.csv',names=["item_id","deal_probability11"], skiprows=[0],header=None)
 df_base12 = p.read_csv('.. /input/public-solutions/base12_0.2230.csv',names=["item_id","deal_probability12"], skiprows=[0],header=None)
 df_base13 = p.read_csv('.. /input/public-solutions/base13_0.2232.csv',names=["item_id","deal_probability13"], skiprows=[0],header=None)
 df_base14 = p.read_csv('.. /input/public-solutions/base14_0.2237.csv',names=["item_id","deal_probability14"], skiprows=[0],header=None)
 df_base15 = p.read_csv('.. /input/public-solutions/base15_0.2237.csv',names=["item_id","deal_probability15"], skiprows=[0],header=None)
 df_base16 = p.read_csv('.. /input/public-solutions/base16_0.2238.csv',names=["item_id","deal_probability16"], skiprows=[0],header=None)
 df_base17 = p.read_csv('.. /input/public-solutions/base17_0.2239.csv',names=["item_id","deal_probability17"], skiprows=[0],header=None)
 df_base18 = p.read_csv('.. /input/public-solutions/base18_0.2246.csv',names=["item_id","deal_probability18"], skiprows=[0],header=None)
 df_base19 = p.read_csv('.. /input/public-solutions/base19_0.2247.csv',names=["item_id","deal_probability19"], skiprows=[0],header=None)
 df_base20 = p.read_csv('.. /input/public-solutions/base20_0.2249.csv',names=["item_id","deal_probability20"], skiprows=[0],header=None)
 df_base21 = p.read_csv('.. /input/public-solutions/base21_0.2255.csv',names=["item_id","deal_probability21"], skiprows=[0],header=None)
 df_base22 = p.read_csv('.. /input/public-solutions/base22_0.2255.csv',names=["item_id","deal_probability22"], skiprows=[0],header=None)
 df_base23 = p.read_csv('.. /input/public-solutions/base23_0.2269.csv',names=["item_id","deal_probability23"], skiprows=[0],header=None )<merge>
%%time
sub_cate_month_sale = create_mean_encoding_table(dataset, ['sub_category_enc'], 'sub_cate')
 lag_months = [1]
lag_features_list = ['sub_cate_avg_cnt_month', 'sub_cate_avg_revenue_month', 'sub_cate_avg_price_month']
on_key = ['sub_category_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,sub_cate_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['sub_cate_avg_cnt_month','sub_cate_avg_revenue_month', 'sub_cate_avg_price_month'],axis=1)
 del sub_cate_month_sale
gc.collect()
df_base = p.merge(df_base0,df_base1,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base2,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base3,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base4,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base5,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base6,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base7,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base8,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base9,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base10,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base11,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base12,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base13,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base14,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base15,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base16,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base17,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base18,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base19,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base20,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base21,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base22,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base23,how='inner',on='item_id' )<compute_test_metric>
%%time
dataset = first_sale_cnt(dataset,dataset,['item_category_id'],'cate')
 dataset = first_sale_cnt(dataset,dataset,['main_category_enc'],'main_cate')
 dataset = first_sale_cnt(dataset,dataset,['sub_category_enc'],'sub_cate' )
M = np.zeros([df_base.iloc[:,1:].shape[1],df_base.iloc[:,1:].shape[1]])
 for i in np.arange(M.shape[1]):
for j in np.arange(M.shape[1]):
M[i,j] = sqrt(metrics.mean_squared_error(df_base.iloc[:,i+1], df_base.iloc[:,j+1]))<merge>
date_features = df.drop_duplicates(['date_block_num'])[['date','date_block_num']]
date_features['year'] = date_features['date'].apply(lambda d: d.split('.')[2])
 date_features['month'] = date_features['date'].apply(lambda d: d.split('.')[1])
 date_features = date_features.drop('date',axis=1 ).reset_index(drop=True)
 test_date = [34,2015,11]
date_features.loc[34] = test_date
dataset = pd.merge(dataset,date_features,how='left',on=['date_block_num'])
 dataset[['month','year']] = dataset[['month','year']].astype('int16')
df_base = p.merge(df_base0,df_base1,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base2,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base14,how='inner',on='item_id')
 df_base = p.merge(df_base,df_base18,how='inner',on='item_id' )<compute_test_metric>
dataset['shop_id'] = dataset['shop_id'].replace({0:57,1:58,10:11})
 df['shop_id'] = df['shop_id'].replace({0:57,1:58,10:11} )
M = np.zeros([df_base.iloc[:,1:].shape[1],df_base.iloc[:,1:].shape[1]])
 for i in np.arange(M.shape[1]):
for j in np.arange(M.shape[1]):
M[i,j] = sqrt(metrics.mean_squared_error(df_base.iloc[:,i+1], df_base.iloc[:,j+1]))<feature_engineering>
try:
shops_plus = pd.read_csv(tables_path + 'shops_plus.csv')
 except FileNotFoundError:
shops_plus = shops.copy()
trans = Translator()
shops_plus['shop_name_en'] = shops_plus['shop_name'].apply(lambda name: trans.translate(name,dest='en' ).text)
 shops_plus.to_csv(tables_path + 'shops_plus.csv')
 shops_plus['shop_city'] = shops_plus['shop_name_en'].apply(lambda shop: shop.split(' ')[0].strip())
 shops_plus.loc[0:1,'shop_city'] = 'Yakutsk'
shops_plus.loc[12,'shop_city'] = 'Online_emergency'
shops_plus.loc[20,'shop_city'] = 'Sale'
shops_plus.loc[42:43,'shop_city'] = 'St.Petersburg'
shop_enc = LabelEncoder()
shops_plus['shop_city_enc'] = shop_enc.fit_transform(shops_plus['shop_city'])
 shops_plus['shop_city_enc'] = shops_plus['shop_city_enc'].astype('int8')
 dataset = dataset.merge(shops_plus[['shop_id','shop_city_enc']],how='left',on=['shop_id'])
 del shops_plus
gc.collect()
df_base['deal_probability'] = df_base.iloc[:,1:].mean(axis=1 )<save_to_csv>
dataset = add_first_last_sale(dataset,df,['shop_id'],'shop')
 dataset = dataset.drop(['shop_first_sold','shop_last_sold'],axis=1 )
df_base[['item_id','deal_probability']].to_csv("best_public_blend.csv",index=False )<set_options>
%%time
shop_month_sale = create_mean_encoding_table(dataset, ['shop_id'], 'shop')
 lag_months = [1,2,3,6,12]
lag_features_list = ['shop_avg_cnt_month', 'shop_avg_revenue_month', 'shop_avg_price_month']
on_key = ['shop_id','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,shop_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['shop_avg_cnt_month', 'shop_avg_revenue_month','shop_avg_price_month'],axis=1)
 del shop_month_sale
gc.collect()
h2o.init(max_mem_size = "10G" )<load_from_csv>
%%time
shop_city_month_sale = create_mean_encoding_table(dataset, ['shop_city_enc'], 'shop_city')
 lag_months = [1]
lag_features_list = ['shop_city_avg_cnt_month', 'shop_city_avg_revenue_month', 'shop_city_avg_price_month']
on_key = ['shop_city_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,shop_city_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['shop_city_avg_cnt_month','shop_city_avg_revenue_month','shop_city_avg_price_month'],axis=1)
 del shop_city_month_sale
gc.collect()
print("
Data Load Stage")
 training = pd.read_csv('.. /input/train.csv', index_col = "item_id", parse_dates = ["activation_date"])
 traindex = training.index
testing = pd.read_csv('.. /input/test.csv', index_col = "item_id", parse_dates = ["activation_date"])
 testdex = testing.index
y = training.deal_probability.copy()
print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))<feature_engineering>
df = pd.concat([training,testing],axis=0)
 del training, testing
gc.collect()
print('
All Data shape: {} Rows, {} Columns'.format(*df.shape))
print("Feature Engineering")
 df["price"] = np.log(df["price"]+0.001)
 df["price"].fillna(-999,inplace=True)
 df["image_top_1"].fillna(-999,inplace=True)
 print("
Create Time Variables")
 df["Weekday"] = df['activation_date'].dt.weekday
df["Weekd of Year"] = df['activation_date'].dt.week
df["Day of Month"] = df['activation_date'].dt.day
df.drop(["activation_date","image"],axis=1,inplace=True)
 print("
Encode Variables")
 categorical = ["user_id","region","city","parent_category_name","category_name","item_seq_number","user_type","image_top_1"]
messy_categorical = ["param_1","param_2","param_3","title","description"]
print("Encoding :",categorical + messy_categorical)
 lbl = preprocessing.LabelEncoder()
for col in categorical + messy_categorical:
df[col] = lbl.fit_transform(df[col].astype(str))
X = df.loc[traindex,:].copy()
print("Training Set shape",X.shape)
 test = df.loc[testdex,:].copy()
print("Submission Set Shape: {} Rows, {} Columns".format(*test.shape))
del df
gc.collect() 
<drop_column>
%%time
shop_main_cate_month_sale = create_mean_encoding_table(dataset, ['shop_id','main_category_enc'], 'shop_main_cate')
 lag_months = [1]
lag_features_list = ['shop_main_cate_avg_cnt_month', 'shop_main_cate_avg_revenue_month', 'shop_main_cate_avg_price_month']
on_key = ['shop_id','main_category_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,shop_main_cate_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['shop_main_cate_avg_cnt_month','shop_main_cate_avg_revenue_month','shop_main_cate_avg_price_month'],axis=1)
 del shop_main_cate_month_sale
gc.collect()
test.drop("deal_probability",axis=1, inplace=True )<split>
%%time
shop_sub_cate_month_sale = create_mean_encoding_table(dataset, ['shop_id','sub_category_enc'], 'shop_sub_cate')
 lag_months = [1,2,3,6,12]
lag_features_list = ['shop_sub_cate_avg_cnt_month', 'shop_sub_cate_avg_revenue_month', 'shop_sub_cate_avg_price_month']
on_key = ['shop_id','sub_category_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,shop_sub_cate_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['shop_sub_cate_avg_cnt_month','shop_sub_cate_avg_revenue_month','shop_sub_cate_avg_price_month'],axis=1)
 del shop_sub_cate_month_sale
gc.collect()
<create_dataframe>
%%time
item_IP = pd.read_csv(items_IP_path)
 IP_label = LabelEncoder()
item_IP['IP_encode'] = IP_label.fit_transform(item_IP['Search_Key_Word'])
 dataset = dataset.merge(item_IP[['item_id','IP_encode']],on='item_id',how='left')
 dataset['IP_encode'] = dataset['IP_encode'].astype('int16')
 IPs = list(item_IP['Search_Key_Word'].unique())
 IPs.remove('Others')
 IP_trend = get_google_trends(IPs, items_IP_trend_path,'Total War')
 date_features[['year','month']] = date_features[['year','month']].astype('int64')
 IP_trend = IP_trend.merge(date_features,on=['year','month'],how='left')
 IP_trend = IP_trend.drop(['month','year'],axis=1)
 IP_trend = IP_trend.set_index('date_block_num' ).stack().reset_index()
IP_trend.columns = ['date_block_num','Search_Key_Word','google_trends']
IP_trend = IP_trend.merge(item_IP[['Search_Key_Word','IP_encode']].drop_duplicates() ,on='Search_Key_Word',how='left')
 IP_trend = IP_trend.dropna()
IP_trend = IP_trend.sort_values(by=['IP_encode','date_block_num'] )
htrain = h2o.H2OFrame(X)
 htest = h2o.H2OFrame(test )<predict_on_test>
%%time
lag_months = [1,2,3,6,12]
lag_features_list = ['google_trends']
on_key = ['IP_encode','date_block_num']
fillna = [0]
dtypes = ['int8']
dataset = add_lag_v3(dataset,IP_trend,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['Search_Key_Word','google_trends'],axis=1)
 del IP_trend
gc.collect()
x =htrain.columns
y ='deal_probability'
x.remove(y)
 aml = H2OAutoML(max_runtime_secs = 18000)
 aml.train(x=x, y =y, training_frame=htrain)
 print('Generate predictions...')
 htrain.drop(['deal_probability'])
<compute_test_metric>
%%time
IP_month_sale = create_mean_encoding_table(dataset, ['IP_encode'], 'IP')
 lag_months = [1]
lag_features_list = ['IP_avg_cnt_month', 'IP_avg_revenue_month', 'IP_avg_price_month']
on_key = ['IP_encode','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,IP_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['IP_avg_cnt_month', 'IP_avg_revenue_month','IP_avg_price_month'],axis=1)
 del IP_month_sale
gc.collect()
<set_options>
del IPs
del item_IP
del date_features
del lag_months
del lag_features_list
del on_key
del fillna
del dtypes
del IP_label
del LabelEncoder
del df
del df_test
del sample_submission
del items
del shops
gc.collect()
aml.leader<predict_on_test>
assert dataset.isnull().sum().sum() == 0
assert np.isfinite(dataset ).sum().sum() == dataset.shape[0] * dataset.shape[1]
dataset = dataset.drop(['avg_price','revenue'],axis=1)
 dataset.to_pickle(processed_data_path + 'dataset.pkl')
preds = aml.leader.predict(htest)
 preds = preds.as_data_frame()<save_to_csv>
dataset = pd.read_pickle(processed_data_path + 'dataset.pkl' )
lgsub = pd.DataFrame(preds.predict.values,columns=["deal_probability"],index=testdex)
 lgsub['deal_probability'].clip(0.0, 1.0, inplace=True)
 lgsub.to_csv("lgsub.csv",index=True,header=True)
 print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))<set_options>
selected = [
'item_cnt_month',
'date_block_num',
'item_cnt_month_lag_1m','item_cnt_month_lag_2m','item_cnt_month_lag_3m',
'item_cnt_month_lag_6m','item_cnt_month_lag_12m','pair_first_sold_offset',
'item_avg_cnt_month_lag_1m','item_first_sold_offset',
'main_cate_avg_cnt_month_lag_1m','main_cate_avg_revenue_month_lag_1m',
'sub_cate_avg_revenue_month_lag_1m','sub_cate_avg_cnt_month_lag_1m',
'cate_avg_first_sold_cnt_month','main_cate_avg_first_sold_cnt_month','sub_cate_avg_first_sold_cnt_month',
'month',
'shop_avg_cnt_month_lag_1m','shop_avg_cnt_month_lag_2m','shop_avg_cnt_month_lag_3m',
'shop_avg_cnt_month_lag_6m','shop_avg_cnt_month_lag_12m','shop_city_avg_cnt_month_lag_1m',
'shop_first_sold_offset','shop_last_sold_offset',
'shop_main_cate_avg_cnt_month_lag_1m','shop_sub_cate_avg_cnt_month_lag_1m','shop_sub_cate_avg_cnt_month_lag_2m',
'shop_sub_cate_avg_cnt_month_lag_3m','shop_sub_cate_avg_cnt_month_lag_6m','shop_sub_cate_avg_cnt_month_lag_12m',
'IP_avg_cnt_month_lag_1m','IP_avg_revenue_month_lag_1m',
'google_trends_lag_1m','google_trends_lag_2m','google_trends_lag_3m',
'google_trends_lag_6m','google_trends_lag_12m',
]
dataset = dataset[selected]
sns.set()
%matplotlib inline
NFOLDS = 5
SEED = 42 
<load_from_csv>
train = dataset[dataset['date_block_num'].between(12,32)]
validate = dataset[dataset['date_block_num']==33]
test = dataset[dataset['date_block_num']==34]
del dataset
gc.collect()
train = pd.read_csv('.. /input/avito-demand-prediction/train.csv')
 test = pd.read_csv('.. /input/avito-demand-prediction/test.csv')
 gp = pd.read_csv('.. /input/aggregated-features-lightgbm/aggregated_features.csv')
 train = train.merge(gp, on='user_id', how='left')
 test = test.merge(gp, on='user_id', how='left')
 agg_cols = list(gp.columns)[1:]
del gp
gc.collect()
train.head()<feature_engineering>
X_train = train.drop(['item_cnt_month','date_block_num'],axis=1)
 y_train = train['item_cnt_month']
X_validate = validate.drop(['item_cnt_month','date_block_num'],axis=1)
 y_validate = validate['item_cnt_month']
X_test = test.drop(['item_cnt_month','date_block_num'],axis=1)
 del train
del test
del validate
gc.collect()
for df in [train, test]:
df['description'].fillna('unknowndesc', inplace=True)
 df['title'].fillna('unknowntitle', inplace=True)
 df['weekday'] = pd.to_datetime(df['activation_date'] ).dt.day
for col in ['description', 'title']:
df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))
df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100
df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100
df['city'] = df['region'] + '_' + df['city']
df['num_desc_punct'] = df['description'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
for col in agg_cols:
df[col].fillna(-1, inplace=True )<feature_engineering>
print(f'training shape: X {X_train.shape}, y {y_train.shape}')
 print(f'validation shape: X {X_validate.shape}, y {y_validate.shape}')
 print(f'testing shape: X {X_test.shape}')
 print(f'training missing: X {X_train.isnull().sum().sum() }, y {y_train.isnull().sum().sum() }')
 print(f'validation missing: X {X_validate.isnull().sum().sum() }, y {y_validate.isnull().sum().sum() }')
 print(f'testing missing: X {X_test.isnull().sum().sum() }' )
count_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True, min_df=2)
 title_counts = count_vectorizer_title.fit_transform(train['title'].append(test['title']))
train_title_counts = title_counts[:len(train)]
test_title_counts = title_counts[len(train):]
count_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'),
lowercase=True, ngram_range=(1, 2),
max_features=17000)
 desc_counts = count_vectorizer_desc.fit_transform(train['description'].append(test['description']))
train_desc_counts = desc_counts[:len(train)]
test_desc_counts = desc_counts[len(train):]
train_title_counts.shape, train_desc_counts.shape 
<define_variables>
def create_model() :
model = xgb.XGBRegressor(eta=0.01, max_depth=10,n_estimators=2000,
colsample_bytree=0.5,subsample=0.8,
gamma=2, reg_alpha=0, reg_lambda=2, min_child_weight=300,
tree_method='gpu_hist', gpu_id=0, max_bin=2048,
n_jobs=-1)
 return model
target = 'deal_probability'
predictors = [
'num_desc_punct',
'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',
'avg_times_up_user', 'avg_days_up_user', 'n_user_items',
'price', 'item_seq_number'
]
categorical = [
'image_top_1', 'param_1', 'param_2', 'param_3',
'city', 'region', 'category_name', 'parent_category_name', 'user_type'
]
predictors = predictors + categorical<categorify>
%%time
model = create_model()
model.fit(X_train,y_train,eval_set=[(X_train,y_train),(X_validate,y_validate)],eval_metric='rmse',early_stopping_rounds=10 )
for feature in categorical:
print(f'Transforming {feature}...')
 encoder = LabelEncoder()
train[feature].fillna('unknown',inplace=True)
 test[feature].fillna('unknown',inplace=True)
 encoder.fit(train[feature].append(test[feature] ).astype(str))
train[feature] = encoder.transform(train[feature].astype(str))
test[feature] = encoder.transform(test[feature].astype(str))<feature_engineering>
with open(model_path + 'model.pkl', 'wb')as f:
pickle.dump(model, f )
train["price"] = np.log(train["price"]+0.001)
 train["price"].fillna(-999,inplace=True)
 train["image_top_1"].fillna(-999,inplace=True)
 test["price"] = np.log(test["price"]+0.001)
 test["price"].fillna(-999,inplace=True)
 test["image_top_1"].fillna(-999,inplace=True)
<init_hyperparams>
pred_test = model.predict(X_test)
 output = pd.DataFrame({'item_cnt_month':pred_test})
 output.index.name = 'ID'
<define_variables><EOS> 
output = output.clip(0,20)
 output.to_csv('./submission.csv' )