Kaggle competition: Predict Future Sales
print('generating test features...')
 test_dir = '.. /input/test'
test_map = collections.defaultdict(list)
 for fname in tqdm(os.listdir(test_dir)) :
path = test_dir + '/' + fname
df = pd.read_csv(path)
 mfcc = librosa.feature.mfcc(df['acoustic_data'].values.astype('float32'))
mfcc_mean = mfcc.mean(axis=1)
 for i, each_mfcc_mean in enumerate(mfcc_mean):
key = 'mfcc_{}'.format(i)
 if key not in test_map:
test_map[key] = []
test_map[key].append(each_mfcc_mean)
 key = 'seg_id'
if key not in test_map:
test_map[key] = []
test_map[key].append(re.sub('.csv$', '', fname))
test_df = pd.DataFrame(test_map)
 test_csv = 'mfcc_test.csv'
test_df.to_csv(test_csv, index=False)
 print('saved {}'.format(test_csv))
def submit(model, file_path):
X = mfcc_ttf_df.drop(['time_to_failure'], axis=1 ).values
y = mfcc_ttf_df['time_to_failure'].values
model.fit(X, y)
 X_submit = test_df.drop(['seg_id'], axis=1 ).values
y_submit = model.predict(X_submit)
 submit_df = pd.DataFrame({
'seg_id': test_df['seg_id'].values,
'time_to_failure': y_submit
})
 submit_df.to_csv(file_path, index=False)
 print('saved {}'.format(file_path)) 
<set_options>
PATH = 'competitive-data-science-predict-future-sales'
items = pd.read_csv('.. /input/' + PATH + '/items.csv')
 shops = pd.read_csv('.. /input/' + PATH + '/shops.csv')
 cats = pd.read_csv('.. /input/' + PATH + '/item_categories.csv')
 train = pd.read_csv('.. /input/' + PATH + '/sales_train.csv')
 test = pd.read_csv('.. /input/' + PATH + '/test.csv' ).set_index('ID')
 print('------------- train info ------------'); print(train.info() , '
')
 print('------------- test info ------------'); print(test.info() , '
')
 print('------------- items info ------------'); print(items.info() , '
')
 print('------------- shops info ------------'); print(shops.info() , '
')
 print('------------- categories info ------------'); print(cats.info() , '
' )
%load_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
print(train.isna().sum() , '
')
 print(test.isna().sum() )
from fastai.imports import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display
from sklearn import metrics
from sklearn.model_selection import train_test_split
import gc<set_options>
train = train[train.item_price > 0].reset_index(drop=True)
 train[train.item_cnt_day <= 0].item_cnt_day.unique()
train.loc[train.item_cnt_day < 1, 'item_cnt_day'] = 0
def set_plot_sizes(sml, med, big):
plt.rc('font', size=sml)
 plt.rc('axes', titlesize=sml)
 plt.rc('axes', labelsize=med)
 plt.rc('xtick', labelsize=sml)
 plt.rc('ytick', labelsize=sml)
 plt.rc('legend', fontsize=sml)
 plt.rc('figure', titlesize=big)
 def parallel_trees(m, fn, n_jobs=8):
return list(ProcessPoolExecutor(n_jobs ).map(fn, m.estimators_))
def draw_tree(t, df, size=10, ratio=0.6, precision=0):

 s=export_graphviz(t, out_file=None, feature_names=df.columns, filled=True,
special_characters=True, rotate=True, precision=precision)
 IPython.display.display(graphviz.Source(re.sub('Tree {',
f'Tree {{ size={size}; ratio={ratio}', s)))
 def combine_date(years, months=1, days=1, weeks=None, hours=None, minutes=None,
seconds=None, milliseconds=None, microseconds=None, nanoseconds=None):
years = np.asarray(years)- 1970
months = np.asarray(months)- 1
days = np.asarray(days)- 1
types =('<M8[Y]', '<m8[M]', '<m8[D]', '<m8[W]', '<m8[h]',
'<m8[m]', '<m8[s]', '<m8[ms]', '<m8[us]', '<m8[ns]')
 vals =(years, months, days, weeks, hours, minutes, seconds,
milliseconds, microseconds, nanoseconds)
 return sum(np.asarray(v, dtype=t)for t, v in zip(types, vals)
 if v is not None)
 def get_sample(df,n):

 idxs = sorted(np.random.permutation(len(df)) [:n])
 return df.iloc[idxs].copy()
def add_datepart(df, fldname, drop=True, time=False):

 fld = df[fldname]
fld_dtype = fld.dtype
if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
fld_dtype = np.datetime64
if not np.issubdtype(fld_dtype, np.datetime64):
df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)
 targ_pre = re.sub('[Dd]ate$', '', fldname)
 attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',
'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']
if time: attr = attr + ['Hour', 'Minute', 'Second']
for n in attr: df[targ_pre + n] = getattr(fld.dt, n.lower())
 df[targ_pre + 'Elapsed'] = fld.astype(np.int64)// 10 ** 9
if drop: df.drop(fldname, axis=1, inplace=True)
 def is_date(x): return np.issubdtype(x.dtype, np.datetime64)
 def train_cats(df):

 for n,c in df.items() :
if is_string_dtype(c): df[n] = c.astype('category' ).cat.as_ordered()
def apply_cats(df, trn):

 for n,c in df.items() :
if(n in trn.columns)and(trn[n].dtype.name=='category'):
df[n] = pd.Categorical(c, categories=trn[n].cat.categories, ordered=True)
 def fix_missing(df, col, name, na_dict):

 if is_numeric_dtype(col):
if pd.isnull(col ).sum() or(name in na_dict):
df[name+'_na'] = pd.isnull(col)
 filler = na_dict[name] if name in na_dict else col.median()
df[name] = col.fillna(filler)
 na_dict[name] = filler
return na_dict
def numericalize(df, col, name, max_n_cat):

 if not is_numeric_dtype(col)and(max_n_cat is None or len(col.cat.categories)>max_n_cat):
df[name] = col.cat.codes+1
def scale_vars(df, mapper):
warnings.filterwarnings('ignore', category=sklearn.exceptions.DataConversionWarning)
 if mapper is None:
map_f = [([n],StandardScaler())for n in df.columns if is_numeric_dtype(df[n])]
mapper = DataFrameMapper(map_f ).fit(df)
 df[mapper.transformed_names_] = mapper.transform(df)
 return mapper
def proc_df(df, y_fld=None, skip_flds=None, ignore_flds=None, do_scale=False, na_dict=None,
preproc_fn=None, max_n_cat=None, subset=None, mapper=None):

 if not ignore_flds: ignore_flds=[]
if not skip_flds: skip_flds=[]
if subset: df = get_sample(df,subset)
 else: df = df.copy()
ignored_flds = df.loc[:, ignore_flds]
df.drop(ignore_flds, axis=1, inplace=True)
 if preproc_fn: preproc_fn(df)
 if y_fld is None: y = None
else:
if not is_numeric_dtype(df[y_fld]): df[y_fld] = df[y_fld].cat.codes
y = df[y_fld].values
skip_flds += [y_fld]
df.drop(skip_flds, axis=1, inplace=True)
 if na_dict is None: na_dict = {}
else: na_dict = na_dict.copy()
na_dict_initial = na_dict.copy()
for n,c in df.items() : na_dict = fix_missing(df, c, n, na_dict)
 if len(na_dict_initial.keys())> 0:
df.drop([a + '_na' for a in list(set(na_dict.keys())- set(na_dict_initial.keys())) ], axis=1, inplace=True)
 if do_scale: mapper = scale_vars(df, mapper)
 for n,c in df.items() : numericalize(df, c, n, max_n_cat)
 df = pd.get_dummies(df, dummy_na=True)
 df = pd.concat([ignored_flds, df], axis=1)
 res = [df, y, na_dict]
if do_scale: res = res + [mapper]
return res
def rf_feat_importance(m, df):
return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}
).sort_values('imp', ascending=False)
 def set_rf_samples(n):

 forest._generate_sample_indices =(lambda rs, n_samples:
forest.check_random_state(rs ).randint(0, n_samples, n))
def reset_rf_samples() :

 forest._generate_sample_indices =(lambda rs, n_samples:
forest.check_random_state(rs ).randint(0, n_samples, n_samples))
def get_nn_mappers(df, cat_vars, contin_vars):
for v in contin_vars: df[v] = df[v].fillna(df[v].max() +100,)
 for v in cat_vars: df[v].fillna('
cat_maps = [(o, LabelEncoder())for o in cat_vars]
contin_maps = [([o], StandardScaler())for o in contin_vars]
return DataFrameMapper(cat_maps ).fit(df), DataFrameMapper(contin_maps ).fit(df )<define_variables>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 11, 'shop_id'] = 10
test.loc[test.shop_id == 11, 'shop_id'] = 10
train.loc[train.shop_id == 40, 'shop_id'] = 39
test.loc[test.shop_id == 40, 'shop_id'] = 39
PATH_INPUT = "/kaggle/input/"
PATH_WORKING = "/kaggle/working/"
PATH_TMP = "/tmp/"<load_from_csv>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops['category'] = shops['shop_name'].str.split(' ' ).map(lambda x:x[1] ).astype(str)
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
category = ['Орджоникидзе,', 'ТЦ', 'ТРК', 'ТРЦ','ул.', 'Магазин', 'ТК', 'склад']
shops.category = shops.category.apply(lambda x: x if(x in category)else 'etc')
 shops.category.unique()
!wc -l {PATH_INPUT}train.csv; wc -l {PATH_INPUT}test.csv<load_from_csv>
category = ['ТЦ', 'ТРК', 'ТРЦ', 'ТК']
shops.category = shops.category.apply(lambda x: x if(x in category)else 'etc')
 print('Category Distribution', shops.groupby(['category'] ).sum() )
df_raw = pd.read_csv(f'{PATH_INPUT}train.csv', low_memory=False )<load_from_csv>
shops['shop_city'] = shops.city
shops['shop_category'] = shops.category
shops['shop_city'] = LabelEncoder().fit_transform(shops['shop_city'])
 shops['shop_category'] = LabelEncoder().fit_transform(shops['shop_category'])
 shops = shops[['shop_id','shop_city', 'shop_category']]
shops.head()
dftest_raw = pd.read_csv(f'{PATH_INPUT}test.csv', low_memory=False )<categorify>
print(len(cats.item_category_name.unique()))
cats.item_category_name.unique()
df_raw.to_feather(f'{PATH_TMP}train_raw')
 dftest_raw.to_feather(f'{PATH_TMP}test_raw' )<define_variables>
cats['type_code'] = cats.item_category_name.apply(lambda x: x.split(' ')[0] ).astype(str)
 cats.loc[(cats.type_code == 'Игровые')|(cats.type_code == 'Аксессуары'), 'category'] = 'Игры'
cats.loc[cats.type_code == 'PC', 'category'] = 'Музыка'
category = ['Игры', 'Карты', 'Кино', 'Книги','Музыка', 'Подарки', 'Программы', 'Служебные', 'Чистые']
cats['type_code'] = cats.type_code.apply(lambda x: x if(x in category)else 'etc')
 print(cats.groupby(['type_code'] ).sum())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type_code'])
 cats['split'] = cats.item_category_name.apply(lambda x: x.split('-'))
cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
Ids = dftest_raw.Id<define_variables>
items = pd.read_csv('.. /input/' + PATH + '/items.csv')
 items['name_1'], items['name_2'] = items['item_name'].str.split('[', 1 ).str
items['name_1'], items['name_3'] = items['item_name'].str.split('(', 1 ).str
items['name_2'] = items['name_2'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items['name_3'] = items['name_3'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items = items.fillna('0')
 result_1 = Counter(' '.join(items['name_2'].values.tolist() ).split(' ')).items()
result_1 = sorted(result_1, key=itemgetter(1))
result_1 = pd.DataFrame(result_1, columns=['feature', 'count'])
 result_1 = result_1[(result_1['feature'].str.len() > 1)&(result_1['count'] > 200)]
result_2 = Counter(' '.join(items['name_3'].values.tolist() ).split(" ")).items()
result_2 = sorted(result_2, key=itemgetter(1))
result_2 = pd.DataFrame(result_2, columns=['feature', 'count'])
 result_2 = result_2[(result_2['feature'].str.len() > 1)&(result_2['count'] > 200)]
result = pd.concat([result_1, result_2])
 result = result.drop_duplicates(subset=['feature'] ).reset_index(drop=True)
 print('Most common aditional features:', result)
 def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x
items['item_name'] = items['item_name'].apply(lambda x: name_correction(x))
items.name_2 = items.name_2.apply(lambda x: x[:-1] if x != '0' else '0' )
cols=['dependency','edjefe', 'edjefa']<feature_engineering>
items['type'] = items.name_2.apply(lambda x: x[0:8] if x.split(' ')[0] == 'xbox' else x.split(' ')[0])
 items.loc[(items.type == 'x360')|(items.type == 'xbox360'), 'type'] = 'xbox 360'
items.loc[items.type == '', 'type'] = 'mac'
items.type = items.type.apply(lambda x: x.replace(' ',''))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == 'рс'), 'type'] = 'pc'
items.loc[(items.type == 'рs3'), 'type'] = 'ps3'
group_sum = items.groupby('type' ).sum()
group_sum.loc[group_sum.item_category_id < 200]
df_raw[cols] = df_raw[cols].replace(['yes','no'],[1,0])
 df_raw[cols].head()<data_type_conversions>
drop_list = ['5c5', '5c7', '5f4', '6dv', '6jv', '6l6', 'android', 'hm3', 'j72', 'kf6', 'kf7','kg4',
'ps2', 's3v', 's4v'	,'англ', 'русская', 'только', 'цифро']
items.name_2 = items.type.apply(lambda x: 'etc' if x in drop_list else x)
 items = items.drop(['type'], axis=1)
 items.groupby('name_2' ).sum()
df_raw[cols] = df_raw[cols].astype('float64')
 df_raw[cols].dtypes<feature_engineering>
items['name_2'] = LabelEncoder().fit_transform(items['name_2'])
 items['name_3'] = LabelEncoder().fit_transform(items['name_3'])
 items.drop(['item_name', 'name_1'], axis=1, inplace=True)
 items.head()
dftest_raw[cols] = dftest_raw[cols].replace(['yes','no'],[1,0])
 dftest_raw[cols].head()<data_type_conversions>
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts
dftest_raw[cols] = dftest_raw[cols].astype('float64')
 dftest_raw[cols].dtypes<concatenate>
train['revenue'] = train['item_price'] * train['item_cnt_day']
train_cats(df_raw )<concatenate>
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts
apply_cats(dftest_raw, df_raw )<prepare_x_and_y>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )
df, y, nas = proc_df(df_raw, 'Target', skip_flds=['Id', 'idhogar'] )<create_dataframe>
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts
dftest, _, _ = proc_df(dftest_raw, skip_flds=['Id', 'idhogar'], na_dict=nas )<compute_train_metric>
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['shop_city'] = matrix['shop_city'].astype(np.int8)
 matrix['shop_category'] = matrix['shop_category'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts
def print_score(m):
res = [metrics.f1_score(y_train, m.predict(X_train), average='macro'), metrics.f1_score(y_valid, m.predict(X_valid), average='macro')]
print(res )<split>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
X_train, X_valid, y_train, y_valid = train_test_split(df, y, test_size=0.2)
 X_train.shape, X_valid.shape, y_train.shape, y_valid.shape<train_model>
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
m = RandomForestClassifier(n_estimators=100, max_features=0.5, n_jobs=-1, class_weight='balanced')
 m.fit(X_train, y_train)
 print_score(m )<prepare_x_and_y>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
df, y, nas = proc_df(df_raw[df_raw.parentesco1==1], 'Target', skip_flds=['Id', 'idhogar'] )<create_dataframe>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
dftest, _, _ = proc_df(dftest_raw, skip_flds=['Id', 'idhogar'], na_dict=nas )<split>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
X_train, X_valid, y_train, y_valid = train_test_split(df, y, test_size=0.2)
 X_train.shape, X_valid.shape, y_train.shape, y_valid.shape<train_model>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
m = RandomForestClassifier(n_estimators=100, n_jobs=-1, class_weight='balanced')
 m.fit(X_train, y_train)
 print_score(m )<compute_test_metric>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'itme_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_item_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'itme_id'], how='left')
 matrix['date_shop_item_avg_item_cnt'] = matrix['date_shop_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_item_avg_item_cnt')
 matrix.drop(['date_shop_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
print(metrics.confusion_matrix(y_valid, m.predict(X_valid)) )<compute_test_metric>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
print(metrics.classification_report(y_valid, m.predict(X_valid)) )<train_model>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_city'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
m = RandomForestClassifier(n_estimators=120, max_features='sqrt', min_samples_leaf=3, n_jobs=-1, class_weight='balanced_subsample')
 m.fit(X_train, y_train)
 print_score(m )<compute_test_metric>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'shop_city'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
fi = rf_feat_importance(m, df); fi[:10]<train_model>
m = RandomForestClassifier(n_estimators=120, max_features="sqrt", min_samples_leaf=3, n_jobs=-1, class_weight='balanced_subsample')
 m.fit(X_train, y_train)
 print_score(m )<predict_on_test>
ts = time.time()
group = matrix.groupby(['date_block_num','shop_id' ,'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id' ,'item_id'], how='left')
 matrix['date_shop_item_avg_item_cnt'] = matrix['date_shop_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_item_avg_item_cnt')
 matrix.drop(['date_shop_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
preds = m.predict(dftest[cols_keep] )<save_to_csv>
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts
submission = pd.DataFrame({"Id": Ids, "Target": preds})
 submission.to_csv(f'{PATH_WORKING}submission.csv', index=False )<set_options>
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
pd.set_option('max_colwidth',1000)
 pd.options.display.max_info_columns = 999
pd.set_option('expand_frame_repr',True)
<load_from_csv>
matrix['month'] = matrix['date_block_num'] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )
train = pd.read_csv('.. /input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('.. /input/costa-rican-household-poverty-prediction/test.csv')
 meta = pd.read_csv('.. /input/metadata/Metadata.csv',index_col='variable', header=0)
 print(f'Train data:{train.shape}, Test data: {test.shape}, Metadata: {meta.shape}' )<count_duplicates>
print(f'Train duplicates: {len(train[train.duplicated() ])}
Test Duplicates: {len(test[test.duplicated() ])}' )<feature_engineering>
test.loc[test['rez_esc'] == 99.0 , 'rez_esc'] = 5<count_missing_values>
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts
print(f'Train:
{train.isnull().sum() [train.isnull().sum() >0]}
')
 print(f'Test:
{test.isnull().sum() [test.isnull().sum() >0]}
')
 pd.testing.assert_series_equal(train.isnull().any() [train.isnull().any() ==True],test.isnull().any() [test.isnull().any() ==True])
 null_list = list(train.isnull().any() [train.isnull().any() ==True].index)
 print(f'Columns with Null values: {null_list}
')
 print('Definitions:')
 for item in null_list:
print(f'{item}
{meta.loc[item]}
' )<concatenate>
ts = time.time()
matrix = matrix[matrix.date_block_num > 3]
time.time() - ts
train_n = train
test_n = test
name = [item + '_null' for item in null_list[:3]]
i = 0
for item in null_list[:3]:
train_n[item].fillna(0,inplace=True)
 test_n[item].fillna(0,inplace=True)
 i+=1
pd.concat([train_n.head() , test_n.head() ], sort=False )<drop_column>
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts
def shift_cols_last(df, col_index):

 cols = df.columns.tolist()
i=0
for item in col_index:
if i==0:
new_cols = cols[:item] + cols[item+1:] + [cols[item]]
elif i>=1:
new_cols = new_cols[:item] + new_cols[item+1:] + [cols[item]]
i+=1
return df[new_cols]<concatenate>
matrix.to_pickle('.. /working/data.pkl')
 del matrix
gc.collect() ;
pd.concat([train_n.head() , test_n.head() ], sort=False )<data_type_conversions>
rcParams['figure.figsize'] = 12, 4
train_n = train_n.astype('float64', errors= 'ignore')
 test_n = test_n.astype('float64', errors= 'ignore' )<train_model>
data = pd.read_pickle('.. /working/data.pkl')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 print(len(data.columns))
data.columns
print('Train')
 print(train_n.info() ,'
')
 print('Test')
 print(test_n.info() )<data_type_conversions>
data = data[[
'date_block_num', 'shop_id', 'item_id', 'item_cnt_month', 'shop_city',
'shop_category', 'item_category_id', 'name_2', 'name_3', 'type_code',
'subtype_code', 'item_cnt_month_lag_1', 'item_cnt_month_lag_2',
'item_cnt_month_lag_3', 'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1', 'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3', 'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2', 'date_shop_avg_item_cnt_lag_3',
'date_cat_avg_item_cnt_lag_1', 'date_shop_cat_avg_item_cnt_lag_1',
'date_shop_subtype_avg_item_cnt_lag_1', 'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'date_shop_item_avg_item_cnt_lag_1',
'delta_price_lag',
'delta_revenue_lag_1', 'month', 'days',
'item_shop_first_sale',
'item_first_sale'
]]
len(data.columns )
sqb_null_trls = train_n[train_n['SQBmeaned'].isnull() ].index.tolist()
sqb_null_tels = test_n[test_n['SQBmeaned'].isnull() ].index.tolist()
print(f'Train SQBmeaned Null rows
{sqb_null_trls}
Test SQBmeaned Null rows
{sqb_null_tels}' )<feature_engineering>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
for item in sqb_null_trls:
train_n.loc[item,'SQBmeaned']=train_n.loc[item,'meaneduc']**2
for item in sqb_null_tels:
test_n.loc[item,'SQBmeaned']=test_n.loc[item,'meaneduc']**2
pd.concat([train_n.loc[sqb_null_trls, ['Id','meaneduc', 'SQBmeaned', 'Target']],
test_n.loc[sqb_null_tels, ['Id','meaneduc', 'SQBmeaned']]], sort=False )<train_model>
del data
gc.collect() ;
print(f'Train data remaining NAs: {len(train_n[train_n.isnull().any(axis=1)])}')
 print(f'Test data remaining NAs: {len(test_n[test_n.isnull().any(axis=1)])}' )<define_variables>
ts = time.time()
model = XGBRegressor(
 max_depth=10,
n_estimators=1000,
min_child_weight=0.5,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
tree_method='gpu_hist',
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 20)
 time.time() - ts
<define_variables><EOS> 
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))